diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8aaf6f7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.venv +.pytest_cache +workspace +acceptance_runs +qa +docs +tests +**/__pycache__ +**/*.pyc +*.tgz diff --git a/deploy/poc/app-template.yaml b/deploy/poc/app-template.yaml new file mode 100644 index 0000000..c7e412a --- /dev/null +++ b/deploy/poc/app-template.yaml @@ -0,0 +1,324 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: real2sim-runtime-config + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +data: + config.toml: | + app_root = "/app" + workspace_dir = "/var/lib/real2sim-poc/workspace" + host = "0.0.0.0" + port = 18126 + default_renderer = "spark" + real2sim2model_root = "/app" + expert_policy_root = "/app" + training_root = "/app" + inference_root = "/app" + discoverse_root = "/opt/DISCOVERSE" + gs_playground_root = "/opt/gs_playground" + images23dgs_root = "/opt/images23dgs" + colmap_binary = "/usr/bin/colmap" + gsplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python" + runtime_root = "/opt/real2sim_paper_runtime" + anysplat_root = "/opt/real2sim_paper_runtime/AnySplat" + anysplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python" + cuda_home = "/usr/local/cuda" + runtime_env_script = "/opt/gs_playground/scripts/real2sim_wuying_paper_env.sh" + allowed_path_roots = "/var/lib/real2sim-poc:/opt/DISCOVERSE:/opt/gs_playground:/opt/motrixsim-docs:/opt/real2sim_paper_runtime:/tmp" + production_mode = true + auth_enabled = true + secure_cookies = true + session_ttl_seconds = 43200 + discoverse_commit = "d67f47c084aba0e0cf422a8725235f8b9238655a" + act_commit = "f8ab39c8d829d85571ab6d0250412ba4d0698fa0" + anysplat_commit = "5f5e208a7dd57d52e43ea0d553a95eab526e8775" + anysplat_model_revision = "d2e8c343672646041ad4ea518184968f94362f01" + vggt_model_revision = "860abec7937da0a4c03c41d3c269c366e82abdf9" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: real2sim-api + namespace: real2sim-poc + labels: + app.kubernetes.io/name: real2sim-api + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/version: "__BUILD_ID__" +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: + app.kubernetes.io/name: real2sim-api + template: + metadata: + labels: + app.kubernetes.io/name: real2sim-api + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/version: "__BUILD_ID__" + spec: + serviceAccountName: real2sim-poc + automountServiceAccountToken: false + nodeSelector: + kubernetes.io/hostname: "192.168.80.136" + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + imagePullSecrets: + - name: real2sim-registry + containers: + - name: api + image: __API_IMAGE__ + imagePullPolicy: IfNotPresent + env: + - {name: REAL2SIM_PLATFORM_CONFIG, value: /etc/real2sim/config.toml} + - {name: REAL2SIM_WORKER_MODE, value: api} + - {name: REAL2SIM_WORKER_ID, value: real2sim-worker-a800} + - {name: REAL2SIM_PRODUCTION_MODE, value: "1"} + - {name: REAL2SIM_AUTH_ENABLED, value: "1"} + - {name: REAL2SIM_SECURE_COOKIES, value: "1"} + - {name: REAL2SIM_BOS_PREFIX, value: real2sim-poc} + - {name: BOS_BUCKET, value: testxyz} + - {name: BOS_ENDPOINT, value: https://bj.bcebos.com} + - name: BOS_ACCESS_KEY_ID + valueFrom: {secretKeyRef: {name: real2sim-bos, key: accessKeyId}} + - name: BOS_SECRET_ACCESS_KEY + valueFrom: {secretKeyRef: {name: real2sim-bos, key: secretAccessKey}} + - name: REAL2SIM_BOOTSTRAP_ADMIN_PASSWORD + valueFrom: {secretKeyRef: {name: real2sim-app-secrets, key: bootstrap-admin-password}} + ports: + - {name: http, containerPort: 18126} + resources: + requests: {cpu: 500m, memory: 1Gi} + limits: {cpu: "4", memory: 8Gi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + readinessProbe: + httpGet: {path: /api/health, port: http} + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + httpGet: {path: /api/health, port: http} + initialDelaySeconds: 20 + periodSeconds: 20 + volumeMounts: + - {name: data, mountPath: /var/lib/real2sim-poc} + - {name: config, mountPath: /etc/real2sim/config.toml, subPath: config.toml, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: data + persistentVolumeClaim: {claimName: real2sim-poc-data} + - name: config + configMap: {name: real2sim-runtime-config} + - name: tmp + emptyDir: {sizeLimit: 2Gi} +--- +apiVersion: v1 +kind: Service +metadata: + name: real2sim-api + namespace: real2sim-poc + labels: + app.kubernetes.io/name: real2sim-api + app.kubernetes.io/part-of: real2sim-poc +spec: + selector: + app.kubernetes.io/name: real2sim-api + ports: + - {name: http, port: 18126, targetPort: http} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: real2sim-worker + namespace: real2sim-poc + labels: + app.kubernetes.io/name: real2sim-worker + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/version: "__BUILD_ID__" +spec: + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: + app.kubernetes.io/name: real2sim-worker + template: + metadata: + labels: + app.kubernetes.io/name: real2sim-worker + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/version: "__BUILD_ID__" + spec: + serviceAccountName: real2sim-poc + automountServiceAccountToken: false + nodeSelector: + kubernetes.io/hostname: "192.168.80.136" + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + imagePullSecrets: + - name: real2sim-registry + initContainers: + - name: runtime-gate + image: __WORKER_IMAGE__ + imagePullPolicy: IfNotPresent + command: [real2sim-runtime-gate, --mode, bootstrap] + env: &worker-env + - {name: REAL2SIM_PLATFORM_CONFIG, value: /etc/real2sim/config.toml} + - {name: REAL2SIM_WORKER_MODE, value: worker} + - {name: REAL2SIM_WORKER_ID, value: real2sim-worker-a800} + - {name: REAL2SIM_PRODUCTION_MODE, value: "1"} + - {name: REAL2SIM_BOS_PREFIX, value: real2sim-poc} + - {name: REAL2SIM_RUNTIME_GATE_DIR, value: /var/lib/real2sim-poc/cache/runtime-gate} + - {name: HF_HOME, value: /var/lib/real2sim-poc/cache/huggingface} + - {name: HF_HUB_DISABLE_XET, value: "1"} + - {name: BOS_BUCKET, value: testxyz} + - {name: BOS_ENDPOINT, value: https://bj.bcebos.com} + - name: BOS_ACCESS_KEY_ID + valueFrom: {secretKeyRef: {name: real2sim-bos, key: accessKeyId}} + - name: BOS_SECRET_ACCESS_KEY + valueFrom: {secretKeyRef: {name: real2sim-bos, key: secretAccessKey}} + resources: + requests: {cpu: "6", memory: 32Gi, nvidia.com/gpu: "1"} + limits: {cpu: "24", memory: 96Gi, nvidia.com/gpu: "1"} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: &worker-mounts + - {name: data, mountPath: /var/lib/real2sim-poc} + - {name: config, mountPath: /etc/real2sim/config.toml, subPath: config.toml, readOnly: true} + - {name: tmp, mountPath: /tmp} + - {name: shm, mountPath: /dev/shm} + containers: + - name: worker + image: __WORKER_IMAGE__ + imagePullPolicy: IfNotPresent + env: *worker-env + resources: + requests: {cpu: "6", memory: 32Gi, nvidia.com/gpu: "1"} + limits: {cpu: "24", memory: 96Gi, nvidia.com/gpu: "1"} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + readinessProbe: + exec: {command: [real2sim-runtime-gate, --mode, readiness]} + initialDelaySeconds: 5 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 4 + livenessProbe: + exec: {command: [real2sim-runtime-gate, --mode, readiness]} + initialDelaySeconds: 45 + periodSeconds: 60 + timeoutSeconds: 15 + failureThreshold: 3 + volumeMounts: *worker-mounts + volumes: + - name: data + persistentVolumeClaim: {claimName: real2sim-poc-data} + - name: config + configMap: {name: real2sim-runtime-config} + - name: tmp + emptyDir: {sizeLimit: 12Gi} + - name: shm + emptyDir: {medium: Memory, sizeLimit: 12Gi} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: real2sim-frontend + namespace: real2sim-poc + labels: + app.kubernetes.io/name: real2sim-frontend + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/version: "__BUILD_ID__" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: real2sim-frontend + template: + metadata: + labels: + app.kubernetes.io/name: real2sim-frontend + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/version: "__BUILD_ID__" + spec: + serviceAccountName: real2sim-poc + automountServiceAccountToken: false + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: ["192.168.80.249", "192.168.81.47"] + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + seccompProfile: {type: RuntimeDefault} + imagePullSecrets: + - name: real2sim-registry + containers: + - name: frontend + image: __FRONTEND_IMAGE__ + imagePullPolicy: IfNotPresent + ports: + - {name: http, containerPort: 8080} + - {name: https, containerPort: 8443} + resources: + requests: {cpu: 100m, memory: 128Mi} + limits: {cpu: "1", memory: 512Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + readinessProbe: + tcpSocket: {port: https} + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + tcpSocket: {port: https} + initialDelaySeconds: 15 + periodSeconds: 20 + volumeMounts: + - {name: tls, mountPath: /etc/real2sim/tls, readOnly: true} + - {name: tmp, mountPath: /tmp} + - {name: nginx-cache, mountPath: /var/cache/nginx} + volumes: + - name: tls + secret: {secretName: real2sim-tls} + - name: tmp + emptyDir: {sizeLimit: 64Mi} + - name: nginx-cache + emptyDir: {sizeLimit: 128Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: real2sim-frontend + namespace: real2sim-poc + labels: + app.kubernetes.io/name: real2sim-frontend + app.kubernetes.io/part-of: real2sim-poc +spec: + type: LoadBalancer + selector: + app.kubernetes.io/name: real2sim-frontend + ports: + - {name: http, port: 80, targetPort: http} + - {name: https, port: 443, targetPort: https} diff --git a/deploy/poc/bootstrap.yaml b/deploy/poc/bootstrap.yaml new file mode 100644 index 0000000..8b35785 --- /dev/null +++ b/deploy/poc/bootstrap.yaml @@ -0,0 +1,184 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: real2sim-poc-local-pv + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + capacity: + storage: 150Gi + volumeMode: Filesystem + accessModes: [ReadWriteOnce] + persistentVolumeReclaimPolicy: Retain + storageClassName: real2sim-poc-local + local: + path: /var/lib/real2sim-poc + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: ["192.168.80.136"] +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: real2sim-poc-data + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + accessModes: [ReadWriteOnce] + storageClassName: real2sim-poc-local + volumeName: real2sim-poc-local-pv + resources: + requests: + storage: 150Gi +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: real2sim-poc-quota + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + hard: + pods: "12" + services: "6" + services.loadbalancers: "1" + requests.cpu: "40" + limits.cpu: "64" + requests.memory: 128Gi + limits.memory: 192Gi + requests.storage: 150Gi + requests.nvidia.com/gpu: "1" + limits.nvidia.com/gpu: "1" +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: real2sim-poc-limits + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + limits: + - type: Container + defaultRequest: + cpu: 100m + memory: 128Mi + default: + cpu: "2" + memory: 4Gi +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: real2sim-poc + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +automountServiceAccountToken: false +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + podSelector: {} + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-dns + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + podSelector: {} + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: frontend-policy + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: real2sim-frontend + policyTypes: [Ingress, Egress] + ingress: + - ports: + - {protocol: TCP, port: 8080} + - {protocol: TCP, port: 8443} + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: real2sim-api + ports: + - {protocol: TCP, port: 18126} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: api-policy + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: real2sim-api + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: real2sim-frontend + ports: + - {protocol: TCP, port: 18126} + egress: + - ports: + - {protocol: TCP, port: 443} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: worker-policy + namespace: real2sim-poc + labels: + app.kubernetes.io/part-of: real2sim-poc +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: real2sim-worker + policyTypes: [Ingress, Egress] + egress: + - ports: + - {protocol: TCP, port: 443} + - {protocol: TCP, port: 80} diff --git a/deploy/poc/nginx.conf b/deploy/poc/nginx.conf new file mode 100644 index 0000000..3e4492c --- /dev/null +++ b/deploy/poc/nginx.conf @@ -0,0 +1,82 @@ +worker_processes auto; +pid /tmp/nginx.pid; + +events { worker_connections 1024; } + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + access_log /dev/stdout; + error_log /dev/stderr warn; + sendfile on; + server_tokens off; + client_max_body_size 70m; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + upstream real2sim_api { server real2sim-api.real2sim-poc.svc.cluster.local:18126; } + + server { + listen 8080; + return 308 https://$host$request_uri; + } + + server { + listen 8443 ssl; + ssl_certificate /etc/real2sim/tls/tls.crt; + ssl_certificate_key /etc/real2sim/tls/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:TLS:10m; + add_header X-Content-Type-Options nosniff always; + add_header Referrer-Policy same-origin always; + add_header X-Frame-Options SAMEORIGIN always; + + root /usr/share/nginx/html; + index index.html; + + location = /api/events/tasks { + proxy_pass http://real2sim_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 1h; + } + + location /api/ { + proxy_pass http://real2sim_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_request_buffering off; + proxy_read_timeout 7200s; + } + + location /datasets/ { + proxy_pass http://real2sim_api; + proxy_set_header Host $host; + } + + location ~ ^/static/.*\.(html|js|css)$ { + try_files $uri =404; + expires -1; + add_header Cache-Control "no-store"; + } + + location /static/ { + try_files $uri =404; + expires 1h; + add_header Cache-Control "public, max-age=3600"; + } + + location / { + try_files $uri $uri/ /index.html; + add_header Cache-Control "no-store"; + } + } +} diff --git a/docker/anysplat-runtime-requirements.txt b/docker/anysplat-runtime-requirements.txt new file mode 100644 index 0000000..a33e28d --- /dev/null +++ b/docker/anysplat-runtime-requirements.txt @@ -0,0 +1,45 @@ +numpy==1.25.0 +wheel +tqdm +lightning +hydra-core +jaxtyping +beartype +einops +colorama +scikit-image +colorspacious +matplotlib +moviepy==1.0.3 +imageio +timm +dacite +lpips +e3nn +plyfile +tabulate +svg.py +scikit-video +opencv-python-headless==4.8.1.78 +Pillow +huggingface_hub +pydantic +open3d +safetensors +omegaconf +pyquaternion +pyyaml +packaging +h5py +dm_env +OneEuroFilter +pymeshlab +scipy +psutil +screeninfo +mediapy +termcolor +imagecodecs +av +wandb +trimesh diff --git a/docker/api.Dockerfile b/docker/api.Dockerfile new file mode 100644 index 0000000..8b78ce3 --- /dev/null +++ b/docker/api.Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim-bookworm + +ARG BUILD_ID=dev +ARG SOURCE_COMMIT=unknown +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \ + REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT} + +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl ffmpeg libglib2.0-0 && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src ./src +RUN python -m pip install --no-cache-dir --upgrade pip && python -m pip install --no-cache-dir . +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin real2sim +USER 10001:10001 +EXPOSE 18126 +CMD ["real2sim-platform", "--host", "0.0.0.0", "--port", "18126"] diff --git a/docker/frontend.Dockerfile b/docker/frontend.Dockerfile new file mode 100644 index 0000000..c60da10 --- /dev/null +++ b/docker/frontend.Dockerfile @@ -0,0 +1,10 @@ +FROM nginxinc/nginx-unprivileged:1.27-alpine + +ARG BUILD_ID=dev +LABEL org.opencontainers.image.title="Real2Sim POC frontend" \ + org.opencontainers.image.version="${BUILD_ID}" +COPY deploy/poc/nginx.conf /etc/nginx/nginx.conf +COPY src/real2sim_platform/static/index.html /usr/share/nginx/html/index.html +COPY src/real2sim_platform/static /usr/share/nginx/html/static +USER 101:101 +EXPOSE 8080 8443 diff --git a/docker/worker-constraints.txt b/docker/worker-constraints.txt new file mode 100644 index 0000000..dbb17b6 --- /dev/null +++ b/docker/worker-constraints.txt @@ -0,0 +1,2 @@ +numpy==1.25.0 +opencv-python-headless==4.8.1.78 diff --git a/docker/worker.Dockerfile b/docker/worker.Dockerfile new file mode 100644 index 0000000..bafc641 --- /dev/null +++ b/docker/worker.Dockerfile @@ -0,0 +1,72 @@ +FROM pytorch/pytorch:2.2.0-cuda12.1-cudnn8-runtime + +ARG BUILD_ID=dev +ARG SOURCE_COMMIT=unknown +ARG ANYSPLAT_COMMIT=5f5e208a7dd57d52e43ea0d553a95eab526e8775 +ARG DISCOVERSE_COMMIT=d67f47c084aba0e0cf422a8725235f8b9238655a +ARG ACT_COMMIT=f8ab39c8d829d85571ab6d0250412ba4d0698fa0 +ARG MOTRIXSIM_DOCS_COMMIT=2beecea67c81d25e29cefa30a80c0ddad7b44dde +ARG VGGT_MODEL_REVISION=860abec7937da0a4c03c41d3c269c366e82abdf9 + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \ + REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT} \ + REAL2SIM_VGGT_MODEL_REVISION=${VGGT_MODEL_REVISION} \ + REAL2SIM_ANYSPLAT_ROOT=/opt/real2sim_paper_runtime/AnySplat \ + REAL2SIM_ANY_SPLAT_PYTHON=/opt/real2sim_paper_runtime/envs/anysplat/bin/python \ + REAL2SIM_POSE_INIT_TRAIN_PYTHON=/opt/real2sim_paper_runtime/envs/anysplat/bin/python \ + CUDA_HOME=/usr/local/cuda \ + PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat:/opt/DISCOVERSE/policies/act:/opt/DISCOVERSE:/opt/gs_playground + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl ffmpeg git build-essential cmake ninja-build pkg-config colmap \ + libgl1 libglib2.0-0 libgomp1 libx11-6 libxext6 libsm6 && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src ./src +COPY scripts ./scripts +COPY vendor/gs_playground /opt/gs_playground +COPY docker/anysplat-runtime-requirements.txt /tmp/anysplat-runtime-requirements.txt +COPY docker/worker-constraints.txt /tmp/worker-constraints.txt + +RUN python -m pip install --no-cache-dir --upgrade pip setuptools wheel && \ + python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt . && \ + python -m pip install --no-cache-dir torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121 && \ + python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt -r /tmp/anysplat-runtime-requirements.txt && \ + python -m pip install --no-cache-dir xformers==0.0.24 --no-deps && \ + python -m pip install --no-cache-dir torch-scatter==2.1.2 -f https://data.pyg.org/whl/torch-2.2.0+cu121.html && \ + python -m pip install --no-cache-dir 'https://github.com/nerfstudio-project/gsplat/releases/download/v1.4.0/gsplat-1.4.0%2Bpt22cu121-cp310-cp310-linux_x86_64.whl' && \ + python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt motrixsim-core==0.8.0 + +RUN git clone --filter=blob:none --no-checkout https://github.com/InternRobotics/AnySplat.git /opt/real2sim_paper_runtime/AnySplat && \ + git -C /opt/real2sim_paper_runtime/AnySplat checkout "${ANYSPLAT_COMMIT}" && \ + test "$(git -C /opt/real2sim_paper_runtime/AnySplat rev-parse HEAD)" = "${ANYSPLAT_COMMIT}" && \ + sed -i 's/VGGT.from_pretrained("facebook\/VGGT-1B")/VGGT.from_pretrained("facebook\/VGGT-1B", revision=os.environ.get("REAL2SIM_VGGT_MODEL_REVISION"))/g' /opt/real2sim_paper_runtime/AnySplat/src/model/encoder/anysplat.py + +RUN git clone --filter=blob:none --no-checkout https://github.com/eust-w/DISCOVERSE.git /opt/DISCOVERSE && \ + git -C /opt/DISCOVERSE checkout "${DISCOVERSE_COMMIT}" && \ + git -C /opt/DISCOVERSE submodule update --init policies/act && \ + test "$(git -C /opt/DISCOVERSE rev-parse HEAD)" = "${DISCOVERSE_COMMIT}" && \ + test "$(git -C /opt/DISCOVERSE/policies/act rev-parse HEAD)" = "${ACT_COMMIT}" && \ + python -m pip install --no-cache-dir --no-deps -e /opt/DISCOVERSE + +RUN git clone --filter=blob:none --no-checkout https://github.com/Motphys/motrixsim-docs.git /opt/motrixsim-docs && \ + git -C /opt/motrixsim-docs checkout "${MOTRIXSIM_DOCS_COMMIT}" && \ + test "$(git -C /opt/motrixsim-docs rev-parse HEAD)" = "${MOTRIXSIM_DOCS_COMMIT}" && \ + mkdir -p /opt/real2sim_paper_runtime/envs/anysplat/bin && \ + ln -s /opt/conda/bin/python /opt/real2sim_paper_runtime/envs/anysplat/bin/python && \ + ln -s /opt/conda/bin/pip /opt/real2sim_paper_runtime/envs/anysplat/bin/pip + +RUN python -c 'import mujoco, motrixsim, torch; from huggingface_hub import snapshot_download; assert torch.__version__.startswith("2.2")' && \ + PYTHONPATH=/opt/DISCOVERSE/policies/act:/opt/DISCOVERSE python -c 'from policy import ACTPolicy; assert ACTPolicy is not None' && \ + PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat python -c 'from src.model.model.anysplat import AnySplat; assert AnySplat is not None' && \ + test -f /opt/gs_playground/scripts/real2sim_video_motrix_scene.py + +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin real2sim && \ + chown -R 10001:10001 /app +USER 10001:10001 +CMD ["real2sim-worker"] diff --git a/docs/POC_PRODUCT_SPEC.md b/docs/POC_PRODUCT_SPEC.md new file mode 100644 index 0000000..6979880 --- /dev/null +++ b/docs/POC_PRODUCT_SPEC.md @@ -0,0 +1,45 @@ +# Real2Sim POC 产品与交互规格 + +## 用户与边界 + +| 角色 | 主要目标 | 写权限 | +| --- | --- | --- | +| 管理员 | 初始化平台、管理用户、存储和删除策略 | 全部;删除磁盘或 BOS 对象必须二次确认 | +| 算法工程师 | 导入数据、重建、采集、训练和推理 | 业务资源和任务,不可管理用户 | +| 只读用户 | 查看场景、报告、模型和 rollout | 无 | + +平台面向内部 POC/Beta,不开放注册、计费或多组织租户。真实 MuJoCo 和 MotrixSim 是允许的领域运行时;禁止 mock、dry-run 验收、硬编码成功率、假进度和通过直接改物体位姿伪造抓取。 + +## 完整用户旅程 + +1. 管理员使用一次性初始密码登录,强制更新密码,再创建工程师或只读账号。 +2. 工程师从 HuggingFace、浏览器分片上传或授权 BOS 前缀导入数据。系统计算 SHA-256、创建版本和清单。 +3. 视频或图片包进入“准备输入”,统一生成 prepared frames;质量预检给出帧数、分辨率、清晰度、视差和可执行建议。 +4. 场景重建向导选择 prepared dataset、质量档位、坐标系、渲染器和仿真器。任务立即进入本页,SSE 显示真实阶段和进度。 +5. 重建成功后在场景管理中打开 Spark/Three Viewer,添加机器人或物体会生成新 revision,不覆盖来源场景。 +6. 数据采集页只展示场景 manifest 声明兼容的真实控制器。RM65 采集要求双指接触、至少抬升 10 cm、放置误差不超过 5 cm。 +7. 模型训练页只允许新建 DISCOVERSE ACT;配置由 HDF5 和场景 manifest 推导,输出 checkpoint、损失历史、模型卡和血缘。 +8. 在线推理在真实引擎运行随机化 episode,输出视频、轨迹、动作统计和成功率。没有真实结果时显示空状态。 + +## 七页交互合同 + +- **总览**:当前闭环阶段、推荐下一步、真实任务、最近场景和模型结果。无运维状态条或调试 JSON。 +- **数据管理**:来源卡打开导入弹窗;表格支持全选、多选、批量准备、预检和删除索引。 +- **场景重建**:资源卡选择、质量评分、参数折叠、任务列表和完成后预览构成单页闭环。 +- **场景管理**:场景库、3D Viewer、资产和仿真信息三栏;移动端按 Viewer、场景库、资产编辑顺序堆叠。 +- **数据采集**:场景、控制器和采集规模在同页;任务完成后直接出现 HDF5 数据集。 +- **模型训练**:采集数据、ACT 配方、真实指标、训练任务和模型卡同页呈现。 +- **在线推理**:模型和场景卡选择、rollout 播放、轨迹产物与任务管理同页呈现。 + +任务详情使用右侧抽屉展示阶段、报告、建议、产物预览和下载;高级 JSON 仅在折叠区。删除默认只删平台索引,不删 PV 或 BOS 对象。 + +## 可行性证据与交付门槛 + +- 本地已用真实 MuJoCo 和 MotrixSim 加载同一 RM65 MJCF 并执行 step。 +- 本地真实物理控制器已完成双指接触抓放:抬升 15.57 cm,放置误差 1.29 cm,输出 RGB/qpos/action/contact HDF5 与 MP4。 +- 集群 worker 必须在 A800 上通过 CUDA 矩阵运算、AnySplat 固定 revision 模型加载、ACT import、MuJoCo step 和 MotrixSim step,之后才 Ready。 +- 最终 `acceptance_report.json.pass=true` 还要求两次非 dry-run 重建、50 个成功采集 episode、真实 ACT checkpoint,以及 10 次推理成功率至少 50%。未达门槛时可保留 Beta 部署,但不得声明全链路通过。 + +## 视觉合同 + +视觉原型保存于 `docs/prototypes/real2sim-poc-product-board-v1.png`。实现采用深色中性底、青绿色命令色、紧凑表格、6px 面板圆角和纯文字导航;禁止大编号、运行时状态条、装饰图标、裸资源 ID/路径主输入及无数据时的虚构图表。 diff --git a/pyproject.toml b/pyproject.toml index a7a9be7..9521f14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,8 @@ version = "0.1.0" description = "Real2Sim scene, collection, training, and inference management platform" requires-python = ">=3.10" dependencies = [ + "argon2-cffi>=23.1", + "bce-python-sdk>=0.9.29", "fastapi>=0.110", "huggingface_hub>=0.24", "imageio>=2.34", @@ -22,6 +24,8 @@ dependencies = [ [project.scripts] real2sim-platform = "real2sim_platform.server:main" +real2sim-worker = "real2sim_platform.worker_main:main" +real2sim-runtime-gate = "real2sim_platform.runtime_gate:main" [tool.setuptools.packages.find] where = ["src"] @@ -36,3 +40,6 @@ where = ["src"] "static/vendor/three/addons/postprocessing/*", "static/vendor/three/addons/utils/*", ] + +[tool.pytest.ini_options] +markers = ["integration: exercises installed real runtimes and physical scene assets"] diff --git a/scripts/deploy_poc_cluster.sh b/scripts/deploy_poc_cluster.sh new file mode 100755 index 0000000..d1a48d9 --- /dev/null +++ b/scripts/deploy_poc_cluster.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +KUBECONFIG_PATH="${REAL2SIM_POC_KUBECONFIG:-/Users/d-robotics/Downloads/poc.config}" +NAMESPACE="real2sim-poc" +GPU_NODE="${REAL2SIM_POC_GPU_NODE:-192.168.80.136}" +REGISTRY="${REAL2SIM_POC_REGISTRY:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc}" +SOURCE_COMMIT="$(git -C "$ROOT" rev-parse --short=12 HEAD)" +BUILD_ID="${REAL2SIM_POC_BUILD_ID:-${SOURCE_COMMIT}-$(date -u +%Y%m%d%H%M%S)}" +OUT="${REAL2SIM_POC_ACCEPTANCE_OUT:-$ROOT/acceptance_runs/poc/$BUILD_ID}" +BUILD_IMAGES=1 +DEPLOY_APP=1 +tls_dir="" +context_archive="" +rendered_manifest="" +trap 'rm -rf "${tls_dir:-}" "${context_archive:-}" "${rendered_manifest:-}"' EXIT + +for arg in "$@"; do + case "$arg" in + --skip-build) BUILD_IMAGES=0 ;; + --build-only) DEPLOY_APP=0 ;; + --help) + echo "usage: $0 [--skip-build|--build-only]" + exit 0 + ;; + *) echo "unknown argument: $arg" >&2; exit 2 ;; + esac +done + +K=(kubectl --kubeconfig "$KUBECONFIG_PATH") +mkdir -p "$OUT" + +require() { command -v "$1" >/dev/null || { echo "missing required command: $1" >&2; exit 1; }; } +for command in kubectl jq openssl tar git sed; do require "$command"; done +test -f "$KUBECONFIG_PATH" || { echo "kubeconfig not found: $KUBECONFIG_PATH" >&2; exit 1; } +test "$(git -C "$ROOT" branch --show-current)" = "feat/poc-cluster-productization" || { + echo "deployment is only allowed from feat/poc-cluster-productization" >&2 + exit 1 +} + +gpu_capacity="$("${K[@]}" get node "$GPU_NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}')" +gpu_in_use="$("${K[@]}" get pods -A --field-selector "spec.nodeName=$GPU_NODE" -o json | jq '[.items[] | select(.status.phase=="Running" or .status.phase=="Pending") | ([.spec.containers[]?.resources.requests["nvidia.com/gpu"] // "0" | tonumber] | add // 0)] | add // 0')" +if (( gpu_capacity - gpu_in_use < 1 )); then + echo "A800 node has no free GPU: capacity=$gpu_capacity in_use=$gpu_in_use" >&2 + exit 1 +fi +printf 'gpu_capacity=%s\ngpu_in_use=%s\nbuild_id=%s\n' "$gpu_capacity" "$gpu_in_use" "$BUILD_ID" > "$OUT/deploy_preflight.txt" + +"${K[@]}" apply -f "$ROOT/deploy/poc/bootstrap.yaml" + +clone_secret() { + local source_namespace="$1" source_name="$2" target_name="$3" + "${K[@]}" -n "$source_namespace" get secret "$source_name" -o json | + jq --arg ns "$NAMESPACE" --arg name "$target_name" ' + del(.metadata.uid,.metadata.resourceVersion,.metadata.creationTimestamp,.metadata.managedFields,.metadata.ownerReferences) | + .metadata.namespace=$ns | .metadata.name=$name | + .metadata.labels={"app.kubernetes.io/part-of":"real2sim-poc"}' | + "${K[@]}" apply -f - >/dev/null +} + +clone_secret droboticflow droboticflow-registry real2sim-registry +clone_secret droboticflow droboticflow-bos-workspace real2sim-bos + +if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets >/dev/null 2>&1; then + admin_password="$(openssl rand -base64 30 | tr -d '/+=' | cut -c1-28)Aa1!" + session_secret="$(openssl rand -hex 32)" + "${K[@]}" -n "$NAMESPACE" create secret generic real2sim-app-secrets \ + --from-literal=bootstrap-admin-password="$admin_password" \ + --from-literal=session-secret="$session_secret" \ + --dry-run=client -o yaml | + "${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml | + "${K[@]}" apply -f - >/dev/null +fi + +if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-tls >/dev/null 2>&1; then + tls_dir="$(mktemp -d)" + openssl req -x509 -newkey rsa:3072 -sha256 -nodes -days 365 \ + -subj "/CN=real2sim-poc.local" \ + -addext "subjectAltName=DNS:real2sim-poc.local,DNS:localhost,IP:127.0.0.1" \ + -keyout "$tls_dir/tls.key" -out "$tls_dir/tls.crt" >/dev/null 2>&1 + "${K[@]}" -n "$NAMESPACE" create secret tls real2sim-tls \ + --key "$tls_dir/tls.key" --cert "$tls_dir/tls.crt" --dry-run=client -o yaml | + "${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml | + "${K[@]}" apply -f - >/dev/null +fi + +"${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets -o jsonpath='{.data.bootstrap-admin-password}' | + base64 --decode > "$OUT/bootstrap-admin-password.txt" +chmod 600 "$OUT/bootstrap-admin-password.txt" +printf 'admin\n' > "$OUT/bootstrap-admin-username.txt" +chmod 600 "$OUT/bootstrap-admin-username.txt" + +prepare_context() { + context_archive="$(mktemp -t real2sim-poc-context.XXXXXX.tgz)" + tar -C "$ROOT" -czf "$context_archive" \ + --exclude='.git' --exclude='.venv' --exclude='.pytest_cache' --exclude='workspace' \ + --exclude='acceptance_runs' --exclude='qa' --exclude='docs' --exclude='tests' . + local loader="real2sim-context-${BUILD_ID//[^a-zA-Z0-9-]/-}" + loader="${loader:0:60}" + "${K[@]}" -n "$NAMESPACE" delete pod "$loader" --ignore-not-found --wait=true >/dev/null + "${K[@]}" -n "$NAMESPACE" run "$loader" \ + --image=busybox:1.36 --restart=Never \ + --labels="app.kubernetes.io/part-of=real2sim-poc,app.kubernetes.io/component=build-context" \ + --overrides="$(jq -cn --arg node "$GPU_NODE" '{spec:{nodeSelector:{"kubernetes.io/hostname":$node},containers:[{name:"context-loader",image:"busybox:1.36",command:["sh","-c","sleep 3600"],resources:{requests:{cpu:"100m",memory:"64Mi"},limits:{cpu:"500m",memory:"256Mi"}},volumeMounts:[{name:"data",mountPath:"/data"}]}],volumes:[{name:"data",persistentVolumeClaim:{claimName:"real2sim-poc-data"}}]}}')" >/dev/null + "${K[@]}" -n "$NAMESPACE" wait --for=condition=Ready "pod/$loader" --timeout=180s + "${K[@]}" -n "$NAMESPACE" cp "$context_archive" "$loader:/tmp/context.tgz" + "${K[@]}" -n "$NAMESPACE" exec "$loader" -- sh -eu -c \ + "mkdir -p '/data/build-context/$BUILD_ID' /data/workspace /data/cache /data/scratch; rm -rf '/data/build-context/$BUILD_ID/'*; tar -xzf /tmp/context.tgz -C '/data/build-context/$BUILD_ID'; chown -R 10001:10001 /data/workspace /data/cache /data/scratch; chmod 0770 /data/workspace /data/cache /data/scratch" + "${K[@]}" -n "$NAMESPACE" delete pod "$loader" --wait=true >/dev/null +} + +build_image() { + local component="$1" dockerfile="$2" destination="$3" + local job="real2sim-build-${component}-${BUILD_ID//[^a-zA-Z0-9-]/-}" + job="${job:0:60}" + "${K[@]}" -n "$NAMESPACE" delete job "$job" --ignore-not-found --wait=true >/dev/null + cat </dev/null +apiVersion: batch/v1 +kind: Job +metadata: + name: $job + namespace: $NAMESPACE + labels: + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/component: image-build +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app.kubernetes.io/part-of: real2sim-poc + app.kubernetes.io/component: image-build + spec: + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: "$GPU_NODE" + containers: + - name: kaniko + image: gcr.io/kaniko-project/executor:v1.23.2-debug + args: + - --context=dir:///data/build-context/$BUILD_ID + - --dockerfile=/data/build-context/$BUILD_ID/$dockerfile + - --destination=$destination + - --build-arg=BUILD_ID=$BUILD_ID + - --build-arg=SOURCE_COMMIT=$SOURCE_COMMIT + - --snapshot-mode=redo + - --use-new-run + - --cache=true + - --cache-repo=$REGISTRY/real2sim-build-cache + - --verbosity=info + resources: + requests: {cpu: "2", memory: 4Gi} + limits: {cpu: "16", memory: 32Gi} + volumeMounts: + - {name: data, mountPath: /data} + - {name: docker-config, mountPath: /kaniko/.docker, readOnly: true} + volumes: + - name: data + persistentVolumeClaim: {claimName: real2sim-poc-data} + - name: docker-config + secret: + secretName: real2sim-registry + items: + - {key: .dockerconfigjson, path: config.json} +YAML + if ! "${K[@]}" -n "$NAMESPACE" wait --for=condition=complete "job/$job" --timeout=7200s; then + "${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log" 2>&1 || true + "${K[@]}" -n "$NAMESPACE" describe "job/$job" > "$OUT/build-${component}-describe.txt" 2>&1 || true + return 1 + fi + "${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log" + "${K[@]}" -n "$NAMESPACE" delete job "$job" --wait=true >/dev/null +} + +FRONTEND_IMAGE="$REGISTRY/real2sim-frontend:$BUILD_ID" +API_IMAGE="$REGISTRY/real2sim-api:$BUILD_ID" +WORKER_IMAGE="$REGISTRY/real2sim-worker:$BUILD_ID" +printf '%s\n' "$FRONTEND_IMAGE" "$API_IMAGE" "$WORKER_IMAGE" > "$OUT/images.txt" + +if [[ "$BUILD_IMAGES" == "1" ]]; then + prepare_context + build_image frontend docker/frontend.Dockerfile "$FRONTEND_IMAGE" + build_image api docker/api.Dockerfile "$API_IMAGE" + build_image worker docker/worker.Dockerfile "$WORKER_IMAGE" +fi + +if [[ "$DEPLOY_APP" == "1" ]]; then + rendered_manifest="$(mktemp -t real2sim-poc-manifest.XXXXXX.yaml)" + sed \ + -e "s|__BUILD_ID__|$BUILD_ID|g" \ + -e "s|__FRONTEND_IMAGE__|$FRONTEND_IMAGE|g" \ + -e "s|__API_IMAGE__|$API_IMAGE|g" \ + -e "s|__WORKER_IMAGE__|$WORKER_IMAGE|g" \ + "$ROOT/deploy/poc/app-template.yaml" > "$rendered_manifest" + "${K[@]}" apply -f "$rendered_manifest" + "${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-api --timeout=600s + "${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-frontend --timeout=600s + "${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-worker --timeout=3600s + "${K[@]}" -n "$NAMESPACE" get pods,svc,pvc -l app.kubernetes.io/part-of=real2sim-poc -o wide > "$OUT/resources.txt" + "${K[@]}" -n "$NAMESPACE" get svc real2sim-frontend -o json > "$OUT/frontend-service.json" +fi + +printf 'build_id=%s\nnamespace=%s\ncredentials=%s\n' "$BUILD_ID" "$NAMESPACE" "$OUT/bootstrap-admin-password.txt" diff --git a/scripts/generate_motrixsim_runtime_scene.py b/scripts/generate_motrixsim_runtime_scene.py index b90c740..4dde397 100755 --- a/scripts/generate_motrixsim_runtime_scene.py +++ b/scripts/generate_motrixsim_runtime_scene.py @@ -51,7 +51,14 @@ def main() -> int: # the runtime copy so pick-place validation measures real bilateral contact. runtime_xml = runtime_xml.replace( 'contype="0" conaffinity="0" density="0" group="2"/>', - 'contype="1" conaffinity="1" density="0" friction="1.2 0.01 0.001" group="3"/>', + 'contype="1" conaffinity="1" density="0" friction="2.0 0.02 0.002" group="3"/>', + ) + runtime_xml = runtime_xml.replace( + 'name="left_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.003"', + 'name="left_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.010"', + ).replace( + 'name="right_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.003"', + 'name="right_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.010"', ) runtime_rm65.write_text(runtime_xml, encoding="utf-8") diff --git a/scripts/rollback_poc_cluster.sh b/scripts/rollback_poc_cluster.sh new file mode 100755 index 0000000..0a4d103 --- /dev/null +++ b/scripts/rollback_poc_cluster.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +KUBECONFIG_PATH="${REAL2SIM_POC_KUBECONFIG:-/Users/d-robotics/Downloads/poc.config}" +K=(kubectl --kubeconfig "$KUBECONFIG_PATH" -n real2sim-poc) +for deployment in real2sim-frontend real2sim-api real2sim-worker; do + "${K[@]}" rollout undo "deployment/$deployment" + "${K[@]}" rollout status "deployment/$deployment" --timeout=3600s +done +echo "rollback completed; PV and BOS objects were preserved" diff --git a/scripts/run_motrixsim_pick_place_rollout.py b/scripts/run_motrixsim_pick_place_rollout.py index c6270b4..bfa504c 100755 --- a/scripts/run_motrixsim_pick_place_rollout.py +++ b/scripts/run_motrixsim_pick_place_rollout.py @@ -34,12 +34,6 @@ def main() -> int: parser.add_argument("--width", type=int, default=960) parser.add_argument("--height", type=int, default=540) parser.add_argument("--no-render", action="store_true") - parser.add_argument( - "--grasp-mode", - choices=("physical", "kinematic_attachment"), - default="physical", - help="Use contact physics by default; kinematic attachment is retained only for diagnostics.", - ) args = parser.parse_args() config = load_config() @@ -61,7 +55,6 @@ def main() -> int: width=args.width, height=args.height, render=not args.no_render, - grasp_mode=args.grasp_mode, ) artifacts = {} @@ -81,7 +74,6 @@ def run_rollout( width: int, height: int, render: bool, - grasp_mode: str = "physical", ) -> dict[str, Any]: import mujoco @@ -105,7 +97,7 @@ def run_rollout( qpos_adrs = [int(model.jnt_qposadr[jid]) for jid in joint_ids] dof_adrs = [int(model.jnt_dofadr[jid]) for jid in joint_ids] - duration_scale = 1 if grasp_mode == "kinematic_attachment" else 8 + duration_scale = 8 phases = [ ("home", np.array([0.42, 0.00, 0.72]), 0.0, 35 * duration_scale, False), ("approach", np.array([0.71, 0.015, 0.42]), 0.0, 55 * duration_scale, False), @@ -123,7 +115,6 @@ def run_rollout( if render: renderer = mujoco.Renderer(model, height=height, width=width) - attached = False contact_steps = 0 bilateral_contact_steps = 0 phase_contacts = {phase: {"contact_steps": 0, "bilateral_contact_steps": 0} for phase, *_ in phases} @@ -135,7 +126,7 @@ def run_rollout( started = time.time() frame_stride = max(1, round((1.0 / model.opt.timestep) / fps)) step_index = 0 - for phase, target, gripper_ctrl, count, attach_cube in phases: + for phase, target, gripper_ctrl, count, _contact_expected in phases: if phase == "grasp": settled_cube_z = float(data.qpos[cube_qadr + 2]) max_cube_z = settled_cube_z @@ -145,14 +136,7 @@ def run_rollout( current = data.ctrl[:6].copy() data.ctrl[:6] = current * (1 - alpha) + q_target * alpha data.ctrl[6] = gripper_ctrl - if grasp_mode == "kinematic_attachment" and (attach_cube or attached): - attached = attach_cube - if attached: - set_cube_pose(data, cube_qadr, data.site_xpos[site_id] + np.array([0.0, 0.0, -0.095])) mujoco.mj_step(model, data) - if grasp_mode == "kinematic_attachment" and attached: - set_cube_pose(data, cube_qadr, data.site_xpos[site_id] + np.array([0.0, 0.0, -0.095])) - mujoco.mj_forward(model, data) touching_fingers = cube_finger_contacts(data, cube_geom, finger_geoms) accumulate_contact_metrics(model, data, cube_geom, finger_geoms, contact_force_metrics) if touching_fingers: @@ -193,7 +177,7 @@ def run_rollout( place_error = float(np.linalg.norm(final_cube - target_cube)) physical_success = bilateral_contact_steps > 0 and lift_height >= 0.10 and place_error <= 0.12 summary = { - "mode": "scripted_ik_physical_contact" if grasp_mode == "physical" else "scripted_ik_with_cube_attachment", + "mode": "scripted_ik_physical_contact", "steps": step_index, "samples": len(trajectory), "sim_time": float(data.time), @@ -210,22 +194,18 @@ def run_rollout( "phase_contacts": phase_contacts, "gripper_qpos_range": gripper_qpos_range, "contact_force_metrics": contact_force_metrics, - "physical_success": physical_success if grasp_mode == "physical" else None, + "physical_success": physical_success, "elapsed_ms": round((time.time() - started) * 1000), } report = { "schema": "real2sim.rm65_pick_place_rollout_report.v1", "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "ok": physical_success if grasp_mode == "physical" else True, + "ok": physical_success, "runtime": "mujoco", "scene": str(sim_path), "summary": summary, "phases": [phase for phase, *_ in phases], - "note": ( - "Scripted IK with contact-physics grasp; the cube pose is never overwritten. This is not a learned policy." - if grasp_mode == "physical" - else "Diagnostic-only rollout. Cube follows the gripper during grasp/lift/place; this is not a learned policy." - ), + "note": "Scripted IK with contact-physics grasp; the cube pose is never overwritten. This is not a learned policy.", } report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") artifact_paths["report"] = str(report_path) @@ -284,12 +264,6 @@ def solve_ik(model: Any, data: Any, site_id: int, target: np.ndarray, qpos_adrs: return q -def set_cube_pose(data: Any, qadr: int, pos: np.ndarray) -> None: - data.qpos[qadr : qadr + 3] = pos - data.qpos[qadr + 3 : qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0]) - data.qvel[qadr : qadr + 6] = 0 - - def sample_state(model: Any, data: Any, phase: str, step: int, site_id: int, cube_qadr: int) -> dict[str, Any]: return { "step": step, diff --git a/src/real2sim_platform/act_inference.py b/src/real2sim_platform/act_inference.py new file mode 100644 index 0000000..f7096c4 --- /dev/null +++ b/src/real2sim_platform/act_inference.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import argparse +import importlib.util +import json +import pickle +import sys +import time +from pathlib import Path +from typing import Any + +import numpy as np + +from .rm65_controller import RM65PickPlaceRuntime, _write_preview_video + + +def load_policy(model_dir: Path, act_root: Path, max_timesteps: int): + import torch + + sys.path.insert(0, str(act_root)) + from configurations.task_configs.config_tools.basic_configer import config_policy + + config_path = model_dir / "task_config.py" + checkpoint = model_dir / "core" / "policy_best.ckpt" + stats_path = model_dir / "core" / "dataset_stats.pkl" + for path in (config_path, checkpoint, stats_path): + if not path.is_file(): + raise FileNotFoundError(path) + spec = importlib.util.spec_from_file_location("real2sim_act_task_config", config_path) + if spec is None or spec.loader is None: + raise RuntimeError("unable to load ACT task config") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + task_config = module.TASK_CONFIG + merged = {**task_config["common"], **task_config["train"]} + merged["learning_rate"] = task_config["train"]["learning_rate"] + policy_config = config_policy(merged) + policy_config["ckpt_path"] = str(checkpoint) + policy_config["max_timesteps"] = max_timesteps + policy = module.policy_maker(policy_config, "eval") + with stats_path.open("rb") as handle: + stats = pickle.load(handle) + if not all(key in stats for key in ("qpos_mean", "qpos_std", "action_mean", "action_std")): + raise ValueError("ACT dataset_stats.pkl is missing normalization arrays") + return policy, stats, task_config["common"]["camera_names"], torch + + +def infer( + *, + scene_path: Path, + model_dir: Path, + act_root: Path, + out_dir: Path, + episodes: int, + steps: int, + seed: int, + minimum_success_rate: float, + width: int, + height: int, +) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + policy, stats, camera_names, torch = load_policy(model_dir, act_root, steps) + if camera_names != ["overview"]: + raise ValueError(f"RM65 runtime currently exposes camera ['overview']; model expects {camera_names}") + runtime = RM65PickPlaceRuntime(scene_path, width=width, height=height, seed=seed) + results: list[dict[str, Any]] = [] + all_actions: list[np.ndarray] = [] + try: + for episode in range(episodes): + policy.reset() + cube, target, settled_z = _reset_randomized(runtime) + trajectory: list[dict[str, Any]] = [] + frames: list[np.ndarray] = [] + max_cube_z = settled_z + bilateral_steps = 0 + for step in range(steps): + runtime.renderer.update_scene(runtime.data, camera="overview") + frame = runtime.renderer.render().copy() + qpos = np.concatenate([runtime.data.qpos[runtime.qpos_adrs], [runtime.data.qpos[runtime.gripper_qadr]]]).astype(np.float32) + normalized_qpos = (qpos - np.asarray(stats["qpos_mean"], dtype=np.float32)) / np.asarray(stats["qpos_std"], dtype=np.float32) + image = torch.from_numpy(frame).permute(2, 0, 1).unsqueeze(0).float().cuda() / 255.0 + qpos_tensor = torch.from_numpy(normalized_qpos).unsqueeze(0).float().cuda() + with torch.inference_mode(): + predicted = policy(qpos_tensor, image) + normalized_action = predicted[0, 0].detach().cpu().numpy() + action = normalized_action * np.asarray(stats["action_std"], dtype=np.float32) + np.asarray(stats["action_mean"], dtype=np.float32) + action = np.asarray(action, dtype=np.float32) + if action.shape != (7,): + raise ValueError(f"ACT action shape must be (7,), received {action.shape}") + for index in range(7): + action[index] = np.clip(action[index], runtime.ctrl_ranges[index, 0], runtime.ctrl_ranges[index, 1]) + runtime.data.ctrl[:7] = action + for _ in range(10): + runtime.mujoco.mj_step(runtime.model, runtime.data) + touching = runtime.finger_contacts() + bilateral_steps += int(len(touching) == 2) + max_cube_z = max(max_cube_z, float(runtime.data.qpos[runtime.cube_qadr + 2])) + all_actions.append(action.copy()) + frames.append(frame) + trajectory.append( + { + "step": step, + "time": float(runtime.data.time), + "qpos": qpos.tolist(), + "action": action.tolist(), + "cube_pos": runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3].astype(float).tolist(), + "target": target.tolist(), + "bilateral_contact": len(touching) == 2, + } + ) + final_cube = runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3].copy() + lift_height = float(max_cube_z - settled_z) + place_error = float(np.linalg.norm(final_cube - target)) + success = bilateral_steps > 0 and lift_height >= 0.10 and place_error <= 0.05 + result = { + "episode": episode, + "success": success, + "lift_height": lift_height, + "place_error": place_error, + "bilateral_contact_steps": bilateral_steps, + "initial_cube": cube.tolist(), + "target": target.tolist(), + } + results.append(result) + (out_dir / f"episode_{episode:03d}_trajectory.json").write_text(json.dumps({"schema": "real2sim.act_rollout.v1", "result": result, "samples": trajectory}, ensure_ascii=False) + "\n", encoding="utf-8") + _write_preview_video(out_dir / f"episode_{episode:03d}.mp4", frames) + finally: + runtime.close() + success_rate = sum(int(item["success"]) for item in results) / max(1, len(results)) + action_matrix = np.asarray(all_actions, dtype=np.float32) + action_std = action_matrix.std(axis=0).tolist() if len(action_matrix) else [] + nonconstant = bool(len(action_matrix) > 1 and np.max(np.std(action_matrix, axis=0)) > 1e-5) + report = { + "schema": "real2sim.discoverse_act_inference.v1", + "runtime": "mujoco", + "scene": str(scene_path), + "model": str(model_dir), + "episodes": results, + "success_rate": success_rate, + "minimum_success_rate": minimum_success_rate, + "action_std": action_std, + "actions_nonconstant": nonconstant, + "pass": success_rate >= minimum_success_rate and nonconstant, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + (out_dir / "inference_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return report + + +def _reset_randomized(runtime: RM65PickPlaceRuntime) -> tuple[np.ndarray, np.ndarray, float]: + runtime.mujoco.mj_resetDataKeyframe(runtime.model, runtime.data, 0) + cube = np.array([runtime.rng.uniform(0.645, 0.655), runtime.rng.uniform(-0.010, 0.010), 0.33], dtype=float) + target = np.array([runtime.rng.uniform(0.42, 0.51), runtime.rng.uniform(0.13, 0.22), 0.31], dtype=float) + runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3] = cube + runtime.data.qpos[runtime.cube_qadr + 3 : runtime.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0]) + cube_dof = int(runtime.model.jnt_dofadr[runtime.cube_joint]) + runtime.data.qvel[cube_dof : cube_dof + 6] = 0 + runtime.model.body_pos[runtime.target_body] = np.array([target[0], target[1], 0.335]) + runtime.mujoco.mj_forward(runtime.model, runtime.data) + for _ in range(120): + runtime.mujoco.mj_step(runtime.model, runtime.data) + return cube, target, float(runtime.data.qpos[runtime.cube_qadr + 2]) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run DISCOVERSE ACT in the physical RM65 MuJoCo scene") + parser.add_argument("--scene", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--act-root", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--episodes", type=int, default=10) + parser.add_argument("--steps", type=int, default=200) + parser.add_argument("--seed", type=int, default=11) + parser.add_argument("--minimum-success-rate", type=float, default=0.5) + parser.add_argument("--width", type=int, default=320) + parser.add_argument("--height", type=int, default=240) + args = parser.parse_args() + report = infer( + scene_path=args.scene, + model_dir=args.model, + act_root=args.act_root, + out_dir=args.out, + episodes=args.episodes, + steps=args.steps, + seed=args.seed, + minimum_success_rate=args.minimum_success_rate, + width=args.width, + height=args.height, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if report["pass"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/real2sim_platform/adapters.py b/src/real2sim_platform/adapters.py index 19b6c83..106a9f9 100644 --- a/src/real2sim_platform/adapters.py +++ b/src/real2sim_platform/adapters.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import hashlib +from functools import lru_cache import math import os import shutil @@ -15,6 +17,7 @@ from urllib.parse import quote, unquote, urlparse from urllib.request import urlopen from .config import PlatformConfig +from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path from .store import PlatformStore @@ -61,16 +64,26 @@ def scan_dataset(path: Path) -> dict[str, Any]: def run_doctor(config: PlatformConfig) -> dict[str, Any]: - core_checks = [ - "workspace_writable", - "expert_policy", - "training", - "inference", - "media_prepare_deps", - "ffmpeg", - "spark_bundle", - "three_bundle", - ] + core_checks = ( + [ + "workspace_writable", + "rm65_controller", + "discoverse_act", + "torch_cuda", + "act_import", + "media_prepare_deps", + "ffmpeg", + "colmap", + "gs_video_reconstruct", + "anysplat_pinned", + "mujoco_python", + "motrixsim_python", + "nvidia_smi", + ] + if config.production_mode + else ["workspace_writable", "expert_policy", "training", "inference", "media_prepare_deps", "ffmpeg", "spark_bundle", "three_bundle"] + ) + act_root = config.discoverse_root / "policies" / "act" checks = { "workspace_writable": _is_writable(config.workspace_dir), "expert_policy": (config.expert_policy_root / "src/real2sim_expert/collect.py").is_file(), @@ -91,9 +104,15 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]: "mujoco_python": _module_available("mujoco"), "service_process": os.getpid() > 0, "nvidia_smi": shutil.which("nvidia-smi") is not None, + "rm65_controller": Path(__file__).with_name("rm65_controller.py").is_file(), + "discoverse_act": _git_head(config.discoverse_root) == config.discoverse_commit and _git_head(act_root) == config.act_commit, + "torch_cuda": _torch_cuda_ready(), + "act_import": _act_runtime_import_ready(str(config.discoverse_root), config.discoverse_commit, config.act_commit), + "anysplat_pinned": _git_head(config.anysplat_root) == config.anysplat_commit and config.anysplat_python.exists(), + "motrixsim_python": _module_available("motrixsim"), } return { - "ok": all(checks[name] for name in core_checks), + "ok": all(bool(checks[name]) for name in core_checks), "core_checks": core_checks, "warnings": [ name @@ -108,6 +127,8 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]: "cuda_home", "colmap", "nvidia_smi", + "torch_cuda", + "act_import", "three_bundle", ] if not checks.get(name) @@ -148,6 +169,8 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A return run_reconstruction_preflight(config, store, task, emit) if task_type == "hf_import": return run_hf_import(config, store, task, emit) + if task_type == "bos_import": + return run_bos_import(config, store, task, emit) if task_type == "simulator_validation": return run_simulator_validation(config, store, task, emit) if task_type == "expert_collection": @@ -165,6 +188,42 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A raise ValueError(f"unknown task type: {task_type}") +def sync_task_outputs(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]: + storage = BosStorage(config) + if not storage.configured: + if config.production_mode: + raise RuntimeError("生产任务产物无法持久化:BOS 尚未配置") + return result + _task_progress(store, task, 96, "持久化产物", "正在将任务报告和业务产物上传到 BOS") + run_dir = Path(task["run_dir"]) + if run_dir.is_dir(): + _upload_directory(storage, run_dir, area="runs", relative_root=task["id"]) + resource_specs = [ + ("dataset_id", "datasets", store.get_dataset), + ("scene_id", "scenes", store.get_scene), + ("model_id", "models", store.get_model), + ] + persisted: dict[str, Any] = {} + for result_key, area, getter in resource_specs: + entity_id = str(result.get(result_key) or "") + if not entity_id: + continue + entity = getter(entity_id) + path = Path(str(entity["path"])) + root = path if path.is_dir() else path.parent + if not root.is_dir(): + raise FileNotFoundError(f"resource output directory not found: {root}") + _upload_directory(storage, root, area=area, relative_root=entity_id) + checksum = _directory_digest(root) + uri = storage.uri_for_key(storage.object_key(area, f"{entity_id}/")) + store.update_entity_storage(area, entity_id, storage_uri=uri, checksum=checksum, cache_state="local") + persisted[result_key] = {"storage_uri": uri, "checksum": checksum} + result = dict(result) + result["persistent_outputs"] = persisted + result["run_storage_uri"] = storage.uri_for_key(storage.object_key("runs", f"{task['id']}/")) + return result + + def _task_progress(store: PlatformStore, task: dict[str, Any] | str, progress: int, stage: str, message: str = "") -> None: task_id = task if isinstance(task, str) else task["id"] store.update_task(task_id, progress=progress, stage=stage, message=message) @@ -504,6 +563,66 @@ def run_hf_import(config: PlatformConfig, store: PlatformStore, task: dict[str, return {"dataset_id": dataset["id"], "dataset_path": str(target), "manifest_path": str(manifest_path), "scan": scan} +def run_bos_import(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: + params = task["params"] + storage = BosStorage(config) + raw_prefix = normalize_relative_path(str(params.get("prefix") or "")) + prefix_parts = raw_prefix.split("/", 1) + area = prefix_parts[0] if prefix_parts[0] in ALLOWED_AREAS else "datasets" + relative_prefix = prefix_parts[1] if prefix_parts[0] in ALLOWED_AREAS and len(prefix_parts) > 1 else raw_prefix + _task_progress(store, task, 18, "读取 BOS", "正在列举授权前缀内的对象") + objects = storage.list_objects(area=area, relative_prefix=relative_prefix, max_keys=int(params.get("max_objects", 1000))) + objects = [item for item in objects if not item["key"].endswith("/")] + if not objects: + raise ValueError("BOS 前缀中没有可导入对象") + dataset_id = uuid.uuid4().hex + out_dir = config.datasets_dir / dataset_id + out_dir.mkdir(parents=True, exist_ok=False) + common_prefix = storage.object_key(area, relative_prefix) + _task_progress(store, task, 30, "下载数据", f"正在下载 {len(objects)} 个真实 BOS 对象") + for index, item in enumerate(objects, start=1): + relative = item["key"][len(common_prefix):].lstrip("/") or Path(item["key"]).name + storage.download_file(item["storage_uri"], out_dir / relative) + if index % max(1, len(objects) // 10) == 0: + _task_progress(store, task, 30 + int(45 * index / len(objects)), "下载数据", f"已下载 {index}/{len(objects)} 个对象") + scan = scan_dataset(out_dir) + digest = _directory_digest(out_dir) + source_uri = f"bos://{storage.bucket}/{common_prefix}" + manifest = { + "schema": "real2sim.dataset_manifest.v2", + "id": dataset_id, + "name": params.get("name") or Path(relative_prefix.rstrip("/")).name, + "source_type": "bos_prefix", + "storage_uri": source_uri, + "checksum": digest, + "version": digest[:12], + "object_count": len(objects), + "objects": objects, + "scan": scan, + } + manifest_path = out_dir / "dataset_manifest.json" + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + dataset = store.create_dataset( + item_id=dataset_id, + name=str(manifest["name"]), + path=out_dir, + scan=scan, + source_type="bos_prefix", + source_uri=source_uri, + version=digest[:12], + parent_task_id=task["id"], + manifest_path=manifest_path, + metrics={"object_count": len(objects), "size_bytes": sum(int(item["size_bytes"]) for item in objects)}, + tags=["bos", "imported"], + storage_uri=source_uri, + checksum=digest, + cache_state="local", + ) + _index_artifacts(store, task["id"], out_dir) + _task_progress(store, task, 95, "登记数据集", "BOS 数据及校验清单已登记") + return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "storage_uri": source_uri, "object_count": len(objects), "checksum": digest} + + def run_simulator_validation(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: params = task["params"] run_dir = Path(task["run_dir"]) @@ -564,8 +683,50 @@ def run_expert_collection(config: PlatformConfig, store: PlatformStore, task: di run_dir = Path(task["run_dir"]) scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json" out_dir = run_dir / "dataset" - policy_preset = str(params.get("policy_preset") or "pick_and_place_v1") - if policy_preset != "pick_and_place_v1": + policy_preset = str(params.get("policy_preset") or ("rm65_pick_place_6d_ik" if config.production_mode else "pick_and_place_v1")) + if policy_preset == "rm65_pick_place_6d_ik": + scene_path = _resolve_rm65_runtime_scene(store, params, scene_path) + required_successes = int(params.get("episodes", 50)) + max_attempts = int(params.get("max_attempts", max(required_successes * 3, required_successes))) + command = [ + sys.executable, + "-m", + "real2sim_platform.rm65_controller", + "--scene", + str(scene_path), + "--out", + str(out_dir), + "--required-successes", + str(required_successes), + "--max-attempts", + str(max_attempts), + "--steps", + str(int(params.get("steps", 200))), + "--seed", + str(int(params.get("seed", 7))), + "--width", + str(int(params.get("image_width", 320))), + "--height", + str(int(params.get("image_height", 240))), + ] + _task_progress(store, task, 28, "物理采集", "RM65 正在通过 6D IK、双指接触和真实物理步进采集成功 episode") + run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src"]), store=store, task_id=task["id"]) + manifest = _read_json(out_dir / "manifest.json") + dataset = store.create_dataset( + name=params.get("name", f"RM65 ACT dataset {task['id'][:8]}"), + path=out_dir, + scan={"input_kind": "discoverse_act_hdf5", "samples": manifest.get("successful_episodes"), "manifest": str(out_dir / "manifest.json")}, + source_type="expert_collection", + source_uri=str(scene_path), + parent_task_id=task["id"], + manifest_path=out_dir / "manifest.json", + metrics={"samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate"), "policy_preset": policy_preset}, + tags=["discoverse_act_hdf5", "rm65", "physical_contact", policy_preset], + ) + _index_artifacts(store, task["id"], out_dir) + _task_progress(store, task, 95, "登记数据集", "真实采集 HDF5、接触记录和成功报告已登记") + return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate")} + if policy_preset != "pick_and_place_v1" or config.production_mode: raise ValueError(f"unsupported expert policy preset: {policy_preset}") _task_progress(store, task, 18, "准备采集", "正在加载场景、专家策略和机器人配置") command = [ @@ -615,7 +776,9 @@ def run_training(config: PlatformConfig, store: PlatformStore, task: dict[str, A _task_progress(store, task, 18, "准备训练", "正在加载训练数据集和训练配方") dataset_path = _dataset_path(store, params) out_dir = run_dir / "model" - recipe = str(params.get("recipe") or "bc_v2") + recipe = str(params.get("recipe") or ("discoverse_act" if config.production_mode else "bc_v2")) + if recipe == "discoverse_act": + return _run_discoverse_act_training(config, store, task, dataset_path, out_dir) recipe_config = { "bc_v2": {"regularization": 1e-3, "base_model": "facebook/dinov2-small"}, "bc_fast": {"regularization": 1e-2, "base_model": "facebook/dinov2-small"}, @@ -673,6 +836,44 @@ def run_inference(config: PlatformConfig, store: PlatformStore, task: dict[str, scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json" model_path = _model_path(store, params) out_dir = run_dir / "inference" + model = store.get_model(str(params["model_id"])) if params.get("model_id") else None + model_recipe = str(((model or {}).get("metadata") or {}).get("training", {}).get("recipe") or params.get("recipe") or "") + if model_recipe == "discoverse_act": + scene_path = _resolve_rm65_runtime_scene(store, params, scene_path) + act_root = config.discoverse_root / "policies" / "act" + if _git_head(act_root) != config.act_commit: + raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}") + command = [ + sys.executable, + "-m", + "real2sim_platform.act_inference", + "--scene", + str(scene_path), + "--model", + str(model_path), + "--act-root", + str(act_root), + "--out", + str(out_dir), + "--episodes", + str(int(params.get("episodes", 10))), + "--steps", + str(int(params.get("steps", 200))), + "--seed", + str(int(params.get("seed", 11))), + "--minimum-success-rate", + str(float(params.get("minimum_success_rate", 0.5))), + ] + _task_progress(store, task, 38, "ACT 在线推理", "DISCOVERSE ACT 正在真实 MuJoCo 场景中闭环控制 RM65") + run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src", act_root, config.discoverse_root]), store=store, task_id=task["id"]) + report = _read_json(out_dir / "inference_report.json") + if not report.get("actions_nonconstant"): + raise RuntimeError("ACT 推理动作是常量,拒绝登记为有效推理") + _index_artifacts(store, task["id"], out_dir) + _task_progress(store, task, 95, "索引推理产物", "真实 rollout 视频、轨迹和成功率报告已登记") + return {"episodes": len(report.get("episodes", [])), "success_rate": report.get("success_rate"), "report": str(out_dir / "inference_report.json")} + if config.production_mode: + raise ValueError("生产模式只允许 DISCOVERSE ACT 模型在线推理") command = [ sys.executable, "-m", @@ -695,6 +896,97 @@ def run_inference(config: PlatformConfig, store: PlatformStore, task: dict[str, return {"rollout_steps": len(output.get("rollout", [])), "viewer": str(out_dir / "viewer.html")} +def _run_discoverse_act_training(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], dataset_path: Path, out_dir: Path) -> dict[str, Any]: + params = task["params"] + run_dir = Path(task["run_dir"]) + act_root = config.discoverse_root / "policies" / "act" + if _git_head(act_root) != config.act_commit: + raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}") + dataset_info = _inspect_act_dataset(dataset_path) + if dataset_info["episode_count"] < 2: + raise ValueError("DISCOVERSE ACT 至少需要 2 个真实 HDF5 episode") + out_dir.mkdir(parents=True, exist_ok=True) + config_root = run_dir / "generated_config" + task_name = f"real2sim_{task['id'][:12]}" + config_path = config_root / "configurations" / "task_configs" / f"{task_name}.py" + config_path.parent.mkdir(parents=True, exist_ok=True) + epochs = int(params.get("epochs", 100)) + validate_every = max(1, min(int(params.get("validate_every", max(1, epochs // 5))), epochs)) + save_every = max(1, min(int(params.get("save_every", validate_every)), epochs)) + batch_size = max(1, min(int(params.get("batch_size", 8)), dataset_info["episode_count"] - 1)) + chunk_size = max(2, min(int(params.get("chunk_size", 32)), dataset_info["episode_length"])) + core_dir = out_dir / "core" + checkpoint_dir = out_dir / "checkpoints" + config_path.write_text( + _act_task_config_source( + task_name=task_name, + dataset_path=dataset_path, + checkpoint_dir=checkpoint_dir, + stats_path=core_dir / "dataset_stats.pkl", + camera_names=dataset_info["camera_names"], + state_dim=dataset_info["state_dim"], + action_dim=dataset_info["action_dim"], + epochs=epochs, + batch_size=batch_size, + chunk_size=chunk_size, + validate_every=validate_every, + save_every=save_every, + ), + encoding="utf-8", + ) + command = [sys.executable, "policy_train.py", "-tn", task_name, "-ne", str(epochs), "-ve", str(validate_every), "-se", str(save_every), "--seed", str(int(params.get("seed", 1)))] + env = _env_with_paths(config, [config_root, act_root, config.discoverse_root]) + env["MPLBACKEND"] = "Agg" + _task_progress(store, task, 34, "ACT 训练", f"正在使用 {dataset_info['episode_count']} 个真实 episode 训练 DISCOVERSE ACT") + run_logged(command, cwd=act_root, log_path=run_dir / "logs" / "task.log", env=env, store=store, task_id=task["id"]) + checkpoint = core_dir / "policy_best.ckpt" + stats_path = core_dir / "dataset_stats.pkl" + if not checkpoint.is_file() or checkpoint.stat().st_size == 0 or not stats_path.is_file(): + raise RuntimeError("DISCOVERSE ACT 训练结束但未生成非空 policy_best.ckpt 或 dataset_stats.pkl") + description = (core_dir / "description.txt").read_text(encoding="utf-8").strip() if (core_dir / "description.txt").is_file() else "" + best_loss = _parse_best_loss(description) + if best_loss is None or not math.isfinite(best_loss): + raise RuntimeError("ACT 训练指标不是有限数值") + model_card = { + "schema": "real2sim.model_card.v2", + "name": params.get("name", f"RM65 ACT {task['id'][:8]}"), + "framework": "DISCOVERSE ACT", + "training": { + "recipe": "discoverse_act", + "discoverse_commit": config.discoverse_commit, + "act_commit": config.act_commit, + "dataset_path": str(dataset_path), + "epochs": epochs, + "batch_size": batch_size, + "chunk_size": chunk_size, + "camera_names": dataset_info["camera_names"], + "state_dim": dataset_info["state_dim"], + "action_dim": dataset_info["action_dim"], + "config_path": str(config_path), + }, + "metrics": {"best_validation_loss": best_loss}, + "artifacts": {"checkpoint": str(checkpoint), "dataset_stats": str(stats_path)}, + "description": description, + } + card_path = out_dir / "model_card.json" + card_path.write_text(json.dumps(model_card, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + shutil.copy2(config_path, out_dir / "task_config.py") + model = store.create_model( + name=str(model_card["name"]), + path=out_dir, + metadata=model_card, + source_type="training", + source_uri=str(dataset_path), + parent_task_id=task["id"], + manifest_path=card_path, + metrics=model_card["metrics"], + tags=["policy", "discoverse_act", "rm65"], + ) + _index_artifacts(store, task["id"], out_dir) + _task_progress(store, task, 95, "索引 ACT 模型", "checkpoint、损失曲线、模型卡和数据血缘已登记") + return {"model_id": model["id"], "model_path": str(out_dir), "metrics": model_card["metrics"], "checkpoint": str(checkpoint)} + + def run_renderer_benchmark(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]: run_dir = Path(task["run_dir"]) run_dir.mkdir(parents=True, exist_ok=True) @@ -1550,6 +1842,35 @@ def _scene_path(store: PlatformStore, params: dict[str, Any]) -> Path | None: return None +def _resolve_rm65_runtime_scene(store: PlatformStore, params: dict[str, Any], fallback: Path) -> Path: + candidates: list[Path] = [] + if fallback.suffix.lower() == ".xml": + candidates.append(fallback) + if params.get("scene_id"): + scene = store.get_scene(str(params["scene_id"])) + metadata = scene.get("metadata") or {} + package = Path(str(metadata.get("package") or "")) + manifest_path = Path(str(scene.get("manifest_path") or "")) + manifest = _read_optional_json(manifest_path) + for asset in manifest.get("assets", []) if isinstance(manifest.get("assets"), list) else []: + if not isinstance(asset, dict): + continue + name = str(asset.get("name") or asset.get("path") or "").lower() + if "rm65" in name and name.endswith(".xml"): + candidates.append(Path(str(asset.get("path") or ""))) + if str(package): + candidates.extend([ + package / "simulator_assets" / "rm65_table_pick_place.xml", + package / "scene_mujoco.xml", + package / "mjcf" / "scene.xml", + ]) + candidates.extend([fallback.parent / "simulator_assets" / "rm65_table_pick_place.xml", fallback.parent / "scene_mujoco.xml"]) + for candidate in candidates: + if candidate.is_file(): + return candidate + raise FileNotFoundError("所选场景没有可执行的 RM65 pick-place MJCF;请先在场景管理中绑定 RM65 控制器和碰撞资产") + + def _model_path(store: PlatformStore, params: dict[str, Any]) -> Path: if params.get("model_path"): return Path(str(params["model_path"])) @@ -1558,6 +1879,147 @@ def _model_path(store: PlatformStore, params: dict[str, Any]) -> Path: raise ValueError("inference requires model_id or model_path") +def _directory_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file()): + relative = path.relative_to(root).as_posix() + digest.update(relative.encode("utf-8")) + with path.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _upload_directory(storage: BosStorage, root: Path, *, area: str, relative_root: str) -> None: + for path in sorted(item for item in root.rglob("*") if item.is_file()): + storage.upload_file( + path, + area=area, + relative_path=f"{relative_root}/{path.relative_to(root).as_posix()}", + metadata={"sha256": _file_sha256(path)}, + ) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _git_head(root: Path) -> str: + try: + return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=5).strip() + except Exception: + return "" + + +def _inspect_act_dataset(dataset_path: Path) -> dict[str, Any]: + try: + import h5py + except ImportError as exc: + raise RuntimeError("DISCOVERSE ACT training requires h5py") from exc + episodes = sorted(dataset_path.glob("episode_*.hdf5")) + if not episodes: + raise ValueError("训练数据目录中没有 episode_*.hdf5") + lengths: list[int] = [] + cameras: list[str] = [] + state_dim = 0 + action_dim = 0 + for path in episodes: + with h5py.File(path, "r") as root: + if "/observations/qpos" not in root or "/action" not in root or "/observations/images" not in root: + raise ValueError(f"ACT HDF5 缺少 qpos/action/images: {path.name}") + current_cameras = sorted(str(name) for name in root["/observations/images"].keys()) + current_state_dim = int(root["/observations/qpos"].shape[1]) + current_action_dim = int(root["/action"].shape[1]) + if not cameras: + cameras = current_cameras + state_dim = current_state_dim + action_dim = current_action_dim + if current_cameras != cameras or current_state_dim != state_dim or current_action_dim != action_dim: + raise ValueError("ACT 数据集的相机、状态维度或动作维度不一致") + length = int(root["/action"].shape[0]) + if length < 2: + raise ValueError(f"ACT episode 过短: {path.name}") + lengths.append(length) + return { + "episode_count": len(episodes), + "episode_length": min(lengths), + "camera_names": cameras, + "state_dim": state_dim, + "action_dim": action_dim, + } + + +def _act_task_config_source( + *, + task_name: str, + dataset_path: Path, + checkpoint_dir: Path, + stats_path: Path, + camera_names: list[str], + state_dim: int, + action_dim: int, + epochs: int, + batch_size: int, + chunk_size: int, + validate_every: int, + save_every: int, +) -> str: + return f'''from configurations.task_configs.template import TASK_CONFIG_DEFAULT, activator, set_paths + + +def policy_maker(config: dict, stage=None): + from policies.act.act import ACTPolicy + from policies.common.maker import post_init_policies + policy = ACTPolicy(config) + post_init_policies([policy], stage, [config["ckpt_path"]]) + return policy + + +@activator(False) +def augment_images(image): + return image + + +set_paths({str(dataset_path)!r}, {str(checkpoint_dir)!r}, {str(stats_path)!r}) +TASK_CONFIG_DEFAULT["common"]["camera_names"] = {camera_names!r} +TASK_CONFIG_DEFAULT["common"]["state_dim"] = {state_dim} +TASK_CONFIG_DEFAULT["common"]["action_dim"] = {action_dim} +TASK_CONFIG_DEFAULT["common"]["policy_config"]["policy_maker"] = policy_maker +TASK_CONFIG_DEFAULT["common"]["policy_config"]["temporal_agg"] = True +TASK_CONFIG_DEFAULT["common"]["policy_config"]["chunk_size"] = {chunk_size} +TASK_CONFIG_DEFAULT["common"]["policy_config"]["num_queries"] = {chunk_size} +TASK_CONFIG_DEFAULT["train"]["load_data"]["data_type"] = "hdf5" +TASK_CONFIG_DEFAULT["train"]["load_data"]["num_episodes"] = "ALL" +TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_train"] = {batch_size} +TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_validate"] = {batch_size} +TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_train"] = 2 +TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_validate"] = 2 +TASK_CONFIG_DEFAULT["train"]["load_data"]["observation_slice"] = None +TASK_CONFIG_DEFAULT["train"]["load_data"]["action_slice"] = None +TASK_CONFIG_DEFAULT["train"]["num_epochs"] = {epochs} +TASK_CONFIG_DEFAULT["train"]["validate_every"] = {validate_every} +TASK_CONFIG_DEFAULT["train"]["save_every"] = {save_every} +TASK_CONFIG_DEFAULT["train"]["eval_every"] = 0 +TASK_CONFIG = TASK_CONFIG_DEFAULT +''' + + +def _parse_best_loss(description: str) -> float | None: + import re + + match = re.search(r"val loss\s+([0-9.eE+-]+)", description) + if not match: + return None + try: + return float(match.group(1)) + except ValueError: + return None + + def _write_dry_run_scene(package_dir: Path, input_path: Path, renderer: str, simulators: list[str]) -> Path: package_dir.mkdir(parents=True, exist_ok=True) scene = { @@ -2465,6 +2927,40 @@ def _module_available(name: str) -> bool: return False +@lru_cache(maxsize=1) +def _torch_cuda_ready() -> bool: + if not _module_available("torch"): + return False + completed = subprocess.run( + [sys.executable, "-c", "import torch; raise SystemExit(0 if torch.cuda.is_available() else 2)"], + text=True, + capture_output=True, + timeout=30, + check=False, + ) + return completed.returncode == 0 + + +@lru_cache(maxsize=4) +def _act_runtime_import_ready(discoverse_root_value: str, discoverse_commit: str, act_commit: str) -> bool: + discoverse_root = Path(discoverse_root_value) + act_root = discoverse_root / "policies" / "act" + if _git_head(discoverse_root) != discoverse_commit or _git_head(act_root) != act_commit or not _torch_cuda_ready(): + return False + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + completed = subprocess.run( + [sys.executable, "-c", "from policy import ACTPolicy; assert ACTPolicy is not None"], + cwd=act_root, + env=env, + text=True, + capture_output=True, + timeout=45, + check=False, + ) + return completed.returncode == 0 + + def _is_writable(path: Path) -> bool: try: path.mkdir(parents=True, exist_ok=True) diff --git a/src/real2sim_platform/auth.py b/src/real2sim_platform/auth.py new file mode 100644 index 0000000..d46b29a --- /dev/null +++ b/src/real2sim_platform/auth.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import hashlib +import re +import secrets +import time +from pathlib import Path +from typing import Any + +from .config import PlatformConfig +from .store import PlatformStore + + +SESSION_COOKIE = "real2sim_session" +VALID_ROLES = {"admin", "engineer", "viewer"} +_USERNAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$") + + +class AuthenticationError(ValueError): + pass + + +class AuthManager: + def __init__(self, store: PlatformStore, config: PlatformConfig) -> None: + self.store = store + self.config = config + self._hasher = _password_hasher() if config.auth_enabled else None + + def bootstrap_admin(self) -> dict[str, Any] | None: + if not self.config.auth_enabled or self.store.count_users() > 0: + return None + password = self.config.bootstrap_admin_password or secrets.token_urlsafe(18) + validate_password(password) + user = self.store.create_user( + username=self.config.bootstrap_admin_username, + password_hash=self.hash_password(password), + role="admin", + must_change_password=True, + ) + if not self.config.bootstrap_admin_password: + _write_bootstrap_file(self.config.auth_bootstrap_path, user["username"], password) + return user + + def hash_password(self, password: str) -> str: + validate_password(password) + if self._hasher is None: + raise RuntimeError("authentication is disabled") + return str(self._hasher.hash(password)) + + def verify_password(self, password_hash: str, password: str) -> bool: + if self._hasher is None: + return False + try: + return bool(self._hasher.verify(password_hash, password)) + except Exception: + return False + + def login(self, username: str, password: str) -> tuple[dict[str, Any], str]: + user = self.store.get_user_by_username(username.strip()) + if not user or user["disabled"] or not self.verify_password(str(user["password_hash"]), password): + raise AuthenticationError("用户名或密码错误") + token = secrets.token_urlsafe(48) + self.store.create_session( + token_hash=token_digest(token), + user_id=user["id"], + expires_at=time.time() + self.config.session_ttl_seconds, + ) + self.store.update_user(user["id"], last_login=time.time()) + return self.store.get_user(user["id"]), token + + def authenticate(self, token: str | None) -> dict[str, Any] | None: + if not self.config.auth_enabled: + return { + "id": "local-development", + "username": "local", + "role": "admin", + "disabled": False, + "must_change_password": False, + } + if not token: + return None + return self.store.get_session_user(token_digest(token)) + + def logout(self, token: str | None) -> None: + if token: + self.store.delete_session(token_digest(token)) + + def create_user(self, *, username: str, password: str, role: str) -> dict[str, Any]: + validate_username(username) + validate_role(role) + return self.store.create_user( + username=username.strip(), + password_hash=self.hash_password(password), + role=role, + must_change_password=True, + ) + + def change_password(self, user_id: str, *, current_password: str | None, new_password: str, admin_reset: bool = False) -> dict[str, Any]: + user_with_hash = self.store.get_user_by_username(self.store.get_user(user_id)["username"]) + if not user_with_hash: + raise KeyError(user_id) + if not admin_reset and not self.verify_password(str(user_with_hash["password_hash"]), current_password or ""): + raise AuthenticationError("当前密码不正确") + password_hash = self.hash_password(new_password) + self.store.delete_user_sessions(user_id) + return self.store.update_user(user_id, password_hash=password_hash, must_change_password=admin_reset) + + +def token_digest(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def validate_username(username: str) -> None: + if not _USERNAME_RE.fullmatch(username.strip()): + raise ValueError("用户名需为 3-64 位字母、数字、点、下划线或连字符") + + +def validate_role(role: str) -> None: + if role not in VALID_ROLES: + raise ValueError("role must be admin, engineer, or viewer") + + +def validate_password(password: str) -> None: + if len(password) < 12: + raise ValueError("密码至少需要 12 个字符") + classes = [any(c.islower() for c in password), any(c.isupper() for c in password), any(c.isdigit() for c in password), any(not c.isalnum() for c in password)] + if sum(classes) < 3: + raise ValueError("密码需包含大小写字母、数字或符号中的至少三类") + + +def _password_hasher(): + try: + from argon2 import PasswordHasher + from argon2.low_level import Type + except ImportError as exc: + raise RuntimeError("auth_enabled requires argon2-cffi") from exc + return PasswordHasher(time_cost=3, memory_cost=65536, parallelism=2, hash_len=32, salt_len=16, type=Type.ID) + + +def _write_bootstrap_file(path: Path, username: str, password: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"username={username}\npassword={password}\nforce_password_change=true\n", encoding="utf-8") + path.chmod(0o600) diff --git a/src/real2sim_platform/config.py b/src/real2sim_platform/config.py index 9367714..e2d387c 100644 --- a/src/real2sim_platform/config.py +++ b/src/real2sim_platform/config.py @@ -33,6 +33,24 @@ class PlatformConfig: cuda_home: Path = Path("/usr/local/cuda-12.1") runtime_env_script: Path = Path("/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh") allowed_path_roots: tuple[Path, ...] = () + worker_mode: str = "embedded" + worker_id: str = "local-worker" + production_mode: bool = False + auth_enabled: bool = False + secure_cookies: bool = False + session_ttl_seconds: int = 43200 + bootstrap_admin_username: str = "admin" + bootstrap_admin_password: str = "" + bos_endpoint: str = "https://bj.bcebos.com" + bos_bucket: str = "" + bos_prefix: str = "real2sim-poc" + bos_access_key_id: str = "" + bos_secret_access_key: str = "" + discoverse_commit: str = "d67f47c084aba0e0cf422a8725235f8b9238655a" + act_commit: str = "f8ab39c8d829d85571ab6d0250412ba4d0698fa0" + anysplat_commit: str = "5f5e208a7dd57d52e43ea0d553a95eab526e8775" + anysplat_model_revision: str = "d2e8c343672646041ad4ea518184968f94362f01" + vggt_model_revision: str = "860abec7937da0a4c03c41d3c269c366e82abdf9" @property def db_path(self) -> Path: @@ -62,6 +80,14 @@ class PlatformConfig: def manifests_dir(self) -> Path: return self.workspace_dir / "manifests" + @property + def upload_parts_dir(self) -> Path: + return self.workspace_dir / "upload_parts" + + @property + def auth_bootstrap_path(self) -> Path: + return self.workspace_dir / "bootstrap-admin.txt" + @property def effective_allowed_path_roots(self) -> tuple[Path, ...]: if self.allowed_path_roots: @@ -80,9 +106,7 @@ class PlatformConfig: def load_config(path: Path | None = None) -> PlatformConfig: config_path = path or Path(os.environ.get("REAL2SIM_PLATFORM_CONFIG", "")) - if not config_path or not config_path.is_file(): - return PlatformConfig() - values = _parse_simple_toml(config_path.read_text(encoding="utf-8")) + values = _parse_simple_toml(config_path.read_text(encoding="utf-8")) if config_path and config_path.is_file() else {} app_root = Path(str(values.get("app_root", APP_ROOT))) workspace_dir = Path(str(values.get("workspace_dir", WORKSPACE_ROOT))) return PlatformConfig( @@ -106,11 +130,29 @@ def load_config(path: Path | None = None) -> PlatformConfig: cuda_home=Path(str(values.get("cuda_home", "/usr/local/cuda-12.1"))), runtime_env_script=Path(str(values.get("runtime_env_script", "/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh"))), allowed_path_roots=_parse_path_tuple(str(values.get("allowed_path_roots", ""))), + worker_mode=str(values.get("worker_mode", os.environ.get("REAL2SIM_WORKER_MODE", "embedded"))), + worker_id=str(values.get("worker_id", os.environ.get("REAL2SIM_WORKER_ID", "local-worker"))), + production_mode=_as_bool(values.get("production_mode", os.environ.get("REAL2SIM_PRODUCTION_MODE", "0"))), + auth_enabled=_as_bool(values.get("auth_enabled", os.environ.get("REAL2SIM_AUTH_ENABLED", "0"))), + secure_cookies=_as_bool(values.get("secure_cookies", os.environ.get("REAL2SIM_SECURE_COOKIES", "0"))), + session_ttl_seconds=int(values.get("session_ttl_seconds", os.environ.get("REAL2SIM_SESSION_TTL_SECONDS", "43200"))), + bootstrap_admin_username=str(values.get("bootstrap_admin_username", os.environ.get("REAL2SIM_BOOTSTRAP_ADMIN_USERNAME", "admin"))), + bootstrap_admin_password=str(values.get("bootstrap_admin_password", os.environ.get("REAL2SIM_BOOTSTRAP_ADMIN_PASSWORD", ""))), + bos_endpoint=str(values.get("bos_endpoint", os.environ.get("BOS_ENDPOINT", "https://bj.bcebos.com"))), + bos_bucket=str(values.get("bos_bucket", os.environ.get("BOS_BUCKET", ""))), + bos_prefix=str(values.get("bos_prefix", os.environ.get("REAL2SIM_BOS_PREFIX", "real2sim-poc"))), + bos_access_key_id=str(values.get("bos_access_key_id", os.environ.get("BOS_ACCESS_KEY_ID", ""))), + bos_secret_access_key=str(values.get("bos_secret_access_key", os.environ.get("BOS_SECRET_ACCESS_KEY", ""))), + discoverse_commit=str(values.get("discoverse_commit", os.environ.get("REAL2SIM_DISCOVERSE_COMMIT", "d67f47c084aba0e0cf422a8725235f8b9238655a"))), + act_commit=str(values.get("act_commit", os.environ.get("REAL2SIM_ACT_COMMIT", "f8ab39c8d829d85571ab6d0250412ba4d0698fa0"))), + anysplat_commit=str(values.get("anysplat_commit", os.environ.get("REAL2SIM_ANYSPLAT_COMMIT", "5f5e208a7dd57d52e43ea0d553a95eab526e8775"))), + anysplat_model_revision=str(values.get("anysplat_model_revision", os.environ.get("REAL2SIM_ANYSPLAT_MODEL_REVISION", "d2e8c343672646041ad4ea518184968f94362f01"))), + vggt_model_revision=str(values.get("vggt_model_revision", os.environ.get("REAL2SIM_VGGT_MODEL_REVISION", "860abec7937da0a4c03c41d3c269c366e82abdf9"))), ) def ensure_workspace(config: PlatformConfig) -> None: - for path in [config.workspace_dir, config.uploads_dir, config.datasets_dir, config.scenes_dir, config.runs_dir, config.models_dir, config.manifests_dir]: + for path in [config.workspace_dir, config.uploads_dir, config.upload_parts_dir, config.datasets_dir, config.scenes_dir, config.runs_dir, config.models_dir, config.manifests_dir]: path.mkdir(parents=True, exist_ok=True) @@ -138,6 +180,21 @@ def write_default_config(path: Path, config: PlatformConfig | None = None) -> No f'cuda_home = "{cfg.cuda_home.as_posix()}"', f'runtime_env_script = "{cfg.runtime_env_script.as_posix()}"', f'allowed_path_roots = "{_format_path_tuple(cfg.effective_allowed_path_roots)}"', + f'worker_mode = "{cfg.worker_mode}"', + f'worker_id = "{cfg.worker_id}"', + f"production_mode = {str(cfg.production_mode).lower()}", + f"auth_enabled = {str(cfg.auth_enabled).lower()}", + f"secure_cookies = {str(cfg.secure_cookies).lower()}", + f"session_ttl_seconds = {cfg.session_ttl_seconds}", + f'bootstrap_admin_username = "{cfg.bootstrap_admin_username}"', + f'bos_endpoint = "{cfg.bos_endpoint}"', + f'bos_bucket = "{cfg.bos_bucket}"', + f'bos_prefix = "{cfg.bos_prefix}"', + f'discoverse_commit = "{cfg.discoverse_commit}"', + f'act_commit = "{cfg.act_commit}"', + f'anysplat_commit = "{cfg.anysplat_commit}"', + f'anysplat_model_revision = "{cfg.anysplat_model_revision}"', + f'vggt_model_revision = "{cfg.vggt_model_revision}"', "", ] path.write_text("\n".join(lines), encoding="utf-8") @@ -169,6 +226,9 @@ def _parse_simple_toml(text: str) -> dict[str, Any]: if value.startswith('"') and value.endswith('"'): values[key] = value[1:-1] else: + if value.lower() in {"true", "false"}: + values[key] = value.lower() == "true" + continue try: values[key] = int(value) except ValueError: @@ -184,3 +244,9 @@ def _parse_path_tuple(value: str) -> tuple[Path, ...]: def _format_path_tuple(paths: tuple[Path, ...]) -> str: return ":".join(path.as_posix() for path in paths) + + +def _as_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes", "on"} diff --git a/src/real2sim_platform/rm65_controller.py b/src/real2sim_platform/rm65_controller.py new file mode 100644 index 0000000..8fc4cd3 --- /dev/null +++ b/src/real2sim_platform/rm65_controller.py @@ -0,0 +1,379 @@ +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + + +@dataclass +class EpisodeResult: + success: bool + lift_height: float + place_error: float + bilateral_contact_steps: int + observations: list[np.ndarray] + actions: list[np.ndarray] + contacts: list[np.ndarray] + images: list[np.ndarray] + trajectory: list[dict[str, Any]] + initial_cube: list[float] + target: list[float] + + +class RM65PickPlaceRuntime: + def __init__(self, scene_path: Path, *, width: int, height: int, seed: int) -> None: + import mujoco + + self.mujoco = mujoco + self.model = mujoco.MjModel.from_xml_path(str(scene_path)) + self.data = mujoco.MjData(self.model) + self.renderer = mujoco.Renderer(self.model, height=height, width=width) + self.rng = np.random.default_rng(seed) + self.site_id = self._id(mujoco.mjtObj.mjOBJ_SITE, "gripper") + self.cube_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "pick_cube_free") + self.cube_qadr = int(self.model.jnt_qposadr[self.cube_joint]) + self.cube_geom = self._id(mujoco.mjtObj.mjOBJ_GEOM, "pick_cube_geom") + self.finger_geoms = { + self._id(mujoco.mjtObj.mjOBJ_GEOM, "left_finger_pad_z_axis"), + self._id(mujoco.mjtObj.mjOBJ_GEOM, "right_finger_pad_z_axis"), + } + self.target_body = self._id(mujoco.mjtObj.mjOBJ_BODY, "place_target") + self.gripper_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "gripper_Left_1_Joint") + self.gripper_qadr = int(self.model.jnt_qposadr[self.gripper_joint]) + self.joint_ids = [self._id(mujoco.mjtObj.mjOBJ_JOINT, f"joint_{index}") for index in range(1, 7)] + self.qpos_adrs = [int(self.model.jnt_qposadr[joint]) for joint in self.joint_ids] + self.dof_adrs = [int(self.model.jnt_dofadr[joint]) for joint in self.joint_ids] + self.ctrl_ranges = self.model.actuator_ctrlrange.copy() + + def _id(self, kind: Any, name: str) -> int: + value = self.mujoco.mj_name2id(self.model, kind, name) + if value < 0: + raise ValueError(f"required RM65 scene object is missing: {name}") + return int(value) + + def close(self) -> None: + self.renderer.close() + + def run_episode(self, *, max_steps: int) -> EpisodeResult: + mujoco = self.mujoco + mujoco.mj_resetDataKeyframe(self.model, self.data, 0) + cube = np.array([self.rng.uniform(0.645, 0.655), self.rng.uniform(-0.010, 0.010), 0.33], dtype=float) + target = np.array([self.rng.uniform(0.42, 0.51), self.rng.uniform(0.13, 0.22), 0.31], dtype=float) + self.data.qpos[self.cube_qadr : self.cube_qadr + 3] = cube + self.data.qpos[self.cube_qadr + 3 : self.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0]) + self.data.qvel[self.model.jnt_dofadr[self.cube_joint] : self.model.jnt_dofadr[self.cube_joint] + 6] = 0 + self.model.body_pos[self.target_body] = np.array([target[0], target[1], 0.335]) + mujoco.mj_forward(self.model, self.data) + for _ in range(120): + mujoco.mj_step(self.model, self.data) + settled_z = float(self.data.qpos[self.cube_qadr + 2]) + + # The named site is about 3 cm above the cube center at bilateral pad contact. + grasp_site = np.array([cube[0] - 0.014, cube[1] - 0.020, settled_z + 0.020]) + # Contact-calibrated negative-wrist branch for the packaged RM65 gripper. + # This seed is a robot calibration pose, not an object-pose override. + seed_q = np.array([-1.2468, -0.1011, 1.8666, -1.9230, -1.5982, 1.5636], dtype=float) + q_backup = self.data.qpos.copy() + self.data.qpos[self.qpos_adrs] = seed_q + mujoco.mj_forward(self.model, self.data) + grasp_orientation = self.data.site_xmat[self.site_id].reshape(3, 3).copy() + self.data.qpos[:] = q_backup + mujoco.mj_forward(self.model, self.data) + + phases = [ + ("approach", grasp_site + np.array([0.0, 0.0, 0.20]), 0.0, 24), + ("descend", grasp_site, 0.0, 24), + ("grasp", grasp_site, -0.28, 30), + ("lift", grasp_site + np.array([0.0, 0.0, 0.20]), -0.28, 60), + ("transfer", target + np.array([-0.040, 0.0, 0.21]), -0.28, 34), + ("place", target + np.array([-0.040, 0.0, 0.02]), -0.28, 26), + ("release", target + np.array([-0.040, 0.0, 0.02]), 0.0, 18), + ("retreat", target + np.array([-0.040, 0.0, 0.20]), 0.0, 20), + ] + scale = max(0.25, max_steps / sum(phase[3] for phase in phases)) + observations: list[np.ndarray] = [] + actions: list[np.ndarray] = [] + contacts: list[np.ndarray] = [] + images: list[np.ndarray] = [] + trajectory: list[dict[str, Any]] = [] + bilateral_contact_steps = 0 + max_cube_z = settled_z + step_index = 0 + for phase, position, gripper, nominal_steps in phases: + target_q = self.solve_ik_6d(position, grasp_orientation, seed_q=seed_q) + phase_steps = max(2, int(round(nominal_steps * scale))) + start_ctrl = self.data.ctrl[:6].copy() + for local_step in range(phase_steps): + alpha = _smoothstep((local_step + 1) / phase_steps) + action = np.empty(7, dtype=np.float32) + action[:6] = start_ctrl * (1 - alpha) + target_q * alpha + action[6] = gripper + self.data.ctrl[:7] = action + # The RM65 model has conservative actuator force limits; each policy + # command is held long enough for the physical joints to settle. + for _ in range(35): + mujoco.mj_step(self.model, self.data) + touching = self.finger_contacts() + bilateral_contact_steps += int(len(touching) == 2) + max_cube_z = max(max_cube_z, float(self.data.qpos[self.cube_qadr + 2])) + self.renderer.update_scene(self.data, camera="overview") + frame = self.renderer.render().copy() + observation = np.concatenate([self.data.qpos[self.qpos_adrs], [self.data.qpos[self.gripper_qadr]]]).astype(np.float32) + observations.append(observation) + actions.append(action.copy()) + contacts.append(np.array([int(geom in touching) for geom in sorted(self.finger_geoms)], dtype=np.uint8)) + images.append(frame) + trajectory.append( + { + "step": step_index, + "time": float(self.data.time), + "phase": phase, + "qpos": observation.tolist(), + "action": action.tolist(), + "cube_pos": self.data.qpos[self.cube_qadr : self.cube_qadr + 3].astype(float).tolist(), + "gripper_pos": self.data.site_xpos[self.site_id].astype(float).tolist(), + "bilateral_contact": len(touching) == 2, + "touching_fingers": sorted(int(item) for item in touching), + "finger_positions": [self.data.geom_xpos[item].astype(float).tolist() for item in sorted(self.finger_geoms)], + } + ) + step_index += 1 + + final_cube = self.data.qpos[self.cube_qadr : self.cube_qadr + 3].copy() + lift_height = float(max_cube_z - settled_z) + place_error = float(np.linalg.norm(final_cube - target)) + success = bilateral_contact_steps > 0 and lift_height >= 0.10 and place_error <= 0.05 + return EpisodeResult( + success=success, + lift_height=lift_height, + place_error=place_error, + bilateral_contact_steps=bilateral_contact_steps, + observations=observations, + actions=actions, + contacts=contacts, + images=images, + trajectory=trajectory, + initial_cube=cube.tolist(), + target=target.tolist(), + ) + + def finger_contacts(self) -> set[int]: + touching: set[int] = set() + for index in range(self.data.ncon): + contact = self.data.contact[index] + pair = {int(contact.geom1), int(contact.geom2)} + if self.cube_geom in pair: + touching.update(pair & self.finger_geoms) + return touching + + def solve_position_ik(self, target_position: np.ndarray) -> np.ndarray: + q_backup = self.data.qpos.copy() + q = self.data.qpos[self.qpos_adrs].copy() + for _ in range(120): + self.data.qpos[self.qpos_adrs] = q + self.mujoco.mj_forward(self.model, self.data) + error = target_position - self.data.site_xpos[self.site_id] + if np.linalg.norm(error) < 0.004: + break + jacp = np.zeros((3, self.model.nv)) + jacr = np.zeros((3, self.model.nv)) + self.mujoco.mj_jacSite(self.model, self.data, jacp, jacr, self.site_id) + jacobian = jacp[:, self.dof_adrs] + q += np.clip(_damped_least_squares(jacobian, error, 2e-3), -0.06, 0.06) + q = self._clip_arm(q) + self.data.qpos[:] = q_backup + self.mujoco.mj_forward(self.model, self.data) + return q + + def solve_ik_6d(self, target_position: np.ndarray, target_rotation: np.ndarray, *, seed_q: np.ndarray | None = None) -> np.ndarray: + q_backup = self.data.qpos.copy() + q = seed_q.copy() if seed_q is not None else self.data.qpos[self.qpos_adrs].copy() + for _ in range(160): + self.data.qpos[self.qpos_adrs] = q + self.mujoco.mj_forward(self.model, self.data) + position_error = target_position - self.data.site_xpos[self.site_id] + current_rotation = self.data.site_xmat[self.site_id].reshape(3, 3) + rotation_error = _rotation_vector(target_rotation @ current_rotation.T) + error = np.concatenate([position_error, 0.45 * rotation_error]) + if np.linalg.norm(position_error) < 0.005 and np.linalg.norm(rotation_error) < 0.04: + break + jacp = np.zeros((3, self.model.nv)) + jacr = np.zeros((3, self.model.nv)) + self.mujoco.mj_jacSite(self.model, self.data, jacp, jacr, self.site_id) + jacobian = np.vstack([jacp[:, self.dof_adrs], 0.45 * jacr[:, self.dof_adrs]]) + q += np.clip(_damped_least_squares(jacobian, error, 4e-3), -0.05, 0.05) + q = self._clip_arm(q) + self.data.qpos[:] = q_backup + self.mujoco.mj_forward(self.model, self.data) + return q + + def _clip_arm(self, q: np.ndarray) -> np.ndarray: + return np.array([np.clip(q[index], self.ctrl_ranges[index, 0], self.ctrl_ranges[index, 1]) for index in range(6)]) + + +def collect(scene_path: Path, out_dir: Path, *, required_successes: int, max_attempts: int, max_steps: int, seed: int, width: int, height: int) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + runtime = RM65PickPlaceRuntime(scene_path, width=width, height=height, seed=seed) + reports: list[dict[str, Any]] = [] + success_count = 0 + try: + for attempt in range(max_attempts): + result = runtime.run_episode(max_steps=max_steps) + episode_report = { + "attempt": attempt, + "success": result.success, + "lift_height": result.lift_height, + "place_error": result.place_error, + "bilateral_contact_steps": result.bilateral_contact_steps, + "initial_cube": result.initial_cube, + "target": result.target, + } + reports.append(episode_report) + trajectory_path = out_dir / f"attempt_{attempt:04d}_trajectory.json" + trajectory_path.write_text(json.dumps({"schema": "real2sim.rm65.trajectory.v2", "report": episode_report, "samples": result.trajectory}, ensure_ascii=False) + "\n", encoding="utf-8") + if result.success: + _write_hdf5_episode(out_dir / f"episode_{success_count}.hdf5", result) + if success_count < 3: + _write_preview_video(out_dir / f"episode_{success_count}.mp4", result.images) + success_count += 1 + if success_count >= required_successes: + break + finally: + runtime.close() + manifest = { + "schema": "real2sim.discoverse_act_dataset.v1", + "controller": "rm65_pick_place_6d_ik", + "runtime": "mujoco", + "scene": str(scene_path), + "camera_names": ["overview"], + "state_dim": 7, + "action_dim": 7, + "required_successes": required_successes, + "successful_episodes": success_count, + "attempts": len(reports), + "success_rate": success_count / max(1, len(reports)), + "success_definition": {"minimum_lift_m": 0.10, "maximum_place_error_m": 0.05, "bilateral_contact_required": True}, + "episodes": reports, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + (out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return manifest + + +def _write_hdf5_episode(path: Path, result: EpisodeResult) -> None: + import cv2 + import h5py + + observations = np.asarray(result.observations, dtype=np.float32) + actions = np.asarray(result.actions, dtype=np.float32) + contacts = np.asarray(result.contacts, dtype=np.uint8) + encoded = [] + for image in result.images: + ok, buffer = cv2.imencode(".jpg", image[:, :, ::-1], [int(cv2.IMWRITE_JPEG_QUALITY), 88]) + if not ok: + raise RuntimeError("failed to encode collection frame") + encoded.append(buffer.astype(np.uint8)) + with h5py.File(path, "w") as root: + root.attrs["sim"] = True + root.attrs["compress"] = True + root.attrs["success"] = bool(result.success) + root.create_dataset("/observations/qpos", data=observations, dtype="f4") + root.create_dataset("/action", data=actions, dtype="f4") + root.create_dataset("/observations/contact", data=contacts, dtype="u1") + root.create_dataset("/success", data=np.full(len(actions), result.success, dtype=np.uint8)) + variable = h5py.vlen_dtype(np.dtype("uint8")) + dataset = root.create_dataset("/observations/images/overview", (len(encoded),), dtype=variable) + for index, value in enumerate(encoded): + dataset[index] = value + + +def _write_preview_video(path: Path, images: list[np.ndarray]) -> None: + if not images: + raise ValueError("cannot encode an empty rollout") + height, width = images[0].shape[:2] + command = [ + "ffmpeg", + "-y", + "-loglevel", + "error", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-s", + f"{width}x{height}", + "-r", + "20", + "-i", + "-", + "-an", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + str(path), + ] + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + assert process.stdin is not None + for image in images: + process.stdin.write(np.ascontiguousarray(image, dtype=np.uint8).tobytes()) + process.stdin.close() + stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else "" + if process.wait() != 0 or not path.is_file() or path.stat().st_size == 0: + raise RuntimeError(f"ffmpeg rollout encoding failed: {stderr[-2000:]}") + + +def _damped_least_squares(jacobian: np.ndarray, error: np.ndarray, damping: float) -> np.ndarray: + identity = np.eye(jacobian.shape[0]) + return jacobian.T @ np.linalg.solve(jacobian @ jacobian.T + damping * identity, error) + + +def _rotation_vector(rotation: np.ndarray) -> np.ndarray: + cosine = float(np.clip((np.trace(rotation) - 1.0) * 0.5, -1.0, 1.0)) + angle = math.acos(cosine) + if angle < 1e-7: + return np.zeros(3) + axis = np.array([rotation[2, 1] - rotation[1, 2], rotation[0, 2] - rotation[2, 0], rotation[1, 0] - rotation[0, 1]]) + denominator = 2.0 * math.sin(angle) + return axis * (angle / denominator) + + +def _smoothstep(value: float) -> float: + value = float(np.clip(value, 0.0, 1.0)) + return value * value * (3.0 - 2.0 * value) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Collect physical RM65 pick-place episodes for DISCOVERSE ACT") + parser.add_argument("--scene", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--required-successes", type=int, default=50) + parser.add_argument("--max-attempts", type=int, default=150) + parser.add_argument("--steps", type=int, default=200) + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--width", type=int, default=320) + parser.add_argument("--height", type=int, default=240) + args = parser.parse_args() + manifest = collect( + args.scene, + args.out, + required_successes=args.required_successes, + max_attempts=args.max_attempts, + max_steps=args.steps, + seed=args.seed, + width=args.width, + height=args.height, + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2)) + return 0 if manifest["successful_episodes"] >= manifest["required_successes"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/real2sim_platform/runtime_gate.py b/src/real2sim_platform/runtime_gate.py new file mode 100644 index 0000000..0f490c7 --- /dev/null +++ b/src/real2sim_platform/runtime_gate.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +from huggingface_hub import snapshot_download + +from .adapters import run_doctor +from .config import PlatformConfig, ensure_workspace, load_config + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate the production Real2Sim GPU runtime") + parser.add_argument("--mode", choices=["bootstrap", "readiness"], default="readiness") + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + config = load_config() + ensure_workspace(config) + marker = marker_path(config) + if args.mode == "readiness": + return readiness(config, marker) + if marker.is_file() and not args.force: + try: + existing = json.loads(marker.read_text(encoding="utf-8")) + if existing.get("runtime_key") == runtime_key(config) and existing.get("ok") is True: + print(json.dumps(existing, ensure_ascii=False)) + return 0 + except (OSError, json.JSONDecodeError): + pass + report = bootstrap(config) + marker.parent.mkdir(parents=True, exist_ok=True) + temporary = marker.with_suffix(".tmp") + temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(marker) + print(json.dumps(report, ensure_ascii=False)) + return 0 if report["ok"] else 1 + + +def marker_path(config: PlatformConfig) -> Path: + root = Path(os.environ.get("REAL2SIM_RUNTIME_GATE_DIR", config.workspace_dir / "runtime_gate")) + return root / f"{runtime_key(config)}.json" + + +def runtime_key(config: PlatformConfig) -> str: + raw = ":".join( + [ + config.anysplat_commit, + config.anysplat_model_revision, + config.vggt_model_revision, + config.discoverse_commit, + config.act_commit, + ] + ) + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:20] + + +def readiness(config: PlatformConfig, marker: Path) -> int: + doctor = run_doctor(config) + if not marker.is_file(): + print("runtime bootstrap report is missing", file=sys.stderr) + return 1 + try: + report = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"runtime bootstrap report is unreadable: {exc}", file=sys.stderr) + return 1 + ready = bool(report.get("ok") and report.get("runtime_key") == runtime_key(config) and doctor.get("ok")) + if not ready: + print(json.dumps({"bootstrap": report, "doctor": doctor}, ensure_ascii=False), file=sys.stderr) + return 0 if ready else 1 + + +def bootstrap(config: PlatformConfig) -> dict[str, Any]: + started = time.time() + checks: dict[str, Any] = {} + doctor = run_doctor(config) + checks["doctor"] = {"ok": doctor.get("ok"), "core_checks": doctor.get("core_checks"), "checks": doctor.get("checks")} + try: + checks["nvidia"] = _nvidia_probe() + checks["simulators"] = _simulator_probe() + checks["act"] = _act_probe(config) + checks["anysplat"] = _anysplat_probe(config) + except Exception as exc: + checks["failure"] = {"type": type(exc).__name__, "message": str(exc)} + ok = bool(doctor.get("ok")) and all(bool(value.get("ok")) for key, value in checks.items() if key != "failure") and "failure" not in checks + return { + "schema": "real2sim.runtime_gate.v1", + "runtime_key": runtime_key(config), + "ok": ok, + "started_at": started, + "finished_at": time.time(), + "checks": checks, + "revisions": { + "anysplat_code": config.anysplat_commit, + "anysplat_model": config.anysplat_model_revision, + "vggt_model": config.vggt_model_revision, + "discoverse": config.discoverse_commit, + "act": config.act_commit, + }, + } + + +def _nvidia_probe() -> dict[str, Any]: + completed = subprocess.run( + ["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"], + text=True, + capture_output=True, + timeout=20, + check=True, + ) + import torch + + if not torch.cuda.is_available(): + raise RuntimeError("PyTorch CUDA is unavailable") + value = torch.ones((1024, 1024), device="cuda") + checksum = float((value @ value).mean().item()) + del value + torch.cuda.empty_cache() + return {"ok": checksum == 1024.0, "device": torch.cuda.get_device_name(0), "nvidia_smi": completed.stdout.strip().splitlines()[0]} + + +def _simulator_probe() -> dict[str, Any]: + xml = """""" + import mujoco + + model = mujoco.MjModel.from_xml_string(xml) + data = mujoco.MjData(model) + for _ in range(8): + mujoco.mj_step(model, data) + with tempfile.TemporaryDirectory(prefix="real2sim-runtime-gate-") as temporary: + scene_path = Path(temporary) / "scene.xml" + scene_path.write_text(xml, encoding="utf-8") + from motrixsim import SceneData, load_model, step + + motrix_model = load_model(str(scene_path)) + motrix_data = SceneData(motrix_model) + for _ in range(8): + step(motrix_model, motrix_data) + return {"ok": True, "mujoco_time": float(data.time), "motrix_timestep": float(motrix_model.options.timestep)} + + +def _act_probe(config: PlatformConfig) -> dict[str, Any]: + act_root = config.discoverse_root / "policies" / "act" + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(config.discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep) + completed = subprocess.run( + [sys.executable, "-c", "import torch; from policy import ACTPolicy; print(torch.__version__); assert ACTPolicy is not None and torch.cuda.is_available()"], + cwd=act_root, + env=env, + text=True, + capture_output=True, + timeout=60, + check=True, + ) + return {"ok": True, "torch": completed.stdout.strip().splitlines()[-1]} + + +def _anysplat_probe(config: PlatformConfig) -> dict[str, Any]: + cache_dir = Path(os.environ.get("HF_HOME", config.workspace_dir / "cache" / "huggingface")) + cache_dir.mkdir(parents=True, exist_ok=True) + anysplat_snapshot = Path( + snapshot_download( + repo_id="lhjiang/anysplat", + revision=config.anysplat_model_revision, + cache_dir=cache_dir, + ) + ) + snapshot_download(repo_id="facebook/VGGT-1B", revision=config.vggt_model_revision, cache_dir=cache_dir) + if str(config.anysplat_root) not in sys.path: + sys.path.insert(0, str(config.anysplat_root)) + from src.model.model.anysplat import AnySplat + + model = AnySplat.from_pretrained(str(anysplat_snapshot)) + model = model.to("cuda").eval() + parameters = sum(parameter.numel() for parameter in model.parameters()) + if parameters <= 0: + raise RuntimeError("AnySplat model loaded with zero parameters") + del model + gc.collect() + import torch + + torch.cuda.empty_cache() + return {"ok": True, "parameter_count": parameters, "snapshot": anysplat_snapshot.name} + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/real2sim_platform/server.py b/src/real2sim_platform/server.py index 267e3cc..2036869 100644 --- a/src/real2sim_platform/server.py +++ b/src/real2sim_platform/server.py @@ -1,24 +1,33 @@ from __future__ import annotations import argparse +import asyncio +from collections import defaultdict, deque import hashlib import json import os +import math +import secrets import shutil import subprocess +import threading import time import zipfile +from contextlib import asynccontextmanager from pathlib import Path from typing import Any +from urllib.parse import urlparse -from fastapi import FastAPI, File, HTTPException, UploadFile -from fastapi.responses import FileResponse, PlainTextResponse +from fastapi import FastAPI, File, HTTPException, Request, Response, UploadFile +from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel from . import __version__ from .adapters import SPARK_VENDOR_PATH, collect_task_reports, enrich_scene_manifest_assets, run_doctor, scan_dataset +from .auth import AuthManager, AuthenticationError, SESSION_COOKIE, validate_password, validate_role, validate_username from .config import PlatformConfig, ensure_workspace, is_path_allowed, load_config, write_default_config +from .storage import BosStorage, StorageBoundaryError, StorageUnavailable, safe_object_name, sha256_file from .store import PlatformStore from .worker import TaskWorker @@ -63,24 +72,95 @@ class BulkActionRequest(BaseModel): params: dict[str, Any] = {} +class LoginRequest(BaseModel): + username: str + password: str + + +class ChangePasswordRequest(BaseModel): + current_password: str = "" + new_password: str + + +class CreateUserRequest(BaseModel): + username: str + password: str + role: str = "engineer" + + +class UpdateUserRequest(BaseModel): + role: str | None = None + disabled: bool | None = None + reset_password: str | None = None + + +class CreateUploadRequest(BaseModel): + filename: str + content_type: str = "application/octet-stream" + total_size: int + chunk_size: int = 8 * 1024 * 1024 + checksum: str = "" + + +class ImportBosRequest(BaseModel): + prefix: str + name: str | None = None + max_objects: int = 1000 + + def create_app(config: PlatformConfig | None = None) -> FastAPI: cfg = config or load_config() ensure_workspace(cfg) store = PlatformStore(cfg.db_path) worker = TaskWorker(store, cfg) - app = FastAPI(title="Real2Sim Platform", version="0.1.0") + auth = AuthManager(store, cfg) + storage = BosStorage(cfg) + auth.bootstrap_admin() + login_failures: dict[str, deque[float]] = defaultdict(deque) + login_failure_lock = threading.Lock() + @asynccontextmanager + async def lifespan(_app: FastAPI): + if cfg.worker_mode == "embedded": + worker.start() + try: + yield + finally: + if cfg.worker_mode == "embedded": + worker.stop() + + app = FastAPI(title="Real2Sim Platform", version="0.1.0", lifespan=lifespan) static_dir = Path(__file__).parent / "static" app.mount("/static", StaticFiles(directory=static_dir), name="static") app.mount("/datasets", StaticFiles(directory=cfg.datasets_dir), name="datasets") - @app.on_event("startup") - def _startup() -> None: - store.recover_interrupted_tasks() - worker.start() + public_paths = {"/", "/api/health", "/api/version", "/api/capabilities", "/api/auth/login"} - @app.on_event("shutdown") - def _shutdown() -> None: - worker.stop() + @app.middleware("http") + async def authenticate_request(request: Request, call_next): + path = request.url.path + if not cfg.auth_enabled or path in public_paths or path.startswith("/static/"): + request.state.user = auth.authenticate(None) + return await call_next(request) + token = request.cookies.get(SESSION_COOKIE) + user = auth.authenticate(token) + if user is None: + return JSONResponse(status_code=401, content={"detail": "请先登录"}) + request.state.user = user + if user.get("must_change_password") and path not in {"/api/auth/me", "/api/auth/change-password", "/api/auth/logout"}: + return JSONResponse(status_code=428, content={"detail": "首次登录必须先修改密码", "code": "password_change_required"}) + if request.method not in {"GET", "HEAD", "OPTIONS"}: + self_service_paths = {"/api/auth/change-password", "/api/auth/logout"} + if user.get("role") == "viewer" and path not in self_service_paths: + return JSONResponse(status_code=403, content={"detail": "只读用户不能执行修改操作"}) + if path.startswith("/api/users") and user.get("role") != "admin": + return JSONResponse(status_code=403, content={"detail": "仅管理员可以管理用户"}) + origin = request.headers.get("origin") + origin_host = urlparse(origin).hostname if origin else None + if origin_host and origin_host != request.url.hostname: + return JSONResponse(status_code=403, content={"detail": "请求来源校验失败"}) + if path.startswith("/api/users") and user.get("role") != "admin": + return JSONResponse(status_code=403, content={"detail": "仅管理员可以管理用户"}) + return await call_next(request) @app.get("/") def index(): @@ -88,7 +168,110 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: @app.get("/api/health") def health() -> dict[str, Any]: - return {"ok": True, "workspace": str(cfg.workspace_dir)} + return {"ok": True, "worker_mode": cfg.worker_mode, "production_mode": cfg.production_mode} + + @app.post("/api/auth/login") + def login(payload: LoginRequest, request: Request, response: Response) -> dict[str, Any]: + if not cfg.auth_enabled: + return {"user": auth.authenticate(None), "auth_enabled": False} + login_key = f"{request.client.host if request.client else 'unknown'}:{payload.username.strip().lower()}" + now = time.time() + with login_failure_lock: + attempts = login_failures[login_key] + while attempts and now - attempts[0] > 900: + attempts.popleft() + if len(attempts) >= 8: + raise HTTPException(status_code=429, detail="登录失败次数过多,请 15 分钟后再试") + try: + user, token = auth.login(payload.username, payload.password) + except AuthenticationError as exc: + with login_failure_lock: + login_failures[login_key].append(time.time()) + time.sleep(0.2) + raise HTTPException(status_code=401, detail=str(exc)) from exc + with login_failure_lock: + login_failures.pop(login_key, None) + response.set_cookie( + SESSION_COOKIE, + token, + max_age=cfg.session_ttl_seconds, + httponly=True, + secure=cfg.secure_cookies, + samesite="lax", + path="/", + ) + return {"user": user, "auth_enabled": True} + + @app.post("/api/auth/logout") + def logout(request: Request, response: Response) -> dict[str, bool]: + auth.logout(request.cookies.get(SESSION_COOKIE)) + response.delete_cookie(SESSION_COOKIE, path="/") + return {"ok": True} + + @app.get("/api/auth/me") + def auth_me(request: Request) -> dict[str, Any]: + return {"user": request.state.user, "auth_enabled": cfg.auth_enabled} + + @app.post("/api/auth/change-password") + def change_password(payload: ChangePasswordRequest, request: Request, response: Response) -> dict[str, Any]: + try: + user = auth.change_password( + request.state.user["id"], + current_password=payload.current_password, + new_password=payload.new_password, + ) + except (AuthenticationError, ValueError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + response.delete_cookie(SESSION_COOKIE, path="/") + return {"ok": True, "user": user, "login_required": True} + + @app.get("/api/users") + def list_users() -> list[dict[str, Any]]: + return store.list_users() + + @app.post("/api/users") + def create_user(payload: CreateUserRequest) -> dict[str, Any]: + try: + return auth.create_user(username=payload.username, password=payload.password, role=payload.role) + except (ValueError, __import__("sqlite3").IntegrityError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.patch("/api/users/{user_id}") + def update_user(user_id: str, payload: UpdateUserRequest, request: Request) -> dict[str, Any]: + try: + target = store.get_user(user_id) + if payload.role is not None: + validate_role(payload.role) + if target["id"] == request.state.user["id"] and payload.disabled: + raise ValueError("不能禁用当前登录账号") + if target["role"] == "admin" and (payload.disabled or payload.role not in {None, "admin"}) and _active_admin_count(store) <= 1: + raise ValueError("系统至少需要一个可用管理员") + if payload.reset_password: + validate_password(payload.reset_password) + return auth.change_password(user_id, current_password=None, new_password=payload.reset_password, admin_reset=True) + result = store.update_user(user_id, role=payload.role, disabled=payload.disabled) + if payload.disabled: + store.delete_user_sessions(user_id) + return result + except KeyError as exc: + raise HTTPException(status_code=404, detail="用户不存在") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.delete("/api/users/{user_id}") + def delete_user(user_id: str, request: Request) -> dict[str, bool]: + try: + target = store.get_user(user_id) + if target["id"] == request.state.user["id"]: + raise ValueError("不能删除当前登录账号") + if target["role"] == "admin" and not target["disabled"] and _active_admin_count(store) <= 1: + raise ValueError("系统至少需要一个可用管理员") + store.delete_user(user_id) + return {"ok": True} + except KeyError as exc: + raise HTTPException(status_code=404, detail="用户不存在") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc @app.get("/api/version") def version() -> dict[str, Any]: @@ -120,7 +303,227 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: @app.get("/api/doctor") def doctor() -> dict[str, Any]: - return run_doctor(cfg) + result = run_doctor(cfg) + bos_health = storage.health() + sqlite_wal = _sqlite_journal_mode(store) == "wal" + result["checks"]["bos_storage"] = bos_health + result["checks"]["sqlite_wal"] = sqlite_wal + result["checks"]["worker_mode"] = {"ready": cfg.worker_mode in {"embedded", "api", "worker"}, "value": cfg.worker_mode} + heartbeats = store.list_worker_heartbeats() + latest = heartbeats[0] if heartbeats else None + heartbeat_age = time.time() - float(latest["updated_at"]) if latest else None + worker_ready = bool(latest and heartbeat_age is not None and heartbeat_age < 30 and latest["payload"].get("ready")) + result["worker"] = {"ready": worker_ready, "age_seconds": heartbeat_age, "latest": latest} + if cfg.production_mode and cfg.worker_mode == "api": + result["ok"] = bool(bos_health.get("ready") and sqlite_wal and worker_ready) + result["core_checks"] = ["bos_storage", "sqlite_wal", "worker"] + return result + + @app.get("/api/capabilities") + def capabilities() -> dict[str, Any]: + return _capabilities(cfg, storage, store) + + @app.get("/api/recipes") + def recipes() -> dict[str, Any]: + capabilities_payload = _capabilities(cfg, storage, store) + act_ready = bool(capabilities_payload["runtimes"]["discoverse_act"]["ready"]) + rm65_ready = bool(capabilities_payload["runtimes"]["rm65_controller"]["ready"]) + return { + "collection": [ + { + "id": "rm65_pick_place_6d_ik", + "name": "RM65 茶几抓取放置", + "enabled": rm65_ready, + "engine": "MuJoCo / MotrixSim", + "requirements": ["rm65", "parallel_gripper", "pick_place_target", "contact_sensors"], + "reason": "" if rm65_ready else "RM65 物理控制器或运行时尚未安装", + } + ], + "training": [ + { + "id": "discoverse_act", + "name": "DISCOVERSE ACT", + "enabled": act_ready, + "commit": cfg.act_commit, + "input_format": "discoverse_act_hdf5", + "reason": "" if act_ready else "DISCOVERSE ACT 固定版本运行时尚未就绪", + }, + {"id": "bc_v2", "name": "历史岭回归模型", "enabled": False, "read_only": True, "reason": "仅用于查看历史模型,禁止新建"}, + ], + "inference": [ + {"id": "discoverse_act", "name": "DISCOVERSE ACT 在线推理", "enabled": act_ready, "requires_real_simulator": True} + ], + } + + @app.post("/api/uploads") + def create_upload(payload: CreateUploadRequest, request: Request) -> dict[str, Any]: + if not storage.configured: + raise HTTPException(status_code=503, detail="BOS 存储尚未配置") + if payload.total_size <= 0: + raise HTTPException(status_code=400, detail="total_size must be > 0") + if not 1024 * 1024 <= payload.chunk_size <= 64 * 1024 * 1024: + raise HTTPException(status_code=400, detail="chunk_size must be between 1 MiB and 64 MiB") + if payload.checksum and (len(payload.checksum) != 64 or any(c not in "0123456789abcdefABCDEF" for c in payload.checksum)): + raise HTTPException(status_code=400, detail="checksum must be a SHA-256 hex digest") + filename = safe_object_name(payload.filename) + upload_id = secrets.token_hex(16) + local_dir = cfg.upload_parts_dir / upload_id + local_dir.mkdir(parents=True, exist_ok=False) + return store.create_upload( + item_id=upload_id, + owner_user_id=request.state.user.get("id"), + filename=filename, + content_type=payload.content_type, + total_size=payload.total_size, + chunk_size=payload.chunk_size, + total_parts=math.ceil(payload.total_size / payload.chunk_size), + checksum_expected=payload.checksum.lower(), + local_dir=local_dir, + ) + + @app.get("/api/uploads/{upload_id}") + def get_upload(upload_id: str, request: Request) -> dict[str, Any]: + try: + upload = store.get_upload(upload_id) + _check_upload_owner(upload, request.state.user) + return upload + except KeyError as exc: + raise HTTPException(status_code=404, detail="上传任务不存在") from exc + + @app.put("/api/uploads/{upload_id}/parts/{part_number}") + async def upload_part(upload_id: str, part_number: int, request: Request) -> dict[str, Any]: + try: + upload = store.get_upload(upload_id) + _check_upload_owner(upload, request.state.user) + except KeyError as exc: + raise HTTPException(status_code=404, detail="上传任务不存在") from exc + if upload["status"] != "uploading": + raise HTTPException(status_code=409, detail="上传任务已进入终态") + if not 1 <= part_number <= upload["total_parts"]: + raise HTTPException(status_code=400, detail="分片序号超出范围") + expected_size = upload["chunk_size"] + if part_number == upload["total_parts"]: + expected_size = upload["total_size"] - upload["chunk_size"] * (upload["total_parts"] - 1) + target = Path(upload["local_dir"]) / f"part-{part_number:08d}" + temporary = target.with_suffix(".tmp") + digest = hashlib.sha256() + written = 0 + try: + with temporary.open("wb") as handle: + async for chunk in request.stream(): + written += len(chunk) + if written > expected_size: + raise HTTPException(status_code=413, detail="分片大小超过声明值") + handle.write(chunk) + digest.update(chunk) + handle.flush() + os.fsync(handle.fileno()) + if written != expected_size: + raise HTTPException(status_code=400, detail=f"分片大小不匹配,期望 {expected_size} 字节,实际 {written} 字节") + temporary.replace(target) + finally: + if temporary.exists(): + temporary.unlink() + result = store.record_upload_part(upload_id, part_number) + return {"upload": result, "part_number": part_number, "size_bytes": written, "checksum": digest.hexdigest()} + + @app.post("/api/uploads/{upload_id}/complete") + def complete_upload(upload_id: str, request: Request) -> dict[str, Any]: + try: + upload = store.get_upload(upload_id) + _check_upload_owner(upload, request.state.user) + except KeyError as exc: + raise HTTPException(status_code=404, detail="上传任务不存在") from exc + if upload["status"] == "completed": + return {"upload": upload, "dataset": _dataset_for_storage_uri(store, upload["storage_uri"])} + missing = sorted(set(range(1, upload["total_parts"] + 1)) - set(upload["received_parts"])) + if missing: + raise HTTPException(status_code=409, detail={"message": "仍有分片未上传", "missing_parts": missing[:100]}) + local_dir = Path(upload["local_dir"]) + assembled = local_dir / upload["filename"] + digest = hashlib.sha256() + with assembled.open("wb") as output: + for part_number in range(1, upload["total_parts"] + 1): + part = local_dir / f"part-{part_number:08d}" + with part.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + output.write(chunk) + digest.update(chunk) + output.flush() + os.fsync(output.fileno()) + checksum = digest.hexdigest() + if assembled.stat().st_size != upload["total_size"] or (upload["checksum_expected"] and checksum != upload["checksum_expected"]): + store.fail_upload(upload_id, "assembled file checksum or size mismatch") + raise HTTPException(status_code=422, detail="完整文件校验失败,请续传错误分片后重试") + try: + uploaded = storage.upload_file( + assembled, + area="uploads", + relative_path=f"{upload_id}/{upload['filename']}", + metadata={"sha256": checksum, "upload-id": upload_id}, + ) + dataset_id = secrets.token_hex(16) + dataset_dir = cfg.datasets_dir / dataset_id + dataset_dir.mkdir(parents=True, exist_ok=False) + if assembled.suffix.lower() == ".zip": + with zipfile.ZipFile(assembled) as archive: + _safe_extract(archive, dataset_dir) + else: + shutil.copy2(assembled, dataset_dir / upload["filename"]) + scan = scan_dataset(dataset_dir) + manifest_path = dataset_dir / "dataset_manifest.json" + manifest = { + "schema": "real2sim.dataset_manifest.v2", + "id": dataset_id, + "name": Path(upload["filename"]).stem, + "source_type": "browser_upload", + "storage_uri": uploaded["storage_uri"], + "checksum": checksum, + "version": checksum[:12], + "scan": scan, + } + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + storage.upload_file(manifest_path, area="datasets", relative_path=f"{dataset_id}/dataset_manifest.json", metadata={"sha256": sha256_file(manifest_path)}) + dataset = store.create_dataset( + item_id=dataset_id, + name=manifest["name"], + path=dataset_dir, + scan=scan, + source_type="browser_upload", + source_uri=uploaded["storage_uri"], + version=manifest["version"], + manifest_path=manifest_path, + storage_uri=uploaded["storage_uri"], + checksum=checksum, + cache_state="local", + ) + completed = store.complete_upload(upload_id, checksum_actual=checksum, storage_uri=uploaded["storage_uri"]) + return {"upload": completed, "dataset": _with_view_model(dataset, "dataset")} + except (StorageUnavailable, StorageBoundaryError) as exc: + store.fail_upload(upload_id, str(exc)) + raise HTTPException(status_code=503, detail=str(exc)) from exc + + @app.delete("/api/uploads/{upload_id}") + def delete_upload(upload_id: str, request: Request) -> dict[str, bool]: + try: + upload = store.get_upload(upload_id) + _check_upload_owner(upload, request.state.user) + shutil.rmtree(Path(upload["local_dir"]), ignore_errors=True) + store.delete_upload(upload_id) + return {"ok": True} + except KeyError as exc: + raise HTTPException(status_code=404, detail="上传任务不存在") from exc + + @app.get("/api/storage/objects") + def list_storage_objects(area: str = "datasets", prefix: str = "", max_keys: int = 200) -> dict[str, Any]: + try: + return {"bucket": storage.bucket, "prefix": storage.prefix, "objects": storage.list_objects(area=area, relative_prefix=prefix, max_keys=max_keys)} + except (StorageUnavailable, StorageBoundaryError) as exc: + raise HTTPException(status_code=503 if isinstance(exc, StorageUnavailable) else 400, detail=str(exc)) from exc + + @app.post("/api/datasets/import-bos") + def import_bos_dataset(payload: ImportBosRequest) -> dict[str, Any]: + return _queue_task("bos_import", {"prefix": payload.prefix, "name": payload.name or Path(payload.prefix.rstrip("/")).name, "max_objects": payload.max_objects}) @app.get("/api/datasets") def list_datasets() -> list[dict[str, Any]]: @@ -138,6 +541,8 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: @app.post("/api/datasets/upload") def upload_dataset(file: UploadFile = File(...)) -> dict[str, Any]: + if cfg.production_mode: + raise HTTPException(status_code=410, detail="生产模式请使用 /api/uploads 分片上传,旧上传接口已禁用") target = cfg.uploads_dir / Path(file.filename or "upload.bin").name with target.open("wb") as handle: shutil.copyfileobj(file.file, handle) @@ -212,8 +617,15 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: raise HTTPException(status_code=404, detail="asset file not found") if not is_path_allowed(asset_path, cfg): raise HTTPException(status_code=403, detail="asset path is outside configured import whitelist") - manifest_path = _scene_manifest_path_for_write(scene) - manifest = _read_manifest_or_default(scene, manifest_path) + revision_id = secrets.token_hex(16) + revision_dir = cfg.scenes_dir / "revisions" / revision_id + source_scene_path = Path(scene["path"]) + source_package = source_scene_path if source_scene_path.is_dir() else source_scene_path.parent + _copy_scene_revision(source_package, revision_dir) + revision_scene_path = revision_dir / source_scene_path.name if source_scene_path.is_file() else revision_dir + source_manifest_path = _scene_manifest_path_for_write(scene) + manifest_path = revision_dir / source_manifest_path.name + manifest = _read_manifest_or_default(scene, source_manifest_path) artifact = store.add_artifact(task_id=None, name=payload.name or asset_path.name, path=asset_path, kind=(payload.kind or _scene_asset_kind(asset_path))) asset = _scene_asset_entry( asset_path, @@ -226,12 +638,28 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: assets = [item for item in manifest.get("assets", []) if not (isinstance(item, dict) and str(item.get("path") or "") == str(asset_path))] manifest["assets"] = [asset] + assets manifest["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + manifest["revision"] = {"id": revision_id, "parent_scene_id": scene_id, "reason": "asset_added"} manifest["asset_editing"] = {"supports_added_assets": True, "last_added_asset": asset["name"]} manifest_path.parent.mkdir(parents=True, exist_ok=True) manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - tags = list(dict.fromkeys([*(scene.get("tags") or []), "added_asset"])) - store.update_scene(scene_id, tags=tags) - return {"ok": True, "scene": store.get_scene(scene_id), "asset": asset, "manifest_path": str(manifest_path)} + tags = list(dict.fromkeys([*(scene.get("tags") or []), "added_asset", "scene_revision"])) + metadata = dict(scene.get("metadata") or {}) + metadata.update({"parent_scene_id": scene_id, "revision_reason": "asset_added", "package": str(revision_dir)}) + revision = store.create_scene( + item_id=revision_id, + name=f"{scene['name']} · 资产修订", + path=revision_scene_path, + metadata=metadata, + source_type="scene_revision", + source_uri=scene.get("storage_uri") or scene.get("source_uri") or str(source_scene_path), + version=_next_revision_label(str(scene.get("version") or "")), + parent_task_id=scene.get("parent_task_id"), + manifest_path=manifest_path, + metrics=scene.get("metrics") or {}, + tags=tags, + cache_state="local", + ) + return {"ok": True, "scene": revision, "parent_scene_id": scene_id, "asset": asset, "manifest_path": str(manifest_path)} @app.patch("/api/scenes/{scene_id}") def update_scene(scene_id: str, payload: UpdateEntityRequest) -> dict[str, Any]: @@ -277,9 +705,9 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: def _queue_task(task_type: str, params: dict[str, Any]) -> dict[str, Any]: payload = CreateTaskRequest(type=task_type, params=params) - _validate_task_payload(payload) + _validate_task_payload(payload, cfg) _validate_task_resources(store, payload) - if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "simulator_validation"}: + if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "simulator_validation"}: raise ValueError(f"unknown task type: {payload.type}") run_dir = cfg.runs_dir / payload.type / _next_run_name(cfg.runs_dir / payload.type, payload.type) return store.create_task(task_type=payload.type, params=payload.params, run_dir=run_dir) @@ -376,6 +804,22 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI: def list_tasks() -> list[dict[str, Any]]: return [_with_task_view_model(item) for item in store.list_tasks()] + @app.get("/api/events/tasks") + async def task_events(request: Request): + async def stream(): + previous = "" + while not await request.is_disconnected(): + tasks = [_with_task_view_model(item) for item in store.list_tasks()] + fingerprint = hashlib.sha256(json.dumps([(item["id"], item["status"], item["progress"], item["stage"], item["message"], item["updated_at"]) for item in tasks], ensure_ascii=False).encode("utf-8")).hexdigest() + if fingerprint != previous: + yield f"event: tasks\ndata: {json.dumps(tasks, ensure_ascii=False)}\n\n" + previous = fingerprint + else: + yield ": keepalive\n\n" + await asyncio.sleep(1.5) + + return StreamingResponse(stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + @app.post("/api/tasks") def create_task(payload: CreateTaskRequest) -> dict[str, Any]: try: @@ -493,6 +937,29 @@ def _scene_manifest_path_for_write(scene: dict[str, Any]) -> Path: return package_dir / "scene_manifest.json" +def _copy_scene_revision(source: Path, target: Path) -> None: + if not source.is_dir(): + raise FileNotFoundError(source) + + def link_or_copy(src: str, dst: str) -> str: + try: + os.link(src, dst) + return dst + except OSError: + return shutil.copy2(src, dst) + + shutil.copytree(source, target, copy_function=link_or_copy) + + +def _next_revision_label(current: str) -> str: + import re + + match = re.search(r"(?:^|[-_.])r(\d+)$", current, re.IGNORECASE) + number = int(match.group(1)) + 1 if match else 2 + base = re.sub(r"(?:[-_.])r\d+$", "", current, flags=re.IGNORECASE).strip("-_.") or "v1" + return f"{base}-r{number}" + + def _read_manifest_or_default(scene: dict[str, Any], manifest_path: Path) -> dict[str, Any]: if manifest_path.is_file(): payload = json.loads(manifest_path.read_text(encoding="utf-8")) @@ -735,8 +1202,15 @@ def _task_type_label(task_type: str) -> str: }.get(task_type, task_type) -def _validate_task_payload(payload: CreateTaskRequest) -> None: +def _validate_task_payload(payload: CreateTaskRequest, cfg: PlatformConfig) -> None: params = payload.params + if cfg.production_mode and bool(params.get("dry_run")): + raise HTTPException(status_code=400, detail="生产模式禁止 dry-run 任务") + if payload.type == "bos_import": + if not str(params.get("prefix") or "").strip(): + raise HTTPException(status_code=400, detail="bos_import requires prefix") + if not 1 <= int(params.get("max_objects", 1000)) <= 1000: + raise HTTPException(status_code=400, detail="max_objects must be between 1 and 1000") if payload.type == "hf_import": repo_type = params.get("repo_type", "dataset") if repo_type != "dataset": @@ -778,8 +1252,9 @@ def _validate_task_payload(payload: CreateTaskRequest) -> None: if payload.type == "expert_collection": if not (params.get("scene_id") or params.get("scene_path")): raise HTTPException(status_code=400, detail="expert_collection requires scene_id or scene_path") - if params.get("policy_preset", "pick_and_place_v1") != "pick_and_place_v1": - raise HTTPException(status_code=400, detail="当前仅支持抓取放置策略;导航和精细操作策略尚未接入执行器") + supported_policy = "rm65_pick_place_6d_ik" if cfg.production_mode else "pick_and_place_v1" + if params.get("policy_preset", supported_policy) != supported_policy: + raise HTTPException(status_code=400, detail=f"当前运行模式仅允许真实控制器 {supported_policy}") if int(params.get("workers", 1)) != 1: raise HTTPException(status_code=400, detail="当前专家采集执行器仅支持 workers=1") if int(params.get("episodes", 4)) <= 0 or int(params.get("steps", 24)) <= 0: @@ -787,8 +1262,9 @@ def _validate_task_payload(payload: CreateTaskRequest) -> None: if payload.type == "training": if not (params.get("dataset_id") or params.get("dataset_path")): raise HTTPException(status_code=400, detail="training requires dataset_id or dataset_path") - if params.get("recipe", "bc_v2") not in {"bc_v2", "bc_fast"}: - raise HTTPException(status_code=400, detail="当前仅支持行为克隆 v2 和快速验证;扩散策略尚未接入训练器") + allowed_recipes = {"discoverse_act"} if cfg.production_mode else {"bc_v2", "bc_fast", "discoverse_act"} + if params.get("recipe", "discoverse_act" if cfg.production_mode else "bc_v2") not in allowed_recipes: + raise HTTPException(status_code=400, detail="生产模式只允许 DISCOVERSE ACT;历史岭回归配方仅可查看") if int(params.get("epochs", 100)) <= 0: raise HTTPException(status_code=400, detail="epochs must be > 0") if payload.type == "inference": @@ -815,13 +1291,133 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) - raise HTTPException(status_code=400, detail="训练只能使用专家采集产生的 rollout 数据集") if payload.type == "inference": if params.get("model_id"): - store.get_model(str(params["model_id"])) + model = store.get_model(str(params["model_id"])) + recipe = str((model.get("metadata") or {}).get("training", {}).get("recipe") or (model.get("metadata") or {}).get("recipe") or "") + if params.get("recipe") == "discoverse_act" and recipe and recipe != "discoverse_act": + raise HTTPException(status_code=400, detail="所选模型不是 DISCOVERSE ACT 模型") if params.get("scene_id"): store.get_scene(str(params["scene_id"])) except KeyError as exc: raise HTTPException(status_code=400, detail=f"任务引用的资源不存在: {exc.args[0]}") from exc +def _active_admin_count(store: PlatformStore) -> int: + return sum(1 for user in store.list_users() if user["role"] == "admin" and not user["disabled"]) + + +def _check_upload_owner(upload: dict[str, Any], user: dict[str, Any]) -> None: + if user.get("role") != "admin" and upload.get("owner_user_id") != user.get("id"): + raise HTTPException(status_code=403, detail="无权访问其他用户的上传任务") + + +def _dataset_for_storage_uri(store: PlatformStore, storage_uri: str) -> dict[str, Any] | None: + return next((item for item in store.list_datasets() if item.get("storage_uri") == storage_uri), None) + + +def _sqlite_journal_mode(store: PlatformStore) -> str: + with store._connect() as db: + return str(db.execute("pragma journal_mode").fetchone()[0]).lower() + + +def _capabilities(cfg: PlatformConfig, storage: BosStorage, store: PlatformStore | None = None) -> dict[str, Any]: + anysplat_commit = _git_commit_full(cfg.anysplat_root) + discoverse_commit = _git_commit_full(cfg.discoverse_root) + act_root = cfg.discoverse_root / "policies" / "act" + act_commit = _git_commit_full(act_root) + ffmpeg_ready = shutil.which("ffmpeg") is not None + colmap_ready = cfg.colmap_binary.is_file() or shutil.which("colmap") is not None + mujoco_ready = _module_available("mujoco") + motrixsim_ready = _module_available("motrixsim") or bool(shutil.which("motrixsim")) + torch_ready = _module_available("torch") + rm65_controller = Path(__file__).with_name("rm65_controller.py") + local_runtime = { + "ffmpeg": ffmpeg_ready, + "colmap": colmap_ready, + "anysplat": cfg.anysplat_root.is_dir() and cfg.anysplat_python.is_file() and anysplat_commit == cfg.anysplat_commit, + "mujoco": mujoco_ready, + "motrixsim": motrixsim_ready, + "discoverse_act": cfg.discoverse_root.is_dir() and act_root.is_dir() and discoverse_commit == cfg.discoverse_commit and act_commit == cfg.act_commit and torch_ready, + "rm65_controller": rm65_controller.is_file() and mujoco_ready, + } + worker_status: dict[str, Any] = {"connected": cfg.worker_mode != "api", "age_seconds": 0 if cfg.worker_mode != "api" else None} + if store is not None and cfg.worker_mode == "api": + heartbeats = store.list_worker_heartbeats() + latest = heartbeats[0] if heartbeats else None + age = time.time() - float(latest["updated_at"]) if latest else None + connected = bool(latest and age is not None and age < 30) + worker_status = {"connected": connected, "age_seconds": age, "worker_id": latest.get("worker_id") if latest else None} + if connected: + checks = latest["payload"].get("checks", {}) + local_runtime.update( + ffmpeg=bool(checks.get("ffmpeg")), + colmap=bool(checks.get("colmap")), + anysplat=all(bool(checks.get(name)) for name in ("anysplat_root", "anysplat_python", "anysplat_pinned")), + mujoco=bool(checks.get("mujoco_python")), + motrixsim=bool(checks.get("motrixsim_python")), + discoverse_act=all(bool(checks.get(name)) for name in ("discoverse_act", "torch_cuda", "act_import")), + rm65_controller=bool(checks.get("rm65_controller")) and bool(checks.get("mujoco_python")), + ) + else: + for name in local_runtime: + local_runtime[name] = False + return { + "schema": "real2sim.capabilities.v1", + "production_mode": cfg.production_mode, + "auth_enabled": cfg.auth_enabled, + "worker_mode": cfg.worker_mode, + "worker": worker_status, + "storage": { + "bos": { + "configured": storage.configured, + "bucket": storage.bucket, + "prefix": storage.prefix, + "resumable_upload": True, + "sha256": True, + } + }, + "runtimes": { + "ffmpeg": {"ready": local_runtime["ffmpeg"]}, + "colmap": {"ready": local_runtime["colmap"]}, + "anysplat": { + "ready": local_runtime["anysplat"], + "commit": anysplat_commit, + "required_commit": cfg.anysplat_commit, + "model_revision": cfg.anysplat_model_revision, + "vggt_model_revision": cfg.vggt_model_revision, + }, + "mujoco": {"ready": local_runtime["mujoco"]}, + "motrixsim": {"ready": local_runtime["motrixsim"]}, + "discoverse_act": { + "ready": local_runtime["discoverse_act"], + "discoverse_commit": discoverse_commit, + "required_discoverse_commit": cfg.discoverse_commit, + "act_commit": act_commit, + "required_act_commit": cfg.act_commit, + }, + "rm65_controller": {"ready": local_runtime["rm65_controller"]}, + "spark": {"ready": SPARK_VENDOR_PATH.is_file()}, + }, + } + + +def _git_commit_full(root: Path) -> str: + if not root.is_dir(): + return "" + try: + return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=3).strip() + except Exception: + return "" + + +def _module_available(name: str) -> bool: + try: + import importlib.util + + return importlib.util.find_spec(name) is not None + except Exception: + return False + + def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]: return { "gs_playground_root": str(cfg.gs_playground_root), @@ -831,6 +1427,11 @@ def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]: "gsplat_python": str(cfg.gsplat_python), "cuda_home": str(cfg.cuda_home), "runtime_env_script": str(cfg.runtime_env_script), + "discoverse_commit": cfg.discoverse_commit, + "act_commit": cfg.act_commit, + "anysplat_commit": cfg.anysplat_commit, + "anysplat_model_revision": cfg.anysplat_model_revision, + "vggt_model_revision": cfg.vggt_model_revision, } diff --git a/src/real2sim_platform/static/app.js b/src/real2sim_platform/static/app.js index fe605df..c6f918c 100644 --- a/src/real2sim_platform/static/app.js +++ b/src/real2sim_platform/static/app.js @@ -1,4 +1,4 @@ -const STATIC_VERSION = "20260710-motrix-runtime-v7"; +const STATIC_VERSION = "20260710-poc-product-v1"; const state = { datasets: [], @@ -11,7 +11,13 @@ const state = { editing: null, assetLibrary: [], refreshTimer: null, + taskEvents: null, selections: {}, + user: null, + capabilities: null, + recipes: null, + initialized: false, + pendingPassword: "", }; const $ = (id) => document.getElementById(id); @@ -25,13 +31,97 @@ const taskScopes = { }; async function api(path, options = {}) { - const response = await fetch(path, options); + const response = await fetch(path, { credentials: "same-origin", ...options }); const type = response.headers.get("content-type") || ""; const payload = type.includes("application/json") ? await response.json() : await response.text(); - if (!response.ok) throw new Error(payload?.detail || payload?.error || response.statusText); + if (!response.ok) { + const detail = typeof payload?.detail === "object" ? payload.detail.message || JSON.stringify(payload.detail) : payload?.detail || payload?.error || response.statusText; + const error = new Error(detail); + error.status = response.status; + error.payload = payload; + throw error; + } return payload; } +function showAuth(mode = "login") { + $("appShell").hidden = true; + $("authShell").hidden = false; + $("loginForm").hidden = mode !== "login"; + $("passwordForm").hidden = mode !== "password"; + if (mode === "password" && state.pendingPassword) $("currentPassword").value = state.pendingPassword; +} + +function showApp() { + $("authShell").hidden = true; + $("appShell").hidden = false; + setText("currentUserLabel", `${state.user?.username || "本地用户"} · ${roleLabel(state.user?.role)}`); + const usersButton = $("openUsers"); + if (usersButton) usersButton.hidden = state.user?.role !== "admin"; +} + +async function bootstrap() { + state.capabilities = await api("/api/capabilities"); + let authState; + try { + authState = await api("/api/auth/me"); + } catch (error) { + if (error.status === 401) return showAuth("login"); + throw error; + } + state.user = authState.user; + if (state.user?.must_change_password) return showAuth("password"); + state.recipes = await api("/api/recipes"); + applyRuntimeOptions(); + showApp(); + await refresh(); + startTaskEvents(); + clearInterval(state.refreshTimer); + state.refreshTimer = setInterval(() => refresh({ quiet: true }).catch((error) => console.warn(error)), 15000); + state.initialized = true; +} + +function startTaskEvents() { + state.taskEvents?.close(); + state.taskEvents = new EventSource("/api/events/tasks"); + state.taskEvents.addEventListener("tasks", (event) => { + const previousTerminal = new Set(state.tasks.filter((task) => terminalStatus(task.status)).map((task) => task.id)); + state.tasks = JSON.parse(event.data); + renderScopedTaskLists(); + renderOverview(); + if (state.selectedTaskId && $("taskDrawer")?.classList.contains("open")) showTask(state.selectedTaskId, { preservePreview: true }).catch(console.warn); + if (state.tasks.some((task) => terminalStatus(task.status) && !previousTerminal.has(task.id))) refresh({ quiet: true }).catch(console.warn); + }); + state.taskEvents.onerror = () => console.warn("任务事件连接暂时中断,浏览器将自动重连"); +} + +function terminalStatus(status) { + return ["succeeded", "failed", "failed_recovered", "canceled"].includes(status); +} + +function roleLabel(role) { + return { admin: "管理员", engineer: "工程师", viewer: "只读" }[role] || "本地模式"; +} + +function applyRuntimeOptions() { + const collection = state.recipes?.collection || []; + const training = state.recipes?.training || []; + fillRecipeSelect("policyPreset", collection, "暂无可用真实控制器"); + fillRecipeSelect("trainRecipe", training.filter((item) => !item.read_only), "暂无可用训练配方"); +} + +function fillRecipeSelect(id, items, emptyLabel) { + const select = $(id); + if (!select) return; + const enabled = items.find((item) => item.enabled); + const unavailableReason = items.find((item) => !item.enabled)?.reason; + select.innerHTML = enabled + ? items.map((item) => ``).join("") + : `${items.map((item) => ``).join("")}`; + select.value = enabled?.id || ""; + select.disabled = !enabled; +} + async function refresh({ quiet = false } = {}) { try { const [datasets, scenes, models, tasks] = await Promise.all([ @@ -77,13 +167,15 @@ function renderAll() { } function updateActionAvailability() { - setActionAvailability("createMediaPrepare", state.datasets.length > 0, "请先新增数据集"); - setActionAvailability("createPreflightTask", preparedDatasets().length > 0, "请先准备帧数据"); - setActionAvailability("createSceneTask", preparedDatasets().length > 0, "请先准备帧数据"); - setActionAvailability("createCollectTask", state.scenes.length > 0, "请先完成场景重建"); - setActionAvailability("createTrainTask", collectionDatasets().length > 0, "请先完成专家采集"); - setActionAvailability("createInferTask", state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型"); - setActionAvailability("addSceneAsset", state.scenes.length > 0, "请先完成场景重建"); + const writable = state.user?.role !== "viewer"; + document.body.classList.toggle("viewer-mode", !writable); + setActionAvailability("createMediaPrepare", writable && state.datasets.length > 0, writable ? "请先新增数据集" : "只读用户不能创建任务"); + setActionAvailability("createPreflightTask", writable && preparedDatasets().length > 0, "请先准备帧数据"); + setActionAvailability("createSceneTask", writable && preparedDatasets().length > 0, "请先准备帧数据"); + setActionAvailability("createCollectTask", writable && state.scenes.length > 0 && Boolean($("policyPreset")?.value), "请先选择兼容真实控制器的场景"); + setActionAvailability("createTrainTask", writable && collectionDatasets().length > 0 && Boolean($("trainRecipe")?.value), "请先完成真实专家采集并确认 ACT 运行时"); + setActionAvailability("createInferTask", writable && state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型"); + setActionAvailability("addSceneAsset", writable && state.scenes.length > 0, "请先完成场景重建"); } function setActionAvailability(id, enabled, reason) { @@ -293,14 +385,24 @@ function renderTrainingChart() { const best = state.models[0]; const score = best ? modelScore(best) : 0; const loss = best?.metrics?.loss ?? best?.metrics?.val_loss ?? null; + if (!best) { + setHtml("trainingChart", emptyState("完成一次真实训练后,这里会显示损失曲线和验证指标。")); + return; + } + const history = [best?.metrics?.loss_history, best?.metrics?.train_loss_history, best?.metrics?.loss_curve] + .find((value) => Array.isArray(value) && value.length > 1) || []; + const finiteHistory = history.map(Number).filter(Number.isFinite).slice(-48); + const maxLoss = finiteHistory.length ? Math.max(...finiteHistory) : 0; + const minLoss = finiteHistory.length ? Math.min(...finiteHistory) : 0; + const range = Math.max(maxLoss - minLoss, Number.EPSILON); setHtml("trainingChart", `
-
成功率${best ? `${Math.round(score)}%` : "--"}
+
成功率${Number.isFinite(score) ? `${Math.round(score)}%` : "--"}
验证损失${loss !== null ? Number(loss).toFixed(3) : "--"}
-
- ${[35, 48, 54, 63, Math.max(38, score || 72), Math.max(44, Math.min(96, (score || 76) + 8))].map((height) => ``).join("")} -
+ ${finiteHistory.length > 1 + ? `
${finiteHistory.map((value) => ``).join("")}
` + : `
该模型没有可绘制的损失历史;可在模型卡中查看最终指标。
`} `); } @@ -653,7 +755,7 @@ function showBulkResult(title, result) { function openImportDialog(type) { state.currentImport = type; - const title = { hf: "导入 HuggingFace 数据", upload: "上传数据文件", local: "导入本地目录" }[type] || "新增数据"; + const title = { hf: "导入 HuggingFace 数据", upload: "上传数据文件", bos: "从 BOS 导入", local: "导入本地目录" }[type] || "新增数据"; setText("importTitle", title); setText("importSubtitle", "主流程只展示资源选择;技术参数集中在这里配置。"); if (type === "hf") { @@ -672,7 +774,18 @@ function openImportDialog(type) { `); } else if (type === "upload") { - setHtml("importFields", ``); + setHtml("importFields", ` + +
等待选择文件
+

上传会按 8MB 分片写入 BOS,刷新或网络中断后重新选择同一文件即可续传。

+ `); + } else if (type === "bos") { + setHtml("importFields", ` + + + +

只能访问 real2sim-poc/ 下的授权区域,无法列举或删除其他项目数据。

+ `); } else { setHtml("importFields", ` @@ -695,10 +808,24 @@ async function submitImport(event) { } else if (state.currentImport === "upload") { const file = $("uploadFile")?.files?.[0]; if (!file) return toast("请先选择文件。"); - const body = new FormData(); - body.append("file", file); - await api("/api/datasets/upload", { method: "POST", body }); - toast("文件已上传并登记为数据集"); + if (state.capabilities?.storage?.bos?.configured) { + await uploadFileResumable(file); + } else { + const body = new FormData(); + body.append("file", file); + await api("/api/datasets/upload", { method: "POST", body }); + } + toast("文件已校验并登记为数据集"); + await refresh({ quiet: true }); + } else if (state.currentImport === "bos") { + const prefix = $("bosPrefix")?.value.trim(); + if (!prefix) return toast("请填写授权前缀。"); + await api("/api/datasets/import-bos", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ prefix: `${$("bosArea")?.value || "datasets"}/${prefix}`, name: $("bosDatasetName")?.value.trim() || undefined }), + }); + toast("BOS 导入任务已创建"); await refresh({ quiet: true }); } else { const path = $("datasetPath")?.value.trim(); @@ -714,6 +841,50 @@ async function submitImport(event) { $("importDialog")?.close(); } +async function uploadFileResumable(file) { + const chunkSize = 8 * 1024 * 1024; + const resumeKey = `real2sim-upload:${file.name}:${file.size}:${file.lastModified}`; + let upload = null; + const savedId = localStorage.getItem(resumeKey); + if (savedId) { + try { + upload = await api(`/api/uploads/${savedId}`); + if (upload.filename !== file.name || upload.total_size !== file.size || upload.status !== "uploading") upload = null; + } catch (_error) { + upload = null; + } + } + if (!upload) { + upload = await api("/api/uploads", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ filename: file.name, content_type: file.type || "application/octet-stream", total_size: file.size, chunk_size: chunkSize }), + }); + localStorage.setItem(resumeKey, upload.id); + } + const received = new Set(upload.received_parts || []); + for (let part = 1; part <= upload.total_parts; part += 1) { + if (received.has(part)) continue; + const start = (part - 1) * upload.chunk_size; + const end = Math.min(file.size, start + upload.chunk_size); + await api(`/api/uploads/${upload.id}/parts/${part}`, { method: "PUT", headers: { "Content-Type": "application/octet-stream" }, body: file.slice(start, end) }); + received.add(part); + renderUploadProgress(received.size / upload.total_parts, `已上传 ${received.size}/${upload.total_parts} 个分片`); + } + renderUploadProgress(0.98, "正在校验 SHA-256 并写入 BOS"); + const completed = await api(`/api/uploads/${upload.id}/complete`, { method: "POST" }); + localStorage.removeItem(resumeKey); + renderUploadProgress(1, `上传完成 · ${completed.upload.checksum_actual.slice(0, 12)}`); + return completed; +} + +function renderUploadProgress(value, label) { + const node = $("uploadProgress"); + if (!node) return; + node.querySelector("span").style.width = `${Math.max(0, Math.min(100, value * 100))}%`; + node.querySelector("strong").textContent = label; +} + function openViewer(sceneId) { state.activeSceneId = sceneId; const assetScene = $("assetScene"); @@ -726,9 +897,19 @@ function openViewer(sceneId) { } function openViewerFullscreen(url) { - const viewerUrl = url || $("openSceneViewerFull")?.dataset.viewerUrl || $("scenePreview")?.getAttribute("src"); - if (!viewerUrl) return toast("请先选择一个场景。"); - window.open(viewerUrl, "_blank", "noopener,noreferrer"); + const panel = document.querySelector(".viewer-panel"); + const button = $("openSceneViewerFull"); + const iframe = $("scenePreview"); + const viewerUrl = url || button?.dataset.viewerUrl || iframe?.getAttribute("src"); + if (!panel || !viewerUrl) return toast("请先选择一个场景。"); + const maximizing = !panel.classList.contains("is-maximized"); + panel.classList.toggle("is-maximized", maximizing); + document.body.classList.toggle("viewer-maximized", maximizing); + if (button) { + button.textContent = maximizing ? "退出最大化" : "最大化预览"; + button.setAttribute("aria-pressed", String(maximizing)); + } + if (iframe && maximizing && !iframe.getAttribute("src")) iframe.src = viewerUrl; } function openAssetEditor(sceneId) { @@ -822,7 +1003,7 @@ function reconstructionOptions() { scene_name: $("sceneName")?.value.trim() || undefined, renderer: $("rendererSelect")?.value || "auto", simulators: simulatorValues(), - dry_run: $("dryRunScene")?.checked || false, + dry_run: false, auto_prepare: $("autoPrepareScene")?.checked || false, allow_low_quality: $("allowLowQualityScene")?.checked || false, min_frames: Number($("sceneMinFrames")?.value || 30), @@ -1267,7 +1448,100 @@ function toast(message) { node._timer = setTimeout(() => node.classList.remove("show"), 2600); } +async function submitLogin(event) { + event.preventDefault(); + setText("loginError", ""); + const username = $("loginUsername").value.trim(); + const password = $("loginPassword").value; + try { + const result = await api("/api/auth/login", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ username, password }), + }); + state.user = result.user; + state.pendingPassword = password; + if (state.user?.must_change_password) { + showAuth("password"); + return; + } + state.pendingPassword = ""; + await bootstrap(); + } catch (error) { + setText("loginError", error.message || "登录失败"); + } +} + +async function submitPasswordChange(event) { + event.preventDefault(); + setText("passwordError", ""); + const currentPassword = $("currentPassword").value; + const newPassword = $("newPassword").value; + if (newPassword !== $("confirmPassword").value) return setText("passwordError", "两次输入的新密码不一致"); + try { + await api("/api/auth/change-password", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ current_password: currentPassword, new_password: newPassword }), + }); + state.pendingPassword = ""; + $("loginPassword").value = ""; + showAuth("login"); + setText("loginError", "密码已更新,请使用新密码登录"); + } catch (error) { + setText("passwordError", error.message || "密码更新失败"); + } +} + +async function logout() { + try { + await api("/api/auth/logout", { method: "POST" }); + } finally { + state.taskEvents?.close(); + clearInterval(state.refreshTimer); + state.user = null; + state.initialized = false; + showAuth("login"); + } +} + +async function openUserManager() { + await renderUsers(); + $("usersDialog")?.showModal(); +} + +async function renderUsers() { + const users = await api("/api/users"); + setHtml("userList", users.map((user) => ` +
+
${escapeHtml(user.username)}${escapeHtml(roleLabel(user.role))} · ${user.disabled ? "已禁用" : user.must_change_password ? "待首次改密" : "可用"}
+
+ + + +
+
+ `).join("") || emptyState("暂无团队用户。")); +} + +async function createUser(event) { + event.preventDefault(); + await api("/api/users", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ username: $("newUsername").value.trim(), password: $("newUserPassword").value, role: $("newUserRole").value }), + }); + event.target.reset(); + toast("用户已创建,首次登录必须修改密码"); + await renderUsers(); +} + function bindEvents() { + $("loginForm")?.addEventListener("submit", submitLogin); + $("passwordForm")?.addEventListener("submit", submitPasswordChange); + $("logoutButton")?.addEventListener("click", logout); + $("openUsers")?.addEventListener("click", openUserManager); + $("createUserForm")?.addEventListener("submit", createUser); document.querySelectorAll(".tab").forEach((button) => button.addEventListener("click", () => switchTab(button.dataset.tab))); $("closeDrawer")?.addEventListener("click", closeDrawer); $("importForm")?.addEventListener("submit", submitImport); @@ -1323,13 +1597,13 @@ function bindEvents() { if (!sceneId) return toast("请先选择一个场景。"); const payload = sceneAssetPayload(); if (!payload.path) return toast("请选择资产来源,或在高级入口填写白名单文件。"); - await api(`/api/scenes/${sceneId}/assets`, { + const result = await api(`/api/scenes/${sceneId}/assets`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload), }); - state.activeSceneId = sceneId; - toast("资产已添加到场景清单"); + state.activeSceneId = result.scene?.id || sceneId; + toast("资产已添加,并创建了新的场景修订版本"); await refresh({ quiet: true }); renderSceneViewerShell(); scrollIntoViewIfPossible("#assetRail"); @@ -1358,7 +1632,15 @@ function bindEvents() { const sceneId = $("inferScene")?.value || state.scenes[0]?.id; if (!modelId) return toast("请先训练模型。"); if (!sceneId) return toast("请先完成场景重建。"); - await createTask("inference", { model_id: modelId, scene_id: sceneId, steps: Number($("inferSteps")?.value || 80), workers: Number($("inferWorkers")?.value || 1) }); + await createTask("inference", { + model_id: modelId, + scene_id: sceneId, + recipe: "discoverse_act", + episodes: Number($("inferEpisodes")?.value || 10), + steps: Number($("inferSteps")?.value || 240), + workers: Number($("inferWorkers")?.value || 1), + minimum_success_rate: 0.5, + }); }); document.body.addEventListener("change", (event) => { @@ -1378,6 +1660,37 @@ async function handleBodyClick(event) { const closeDialog = event.target.closest("[data-close-dialog]"); if (closeDialog) $(closeDialog.dataset.closeDialog)?.close(); + const userToggle = event.target.closest("[data-user-toggle]"); + if (userToggle) { + await api(`/api/users/${userToggle.dataset.userToggle}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ disabled: userToggle.dataset.userDisabled !== "true" }), + }); + await renderUsers(); + } + + const userReset = event.target.closest("[data-user-reset]"); + if (userReset) { + const password = prompt("输入至少 12 位的新初始密码;用户下次登录仍需修改密码。") || ""; + if (password) { + await api(`/api/users/${userReset.dataset.userReset}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ reset_password: password }), + }); + toast("密码已重置"); + await renderUsers(); + } + } + + const userDelete = event.target.closest("[data-user-delete]"); + if (userDelete && confirm("删除用户会终止其会话,但不会删除业务资源。确认继续?")) { + await api(`/api/users/${userDelete.dataset.userDelete}`, { method: "DELETE" }); + toast("用户已删除"); + await renderUsers(); + } + const openImport = event.target.closest("[data-open-import]"); if (openImport) openImportDialog(openImport.dataset.openImport); @@ -1482,9 +1795,20 @@ async function handleBodyClick(event) { } } -window.addEventListener("unhandledrejection", (event) => toast(event.reason?.message || String(event.reason || "操作失败"))); +window.addEventListener("unhandledrejection", (event) => { + if (event.reason?.status === 401) showAuth("login"); + if (event.reason?.status === 428) showAuth("password"); + toast(event.reason?.message || String(event.reason || "操作失败")); +}); window.addEventListener("error", (event) => event.message && toast(event.message)); +window.addEventListener("keydown", (event) => { + if (event.key === "Escape" && document.querySelector(".viewer-panel.is-maximized")) { + openViewerFullscreen(); + } +}); bindEvents(); -refresh().catch((error) => toast(error.message)); -state.refreshTimer = setInterval(() => refresh({ quiet: true }).catch((error) => console.warn(error)), 4000); +bootstrap().catch((error) => { + toast(error.message || "平台初始化失败"); + if (error.status === 401) showAuth("login"); +}); diff --git a/src/real2sim_platform/static/index.html b/src/real2sim_platform/static/index.html index 7482a7e..84edf1d 100644 --- a/src/real2sim_platform/static/index.html +++ b/src/real2sim_platform/static/index.html @@ -5,10 +5,43 @@ Real2Sim - + -
+
+
+ Real2Sim + 机器人从真实场景走向可训练智能体 + +

数据、场景、采集、训练与推理使用同一条可追踪链路。

+
+
+
+

登录平台

+

使用管理员分配的账号进入工作台

+
+ + + + +
+ +
+ + @@ -228,7 +266,6 @@ -
@@ -362,10 +399,10 @@
- + - - + +
@@ -417,7 +454,7 @@
- +
@@ -488,7 +525,8 @@
- + +
@@ -580,7 +618,23 @@ + +
+
+

用户管理

创建、禁用和重置团队账号

+ +
+
+ + + + +
+
+
+
+
- + diff --git a/src/real2sim_platform/static/styles.css b/src/real2sim_platform/static/styles.css index 096e872..e86cccc 100644 --- a/src/real2sim_platform/static/styles.css +++ b/src/real2sim_platform/static/styles.css @@ -1,2032 +1,324 @@ :root { - --bg: #071019; - --bg-2: #0a1420; - --surface: rgba(14, 29, 43, 0.88); - --surface-2: rgba(18, 38, 54, 0.9); - --surface-3: rgba(8, 20, 31, 0.72); - --line: rgba(155, 185, 203, 0.17); - --line-strong: rgba(54, 226, 202, 0.45); - --text: #eef7fb; - --muted: #9db1bf; - --muted-2: #6f8797; - --accent: #34dfc7; - --accent-strong: #54f0d9; - --blue: #5da9ff; - --amber: #ffc35a; - --coral: #ff776c; - --lime: #8ee86b; - --radius: 8px; - --shadow: 0 22px 70px rgba(0, 0, 0, 0.34); - --font: Inter, "PingFang SC", "Microsoft YaHei", system-ui, -apple-system, BlinkMacSystemFont, sans-serif; + --bg: #070b10; + --sidebar: #080d13; + --surface: #0d1721; + --surface-2: #101e2a; + --surface-3: #09131c; + --line: #203340; + --line-soft: #172833; + --text: #edf5f6; + --muted: #91a7ae; + --muted-2: #617880; + --accent: #28d6bd; + --accent-2: #13a98f; + --blue: #57a8ff; + --amber: #f2b84b; + --coral: #ff7268; + --green: #68d391; + --radius: 6px; + --shadow: 0 18px 48px rgba(0, 0, 0, 0.3); + --font: Inter, "PingFang SC", "Microsoft YaHei", system-ui, -apple-system, sans-serif; } * { box-sizing: border-box; } - -html, body { - min-height: 100%; - margin: 0; - background: - linear-gradient(135deg, rgba(52, 223, 199, 0.08), transparent 34%), - linear-gradient(215deg, rgba(93, 169, 255, 0.08), transparent 36%), - linear-gradient(180deg, #06101a, var(--bg)); - color: var(--text); - font-family: var(--font); -} - -body { - overflow-x: hidden; -} - -button, input, select, textarea { - font: inherit; -} - +html, body { min-height: 100%; margin: 0; } +body { overflow-x: hidden; background: var(--bg); color: var(--text); font-family: var(--font); font-size: 13px; letter-spacing: 0; } +button, input, select, textarea { font: inherit; letter-spacing: 0; } button { - min-height: 34px; + min-height: 30px; border: 1px solid var(--line); - border-radius: 7px; - padding: 7px 12px; - background: rgba(10, 24, 36, 0.78); + border-radius: 5px; + padding: 5px 10px; + background: #0b1822; color: var(--text); + font-size: 12px; + font-weight: 700; cursor: pointer; - transition: border-color 150ms ease, background 150ms ease, color 150ms ease, transform 150ms ease; + transition: border-color 140ms ease, background 140ms ease, color 140ms ease, transform 140ms ease; } - -button:hover { - border-color: var(--line-strong); - background: rgba(27, 54, 70, 0.88); -} - -button:active { - transform: translateY(1px); -} - -button:disabled, -select:disabled, -input:disabled { - cursor: not-allowed; - opacity: 0.58; -} - -.primary-button { - border-color: rgba(52, 223, 199, 0.62); - background: linear-gradient(135deg, #38e8d1, #34c99f); - color: #041512; - font-weight: 820; - box-shadow: 0 14px 32px rgba(52, 223, 199, 0.18); -} - -.primary-button:hover { - background: linear-gradient(135deg, #54f0d9, #42dcae); - color: #02110e; -} - -.ghost-button { - background: rgba(8, 20, 31, 0.65); -} - -.danger-button { - border-color: rgba(255, 119, 108, 0.48); - color: #ffb3ae; -} - -.danger-button:hover { - border-color: rgba(255, 119, 108, 0.82); - background: rgba(92, 32, 35, 0.62); -} - +button:hover { border-color: #39606c; background: #112632; } +button:active { transform: translateY(1px); } +button:disabled, input:disabled, select:disabled { cursor: not-allowed; opacity: .48; } +.primary-button { border-color: #35e0c8; background: var(--accent); color: #03120f; box-shadow: 0 8px 22px rgba(40, 214, 189, .16); } +.primary-button:hover { background: #45e4ce; color: #020c0a; } +.ghost-button { background: transparent; } +.danger-button { border-color: #7a3a3a; color: #ffaaa4; background: #1c1216; } +.danger-button:hover { border-color: var(--coral); background: #30171a; } input, select, textarea { width: 100%; - min-height: 38px; + min-height: 34px; border: 1px solid var(--line); - border-radius: 7px; - padding: 9px 11px; - background: rgba(5, 15, 24, 0.82); - color: var(--text); + border-radius: 5px; + padding: 7px 9px; outline: none; -} - -textarea { - min-height: 180px; - resize: vertical; - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + background: #07121a; + color: var(--text); font-size: 12px; - line-height: 1.6; } +input:focus, select:focus, textarea:focus { border-color: var(--accent-2); box-shadow: 0 0 0 2px rgba(40, 214, 189, .08); } +input::placeholder, textarea::placeholder { color: #536b75; } +textarea { min-height: 170px; resize: vertical; font: 12px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; } +label { display: grid; gap: 5px; color: var(--muted); font-size: 11px; font-weight: 700; } +a { color: var(--accent); text-decoration: none; } +[hidden] { display: none !important; } -input:focus, select:focus, textarea:focus { - border-color: var(--line-strong); - box-shadow: 0 0 0 3px rgba(52, 223, 199, 0.12); -} - -label { - display: grid; - gap: 7px; - color: var(--muted); - font-size: 12px; - font-weight: 760; -} - -a { - color: var(--accent-strong); - text-decoration: none; -} - -.app-shell { - display: grid; - grid-template-columns: 230px minmax(0, 1fr); +.auth-shell { min-height: 100vh; + display: grid; + grid-template-columns: minmax(280px, 390px) minmax(340px, 430px); + place-content: center; + gap: 16px; + padding: 28px; + background: #070c12; } +.auth-brand, .auth-card { min-height: 430px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); box-shadow: var(--shadow); } +.auth-brand { display: grid; align-content: center; gap: 16px; padding: 38px; background: #0b151e; } +.auth-brand strong { font-size: 32px; line-height: 1; } +.auth-brand span { color: var(--accent); font-size: 15px; font-weight: 760; } +.auth-brand p { max-width: 290px; margin: 0; color: var(--muted); line-height: 1.65; } +.auth-flow { display: flex; align-items: center; width: 210px; } +.auth-flow i { width: 16px; height: 16px; border: 2px solid #3b5661; border-radius: 50%; background: #0b151e; } +.auth-flow i.done { border-color: var(--accent); background: var(--accent); box-shadow: inset 0 0 0 4px #0b151e; } +.auth-flow b { flex: 1; height: 1px; background: #314952; } +.auth-card { display: grid; align-content: center; gap: 17px; padding: 42px; } +.auth-card h1 { margin: 0; font-size: 24px; } +.auth-card > div p { margin: 7px 0 0; color: var(--muted); } +.auth-card .primary-button { width: 100%; min-height: 38px; margin-top: 3px; } +.form-error { min-height: 18px; margin: -5px 0 0; color: #ff9b94; font-size: 12px; } +.password-hint, .field-note { margin: 0; color: var(--muted-2); font-size: 11px; line-height: 1.5; } +.app-shell { min-height: 100vh; display: grid; grid-template-columns: 152px minmax(0, 1fr); } .sidebar { position: sticky; top: 0; + z-index: 20; height: 100vh; display: grid; - grid-template-rows: auto 1fr; - gap: 22px; - padding: 22px 14px; - border-right: 1px solid var(--line); - background: rgba(4, 11, 18, 0.92); - backdrop-filter: blur(16px); -} - -.brand { - display: grid; - gap: 6px; - padding: 4px 10px 20px; - border-bottom: 1px solid rgba(155, 185, 203, 0.13); -} - -.brand strong { - font-size: 21px; - line-height: 1; - letter-spacing: 0; -} - -.brand span { - color: var(--muted); - font-size: 12px; -} - -.side-nav { - display: grid; - align-content: start; - gap: 8px; -} - -.tab { - width: 100%; - min-height: 44px; - border-color: transparent; - background: transparent; - color: #cbd8df; - text-align: left; - font-weight: 820; -} - -.tab.active { - border-color: rgba(52, 223, 199, 0.35); - background: linear-gradient(90deg, rgba(52, 223, 199, 0.16), rgba(93, 169, 255, 0.06)); - color: var(--accent-strong); -} - -.main-surface { - min-width: 0; - padding: 28px 30px 42px; - overflow: auto; -} - -.page { - display: none; - min-width: 0; -} - -.page.active { - display: block; -} - -.page-head { - display: flex; - align-items: flex-end; - justify-content: space-between; - gap: 22px; - margin-bottom: 20px; -} - -.page-head h1 { - margin: 0; - max-width: 960px; - font-size: clamp(30px, 4vw, 52px); - line-height: 1.02; - letter-spacing: 0; -} - -.page-head p { - margin: 10px 0 0; - max-width: 860px; - color: var(--muted); - font-size: 15px; - line-height: 1.65; -} - -.panel { - min-width: 0; - border: 1px solid var(--line); - border-radius: var(--radius); - background: linear-gradient(145deg, rgba(15, 32, 47, 0.94), rgba(8, 20, 31, 0.92)); - box-shadow: var(--shadow); -} - -.panel + .panel, -.source-grid + .panel, -.two-column + .panel, -.reconstruct-grid + .panel, -.scene-workbench + .panel { - margin-top: 16px; -} - -.panel-head { - display: flex; - align-items: center; - justify-content: space-between; - gap: 16px; - min-height: 64px; - padding: 15px 17px; - border-bottom: 1px solid rgba(155, 185, 203, 0.13); -} - -.panel-head h2 { - margin: 0; - font-size: 17px; - letter-spacing: 0; -} - -.panel-head p { - margin: 5px 0 0; - color: var(--muted); - font-size: 13px; - line-height: 1.5; -} - -.panel > :not(.panel-head) { - margin-left: 17px; - margin-right: 17px; -} - -.panel > :last-child { - margin-bottom: 17px; -} - -.overview-grid { - display: grid; - grid-template-columns: minmax(430px, 1.08fr) minmax(360px, 0.92fr); - gap: 16px; -} - -.wide { - grid-column: 1; -} - -.journey { - display: grid; - grid-template-columns: repeat(6, minmax(0, 1fr)); - gap: 10px; - margin-bottom: 16px; -} - -.journey div { - min-height: 78px; - display: grid; - align-content: center; - gap: 8px; - padding: 13px 14px; - border: 1px solid var(--line); - border-radius: var(--radius); - background: rgba(9, 22, 33, 0.76); -} - -.journey span { - color: var(--muted-2); - font-size: 12px; - font-weight: 820; -} - -.journey strong { - font-size: 15px; -} - -.journey .done { - border-color: rgba(142, 232, 107, 0.38); - background: rgba(42, 74, 46, 0.28); -} - -.journey .done span { - color: var(--lime); -} - -.journey .active { - border-color: rgba(52, 223, 199, 0.62); - box-shadow: 0 0 0 3px rgba(52, 223, 199, 0.09); -} - -.journey .active span { - color: var(--accent-strong); -} - -.metric-strip { - display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); - gap: 10px; - padding-top: 14px; -} - -.metric-strip div, -.quality-grid div, -.chart-kpis div { - min-height: 88px; - display: grid; - align-content: center; - gap: 7px; - padding: 14px; - border: 1px solid rgba(155, 185, 203, 0.15); - border-radius: var(--radius); - background: rgba(5, 16, 26, 0.58); -} - -.metric-strip span, -.quality-grid span, -.chart-kpis span, -.row-meta, -.muted, -small { - color: var(--muted); -} - -.metric-strip strong { - font-size: 30px; -} - -.source-grid, -.two-column, -.reconstruct-grid { - display: grid; - gap: 16px; - margin-bottom: 16px; -} - -.source-grid { - grid-template-columns: repeat(3, minmax(0, 1fr)); -} - -.two-column, -.reconstruct-grid { - grid-template-columns: minmax(420px, 1fr) minmax(360px, 0.82fr); -} - -.source-card { - min-height: 116px; - display: grid; - align-content: center; - gap: 10px; - padding: 18px; - text-align: left; - border-color: var(--line); - background: linear-gradient(140deg, rgba(16, 37, 53, 0.94), rgba(7, 19, 31, 0.88)); -} - -.source-card:hover { - transform: translateY(-1px); -} - -.source-card strong { - font-size: 19px; -} - -.source-card span { - color: var(--muted); - line-height: 1.55; -} - -.toolbar, -.button-row, -.row-actions, -.artifact-buttons, -.action-cluster { - display: flex; - align-items: center; - flex-wrap: wrap; - gap: 8px; -} - -.toolbar { - min-width: min(520px, 100%); -} - -.toolbar select { - max-width: 190px; -} - -.toolbar input { - max-width: 260px; -} - -.bulk-bar { - position: sticky; - top: 0; - z-index: 5; - display: flex; - align-items: center; - flex-wrap: wrap; - gap: 8px; - min-height: 48px; - padding: 10px 0 0; - color: var(--muted); -} - -.bulk-bar.is-active { - color: var(--text); -} - -.bulk-bar label { - display: inline-flex; - grid-auto-flow: column; - align-items: center; - gap: 7px; - min-height: 34px; - padding: 0 8px; -} - -.resource-table, -.list-stack { - display: grid; - gap: 9px; - padding-top: 12px; -} - -.resource-row, -.task-row, -.mini-row { - display: grid; - align-items: center; - gap: 13px; - min-height: 76px; - padding: 11px 12px; - border: 1px solid rgba(155, 185, 203, 0.13); - border-radius: var(--radius); - background: rgba(6, 18, 29, 0.58); -} - -.resource-row { - grid-template-columns: 26px 82px minmax(180px, 1.1fr) minmax(160px, 0.82fr) minmax(150px, 0.78fr) minmax(260px, auto); -} - -.resource-row:not(:has(.row-check)) { - grid-template-columns: 82px minmax(180px, 1fr) minmax(160px, 0.85fr) minmax(150px, 0.85fr) minmax(220px, auto); -} - -.resource-row.active { - border-color: rgba(52, 223, 199, 0.48); - background: linear-gradient(90deg, rgba(52, 223, 199, 0.09), rgba(7, 19, 31, 0.72)); -} - -.task-row { - grid-template-columns: 26px minmax(190px, 1fr) 92px minmax(180px, 0.8fr) minmax(150px, 0.85fr) minmax(290px, auto); -} - -.task-row.compact { - grid-template-columns: minmax(190px, 1fr) 88px minmax(150px, 0.7fr) minmax(210px, auto); -} - -.compact-list .task-row { - min-height: 68px; -} - -.mini-row { - grid-template-columns: minmax(160px, 1fr) minmax(140px, 0.7fr) auto; -} - -.row-check { - display: inline-flex; - align-items: center; - justify-content: center; -} - -.row-check input, -.bulk-bar input, -.check input, -.model-card input { - width: 16px; - min-height: 16px; - accent-color: var(--accent); -} - -.resource-preview { - width: 78px; - height: 54px; - display: grid; - place-items: end start; - overflow: hidden; - padding: 8px; - border: 1px solid rgba(52, 223, 199, 0.22); - border-radius: 7px; - background: - linear-gradient(140deg, rgba(52, 223, 199, 0.14), transparent 46%), - linear-gradient(28deg, rgba(93, 169, 255, 0.18), transparent 48%), - #081724; -} - -.resource-preview span { - padding: 3px 6px; - border-radius: 5px; - background: rgba(3, 13, 20, 0.68); - color: #d8f6f2; - font-size: 11px; - font-weight: 820; -} - -.resource-preview.dataset { - border-color: rgba(93, 169, 255, 0.28); -} - -.resource-preview.model { - border-color: rgba(255, 195, 90, 0.32); -} - -.row-title { - display: grid; - gap: 5px; - min-width: 0; -} - -.row-title strong, -.model-card strong, -.asset-card strong, -.scene-preview-card strong, -.model-summary-card strong { - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.row-title span { - color: var(--muted); - font-size: 12px; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.row-actions { - justify-content: flex-end; -} - -.row-actions button { - min-height: 31px; - padding: 6px 9px; - font-size: 12px; - font-weight: 760; -} - -.status { - justify-self: start; - display: inline-flex; - min-width: 64px; - justify-content: center; - padding: 5px 9px; - border-radius: 999px; - border: 1px solid rgba(155, 185, 203, 0.18); - color: var(--muted); - background: rgba(155, 185, 203, 0.08); - font-size: 12px; - font-weight: 860; -} - -.status.running, -.status.queued, -.status.canceling { - color: var(--amber); - border-color: rgba(255, 195, 90, 0.42); - background: rgba(93, 66, 18, 0.22); -} - -.status.succeeded { - color: var(--lime); - border-color: rgba(142, 232, 107, 0.42); - background: rgba(42, 74, 46, 0.22); -} - -.status.failed, -.status.failed_recovered { - color: var(--coral); - border-color: rgba(255, 119, 108, 0.44); - background: rgba(92, 32, 35, 0.24); -} - -.progress { - width: 100%; - height: 7px; - overflow: hidden; - border-radius: 999px; - background: rgba(155, 185, 203, 0.16); -} - -.progress span { - display: block; - height: 100%; - border-radius: inherit; - background: linear-gradient(90deg, var(--accent), var(--blue)); -} - -.progress-cell { - display: grid; - gap: 6px; -} - -.form-grid { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); + grid-template-rows: auto 1fr auto; gap: 12px; - padding-top: 14px; -} - -.span-2 { - grid-column: 1 / -1; -} - -.check { - display: inline-flex; - align-items: center; - gap: 8px; - padding-top: 24px; -} - -.muted-check { - opacity: 0.72; -} - -.option-group { - display: block; - margin-top: 14px; - padding: 12px; - border: 1px solid rgba(155, 185, 203, 0.14); - border-radius: var(--radius); - background: rgba(5, 16, 26, 0.38); -} - -.option-group summary { - cursor: pointer; - color: var(--text); - font-weight: 820; -} - -.button-row { - padding-top: 14px; -} - -.quality-grid { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 10px; - padding-top: 14px; -} - -.quality-grid div.warn { - border-color: rgba(255, 195, 90, 0.38); -} - -.quality-grid strong { - display: block; - margin-top: 8px; - font-size: 21px; -} - -.overview-scene, -.model-summary { - padding-top: 14px; -} - -.scene-preview-card { - display: grid; - grid-template-columns: 180px minmax(0, 1fr); - gap: 16px; - align-items: center; -} - -.scene-preview-card .resource-preview { - width: 180px; - height: 128px; -} - -.scene-preview-card p, -.model-summary-card p { - color: var(--muted); - line-height: 1.55; -} - -.model-summary-card { - display: grid; - gap: 12px; - padding-top: 14px; -} - -.scene-workbench { - display: grid; - grid-template-columns: minmax(290px, 0.76fr) minmax(460px, 1.36fr) minmax(340px, 0.9fr); - gap: 16px; - margin-bottom: 16px; -} - -.scene-library .resource-row, -.scene-list .resource-row { - grid-template-columns: 26px 70px minmax(150px, 1fr); - align-items: center; -} - -.scene-library .resource-row .row-meta, -.scene-library .resource-row .row-actions, -.scene-list .resource-row .row-meta, -.scene-list .resource-row .row-actions { - grid-column: 2 / -1; -} - -.viewer-panel iframe, -#scenePreview, -#drawerPreview { - width: calc(100% - 34px); - min-height: 520px; - margin-top: 14px; - border: 1px solid var(--line); - border-radius: var(--radius); - background: #050d14; -} - -#drawerPreview { - width: 100%; - min-height: 290px; -} - -.asset-grid { - display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); - gap: 11px; - padding-top: 14px; -} - -.asset-card { - min-height: 104px; - display: grid; - align-content: end; - gap: 6px; - padding: 13px; - border: 1px solid rgba(155, 185, 203, 0.14); - border-radius: var(--radius); - background: linear-gradient(150deg, rgba(93, 169, 255, 0.13), rgba(5, 16, 26, 0.78)); -} - -.asset-card span { - color: var(--muted); - font-size: 12px; -} - -.model-grid { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 12px; - padding-top: 14px; -} - -.model-card { - display: grid; - gap: 9px; - min-height: 184px; - padding: 14px; - border: 1px solid rgba(155, 185, 203, 0.14); - border-radius: var(--radius); - background: rgba(6, 18, 29, 0.58); -} - -.model-card > span, -.model-card p { - margin: 0; - color: var(--muted); - font-size: 13px; -} - -.sparkline { - height: 42px; - display: flex; - align-items: end; - border-bottom: 1px solid rgba(155, 185, 203, 0.18); -} - -.sparkline i { - height: 6px; - border-radius: 999px; - background: linear-gradient(90deg, var(--accent), var(--amber)); -} - -.chart-panel { - padding-top: 14px; -} - -.chart-kpis { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 10px; -} - -.chart-kpis strong { - display: block; - margin-top: 6px; - font-size: 26px; -} - -.line-chart { - height: 155px; - display: flex; - align-items: end; - gap: 11px; - margin-top: 14px; - padding: 14px; - border: 1px solid rgba(155, 185, 203, 0.13); - border-radius: var(--radius); - background: rgba(5, 16, 26, 0.48); -} - -.line-chart span { - flex: 1; - min-width: 16px; - border-radius: 999px 999px 4px 4px; - background: linear-gradient(180deg, var(--accent), var(--blue)); -} - -.rollout-preview { - padding-top: 14px; -} - -.rollout-stage { - min-height: 270px; - display: grid; - align-content: end; - gap: 12px; - padding: 18px; - border: 1px solid rgba(52, 223, 199, 0.24); - border-radius: var(--radius); - background: - linear-gradient(145deg, rgba(52, 223, 199, 0.13), transparent 42%), - linear-gradient(25deg, rgba(255, 195, 90, 0.12), transparent 54%), - #071522; -} - -.rollout-stage span { - color: var(--muted); -} - -.empty-state { - min-height: 120px; - display: grid; - place-items: center; - padding: 20px; - border: 1px dashed rgba(155, 185, 203, 0.22); - border-radius: var(--radius); - color: var(--muted); - text-align: center; -} - -.task-drawer { - position: fixed; - top: 0; - right: 0; - z-index: 40; - width: min(520px, calc(100vw - 18px)); - height: 100vh; - display: grid; - grid-template-rows: auto 1fr; - border-left: 1px solid var(--line); - background: rgba(5, 15, 24, 0.96); - box-shadow: -24px 0 80px rgba(0, 0, 0, 0.42); - transform: translateX(102%); - transition: transform 180ms ease; -} - -.task-drawer.open { - transform: translateX(0); -} - -.drawer-head { - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - padding: 18px; - border-bottom: 1px solid var(--line); -} - -.drawer-head span { - display: block; - color: var(--accent); - font-size: 12px; - font-weight: 840; - margin-bottom: 5px; -} - -.drawer-head h2 { - margin: 0; - font-size: 20px; -} - -#drawerBody { - overflow: auto; - padding: 18px; -} - -.drawer-progress { - display: grid; - grid-template-columns: 1fr auto; - align-items: center; - gap: 10px; - margin-bottom: 14px; -} - -.drawer-progress > span { - grid-column: 1 / -1; - color: var(--muted); -} - -.drawer-section { - margin-top: 14px; - padding: 14px; - border: 1px solid rgba(155, 185, 203, 0.13); - border-radius: var(--radius); - background: rgba(8, 20, 31, 0.6); -} - -.drawer-section h3 { - margin: 0 0 10px; - font-size: 15px; -} - -.task-facts { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 10px; -} - -.task-facts div { - display: grid; - gap: 4px; -} - -.task-facts span { - color: var(--muted); - font-size: 12px; -} - -.report { - margin-top: 8px; - padding: 12px; - border: 1px solid rgba(255, 195, 90, 0.28); - border-radius: var(--radius); - background: rgba(67, 48, 15, 0.18); -} - -.report.ok { - border-color: rgba(142, 232, 107, 0.28); - background: rgba(42, 74, 46, 0.18); -} - -.report.bad { - border-color: rgba(255, 119, 108, 0.32); - background: rgba(92, 32, 35, 0.2); -} - -.report p { - margin: 8px 0 0; - color: var(--muted); - line-height: 1.55; -} - -.artifact-buttons { - align-items: stretch; -} - -.artifact-buttons a { - min-height: 34px; - display: inline-flex; - align-items: center; - padding: 7px 6px; - color: var(--muted); - font-size: 12px; -} - -.technical-details { - margin-top: 14px; - color: var(--muted); -} - -.technical-details summary { - cursor: pointer; - font-weight: 760; -} - -pre { - max-width: 100%; - overflow: auto; - margin: 10px 0 0; - padding: 12px; - border: 1px solid rgba(155, 185, 203, 0.12); - border-radius: var(--radius); - background: #050d14; - color: #d7e7ee; - font: 12px/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; -} - -dialog { - width: min(760px, calc(100vw - 32px)); - padding: 0; - border: 1px solid var(--line); - border-radius: var(--radius); - background: transparent; - color: var(--text); -} - -dialog::backdrop { - background: rgba(0, 0, 0, 0.58); - backdrop-filter: blur(4px); -} - -.dialog-card { - margin: 0; - border-radius: var(--radius); - background: linear-gradient(145deg, rgba(15, 32, 47, 0.98), rgba(8, 20, 31, 0.98)); - box-shadow: var(--shadow); -} - -menu { - display: flex; - justify-content: flex-end; - gap: 10px; - margin: 0; - padding: 16px 17px 17px; -} - -.toast { - position: fixed; - right: 22px; - bottom: 22px; - z-index: 80; - max-width: min(420px, calc(100vw - 44px)); - padding: 12px 14px; - border: 1px solid rgba(52, 223, 199, 0.36); - border-radius: var(--radius); - background: rgba(6, 18, 29, 0.96); - color: var(--text); - box-shadow: var(--shadow); - transform: translateY(18px); - opacity: 0; - pointer-events: none; - transition: opacity 160ms ease, transform 160ms ease; -} - -.toast.show { - transform: translateY(0); - opacity: 1; -} - -body.is-busy .primary-button { - box-shadow: 0 0 0 3px rgba(52, 223, 199, 0.11), 0 14px 32px rgba(52, 223, 199, 0.2); -} - -@media (max-width: 1220px) { - .app-shell { - grid-template-columns: 190px minmax(0, 1fr); - } - - .main-surface { - padding: 24px 22px 36px; - } - - .overview-grid, - .two-column, - .reconstruct-grid, - .scene-workbench { - grid-template-columns: 1fr; - } - - .wide { - grid-column: auto; - } - - .asset-grid, - .model-grid { - grid-template-columns: repeat(2, minmax(0, 1fr)); - } -} - -@media (max-width: 860px) { - .app-shell { - grid-template-columns: 1fr; - } - - .sidebar { - position: relative; - height: auto; - padding: 16px; - } - - .side-nav { - grid-template-columns: repeat(2, minmax(0, 1fr)); - } - - .main-surface { - padding: 18px 14px 32px; - } - - .page-head { - display: grid; - align-items: start; - } - - .page-head h1 { - font-size: 32px; - } - - .journey, - .source-grid, - .metric-strip, - .quality-grid, - .chart-kpis, - .form-grid, - .asset-grid, - .model-grid { - grid-template-columns: 1fr; - } - - .span-2 { - grid-column: auto; - } - - .panel-head { - display: grid; - align-items: start; - } - - .toolbar, - .toolbar select, - .toolbar input { - width: 100%; - max-width: none; - } - - .resource-row, - .resource-row:not(:has(.row-check)), - .task-row, - .task-row.compact, - .mini-row, - .scene-library .resource-row, - .scene-list .resource-row { - grid-template-columns: 1fr; - align-items: start; - } - - .row-check { - justify-content: start; - } - - .resource-preview, - .scene-preview-card .resource-preview { - width: 100%; - height: 120px; - } - - .scene-preview-card { - grid-template-columns: 1fr; - } - - .row-actions { - justify-content: start; - } - - .viewer-panel iframe, - #scenePreview { - width: calc(100% - 34px); - min-height: 360px; - } - - .task-facts { - grid-template-columns: 1fr; - } + padding: 15px 10px 12px; + border-right: 1px solid var(--line-soft); + background: var(--sidebar); +} +.brand { padding: 4px 8px 15px; border-bottom: 1px solid var(--line-soft); } +.brand strong { display: block; font-size: 16px; line-height: 1.1; } +.brand span { display: block; margin-top: 6px; color: var(--muted-2); font-size: 10px; line-height: 1.35; } +.side-nav { display: grid; align-content: start; gap: 4px; } +.tab { width: 100%; min-height: 37px; border-color: transparent; padding: 8px 10px; background: transparent; color: #b4c4ca; text-align: left; font-size: 13px; } +.tab:hover { border-color: #26454e; background: #0e2029; } +.tab.active { border-color: #206b62; background: #0d302e; color: var(--accent); } +.sidebar-actions { display: grid; gap: 5px; padding-top: 10px; border-top: 1px solid var(--line-soft); } +.sidebar-actions button { width: 100%; background: transparent; color: var(--muted); } +.sidebar-actions span { padding: 6px 4px 0; overflow: hidden; color: var(--muted-2); font-size: 10px; text-overflow: ellipsis; white-space: nowrap; } + +.main-surface { min-width: 0; padding: 15px 18px 28px; } +.page { display: none; max-width: 1660px; margin: 0 auto; } +.page.active { display: block; animation: reveal 150ms ease; } +@keyframes reveal { from { opacity: .72; transform: translateY(3px); } to { opacity: 1; transform: none; } } +.page-head { min-height: 52px; display: flex; align-items: center; justify-content: space-between; gap: 18px; margin-bottom: 11px; } +.page-head h1 { margin: 0; font-size: 24px; line-height: 1.1; } +.page-head p { max-width: 760px; margin: 5px 0 0; color: var(--muted); font-size: 12px; line-height: 1.45; } +.panel { min-width: 0; overflow: hidden; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); box-shadow: 0 10px 28px rgba(0, 0, 0, .18); } +.panel + .panel, .source-grid + .panel, .two-column + .panel, .reconstruct-grid + .panel, .scene-workbench + .panel { margin-top: 11px; } +.panel-head { min-height: 52px; display: flex; align-items: center; justify-content: space-between; gap: 14px; padding: 10px 13px; border-bottom: 1px solid var(--line-soft); } +.panel-head h2 { margin: 0; font-size: 14px; } +.panel-head p { margin: 4px 0 0; color: var(--muted); font-size: 11px; } +.panel > :not(.panel-head) { margin-left: 13px; margin-right: 13px; } +.panel > :last-child { margin-bottom: 13px; } + +.overview-grid { display: grid; grid-template-columns: minmax(500px, 1.35fr) minmax(330px, .65fr); gap: 11px; align-items: start; } +.overview-grid .wide { grid-column: 1; } +.journey { display: grid; grid-template-columns: repeat(6, minmax(0, 1fr)); gap: 6px; padding-top: 12px; } +.journey div { min-height: 62px; display: grid; align-content: center; gap: 6px; padding: 9px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } +.journey div.active { border-color: var(--accent-2); background: #0d2929; } +.journey div.done { border-color: #255d4b; } +.journey span { color: var(--muted-2); font-size: 10px; } +.journey div.active span, .journey div.done span { color: var(--accent); } +.journey strong { overflow: hidden; font-size: 12px; text-overflow: ellipsis; white-space: nowrap; } +.metric-strip, .quality-grid, .chart-kpis { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 7px; padding-top: 12px; } +.metric-strip > div, .chart-kpis > div { min-height: 64px; padding: 10px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } +.metric-strip > div > span, .chart-kpis > div > span { display: block; color: var(--muted); font-size: 10px; } +.metric-strip > div > strong, .chart-kpis > div > strong { display: block; margin-top: 6px; font-size: 22px; } + +.source-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 9px; margin-bottom: 11px; } +.source-card { min-height: 76px; display: grid; align-content: center; gap: 5px; padding: 13px; text-align: left; background: var(--surface); } +.source-card:first-child { border-color: var(--accent-2); background: #0d2929; } +.source-card strong { font-size: 13px; } +.source-card span { color: var(--muted); font-size: 11px; font-weight: 500; } +.toolbar, .row-actions, .button-row, .artifact-buttons { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; } +.toolbar input { width: 210px; } +.toolbar select { width: 140px; } +.bulk-bar { min-height: 43px; display: flex; flex-wrap: wrap; align-items: center; gap: 7px; padding: 7px 0; border-bottom: 1px solid var(--line-soft); } +.bulk-bar label { display: flex; align-items: center; gap: 6px; margin-right: 5px; } +.bulk-bar span { color: var(--muted); font-size: 11px; } +input[type="checkbox"] { width: 15px; min-height: 15px; accent-color: var(--accent); } + +.table-shell { max-width: 100%; overflow-x: auto; } +.table-header { min-width: 860px; display: grid; grid-template-columns: 24px 58px minmax(190px, 1fr) 140px 140px 260px; gap: 9px; padding: 8px 8px; color: var(--muted-2); font-size: 10px; } +.resource-table, .list-stack { display: grid; padding-top: 2px; } +.resource-row, .task-row, .mini-row, .user-row { min-width: 0; border-bottom: 1px solid var(--line-soft); background: transparent; } +.resource-row:hover, .task-row:hover, .mini-row:hover, .user-row:hover { background: #10222d; } +.resource-row { min-width: 860px; min-height: 58px; display: grid; grid-template-columns: 24px 58px minmax(190px, 1fr) 140px 140px 260px; align-items: center; gap: 9px; padding: 7px 8px; } +.resource-row:not(:has(.row-check)) { min-width: 0; grid-template-columns: 58px minmax(170px, 1fr) minmax(110px, .6fr) minmax(170px, auto); } +.task-row { min-height: 58px; display: grid; grid-template-columns: 24px minmax(180px, 1fr) 78px minmax(160px, .7fr) minmax(150px, .75fr) minmax(230px, auto); align-items: center; gap: 10px; padding: 8px; } +.task-row.compact { grid-template-columns: minmax(170px, 1fr) 74px minmax(140px, .7fr) minmax(150px, auto); } +.mini-row { min-height: 43px; display: grid; grid-template-columns: minmax(160px, 1fr) minmax(170px, 1.5fr); align-items: center; gap: 10px; padding: 7px 8px; } +.resource-preview { position: relative; width: 52px; height: 40px; overflow: hidden; display: flex; align-items: end; border: 1px solid var(--line); border-radius: 4px; background: #0b2029; } +.resource-preview img { width: 100%; height: 100%; object-fit: cover; } +.resource-preview span { margin: 5px; color: var(--muted); font-size: 9px; } +.row-title { min-width: 0; display: grid; gap: 3px; } +.row-title strong, .model-card strong, .asset-card strong { overflow: hidden; font-size: 12px; text-overflow: ellipsis; white-space: nowrap; } +.row-title span, .row-meta, .mini-row span, small { overflow: hidden; color: var(--muted); font-size: 10px; text-overflow: ellipsis; white-space: nowrap; } +.row-actions { justify-content: flex-end; } +.row-actions button { min-height: 26px; padding: 4px 7px; font-size: 10px; } +.row-check { display: flex; justify-content: center; } +.status { justify-self: start; min-width: 56px; padding: 4px 7px; border: 1px solid #344854; border-radius: 999px; background: #101b23; color: var(--muted); font-size: 10px; font-weight: 800; text-align: center; } +.status.running, .status.queued, .status.canceling { border-color: #725b29; color: var(--amber); background: #241d10; } +.status.succeeded { border-color: #285d48; color: var(--green); background: #10251d; } +.status.failed, .status.failed_recovered { border-color: #713936; color: #ff9b94; background: #281416; } +.progress-cell { display: grid; gap: 6px; } +.progress { width: 100%; height: 6px; overflow: hidden; border-radius: 999px; background: #1b2b34; } +.progress span { display: block; height: 100%; border-radius: inherit; background: linear-gradient(90deg, var(--accent-2), var(--accent), var(--blue)); } + +.form-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px; padding-top: 12px; } +.span-2 { grid-column: 1 / -1; } +.check { display: flex; align-items: center; gap: 7px; padding-top: 20px; } +.option-group { display: block; margin-top: 12px; padding: 10px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } +.option-group summary { cursor: pointer; color: #c9d7da; font-size: 12px; font-weight: 750; } +.button-row { padding-top: 12px; } +.two-column, .reconstruct-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 11px; margin-bottom: 11px; } +.reconstruct-grid { grid-template-columns: minmax(520px, 1.2fr) minmax(300px, .8fr); } +.selection-card-rail { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 7px; padding-top: 11px; } +.selection-card { min-width: 0; min-height: 102px; display: grid; grid-template-rows: 50px auto auto; gap: 5px; padding: 6px; text-align: left; } +.selection-card.active { border-color: var(--accent); background: #0d2929; } +.selection-thumb { overflow: hidden; display: grid; place-items: center; border-radius: 4px; background: #08131b; color: var(--muted); font-size: 10px; } +.selection-thumb img { width: 100%; height: 100%; object-fit: cover; } +.selection-card strong { overflow: hidden; font-size: 11px; text-overflow: ellipsis; white-space: nowrap; } +.selection-card em { overflow: hidden; color: var(--muted); font-size: 9px; font-style: normal; text-overflow: ellipsis; white-space: nowrap; } +.quality-grid { grid-template-columns: 126px minmax(0, 1fr); align-items: stretch; } +.quality-score { min-height: 126px; display: grid; place-content: center; gap: 4px; border: 1px solid var(--line); border-radius: 50%; background: var(--surface-3); text-align: center; } +.quality-score b { font-size: 32px; line-height: 1; } +.quality-score span { color: var(--muted); font-size: 11px; } +.quality-score.ok { border-color: var(--accent-2); box-shadow: inset 0 0 0 5px rgba(40, 214, 189, .05); } +.quality-score.ok b { color: var(--accent); } +.quality-score.warn { border-color: #6c572d; box-shadow: inset 0 0 0 5px rgba(227, 177, 73, .05); } +.quality-score.warn b { color: var(--amber); } +.quality-checks { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 7px; } +.quality-checks > div { min-height: 59px; display: grid; align-content: center; gap: 5px; padding: 9px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } +.quality-checks > div.warn { border-color: #6c572d; } +.quality-checks span { color: var(--muted); font-size: 10px; } +.quality-checks strong { overflow-wrap: anywhere; font-size: 12px; } + +.scene-workbench { display: grid; grid-template-columns: minmax(250px, .75fr) minmax(480px, 1.5fr) minmax(285px, .85fr); gap: 10px; margin-bottom: 11px; } +.scene-library .resource-row, .scene-list .resource-row { min-width: 0; grid-template-columns: 22px 56px minmax(140px, 1fr); } +.scene-library .resource-row .row-meta, .scene-library .resource-row .row-actions, .scene-list .resource-row .row-meta, .scene-list .resource-row .row-actions { grid-column: 2 / -1; justify-content: flex-start; } +.viewer-panel iframe, #scenePreview, #drawerPreview { width: calc(100% - 26px); min-height: 510px; margin-top: 12px; border: 1px solid var(--line); border-radius: 5px; background: #03070a; } +#drawerPreview { width: 100%; min-height: 290px; } +.viewer-panel.is-maximized { position: fixed; inset: 10px; z-index: 80; display: grid; grid-template-rows: auto minmax(0, 1fr); overflow: hidden; margin: 0; background: #050b10; box-shadow: 0 24px 80px rgba(0, 0, 0, .7); } +.viewer-panel.is-maximized iframe { width: calc(100% - 26px); height: calc(100vh - 92px); min-height: 0; } +body.viewer-maximized { overflow: hidden; } +.asset-grid, .model-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 8px; padding-top: 11px; } +.asset-card, .model-card { min-width: 0; min-height: 128px; display: grid; align-content: end; gap: 6px; padding: 10px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } +.asset-card span, .model-card > span, .model-card p { margin: 0; color: var(--muted); font-size: 10px; } +.model-card { min-height: 160px; align-content: start; } +.sparkline { height: 34px; display: flex; align-items: end; border-bottom: 1px solid var(--line-soft); } +.sparkline i { height: 5px; border-radius: 4px; background: var(--accent); } +.chart-panel, .rollout-preview, .overview-scene, .model-summary { padding-top: 11px; } +.line-chart { height: 130px; display: flex; align-items: end; gap: 8px; margin-top: 10px; padding: 11px; border: 1px solid var(--line-soft); background: var(--surface-3); } +.line-chart span { flex: 1; min-width: 12px; background: var(--accent); } +.scene-preview-card { display: grid; grid-template-columns: 180px minmax(0, 1fr); gap: 13px; align-items: center; } +.scene-preview-card .resource-preview { width: 180px; height: 118px; } +.scene-preview-card p, .model-summary-card p { margin: 7px 0; color: var(--muted); line-height: 1.5; } +.model-summary-card { display: grid; gap: 9px; } +.rollout-stage { min-height: 260px; display: grid; align-content: end; gap: 9px; padding: 14px; border: 1px solid #285b57; border-radius: 5px; background: #081922; } +.rollout-stage span { color: var(--muted); } +.empty-state { min-height: 105px; display: grid; place-items: center; padding: 16px; border: 1px dashed #2b414c; border-radius: 5px; color: var(--muted); text-align: center; } + +.task-drawer { position: fixed; top: 0; right: 0; z-index: 50; width: min(470px, calc(100vw - 16px)); height: 100vh; display: grid; grid-template-rows: auto 1fr; border-left: 1px solid var(--line); background: #08121a; box-shadow: -20px 0 60px rgba(0, 0, 0, .44); transform: translateX(102%); transition: transform 180ms ease; } +.task-drawer.open { transform: translateX(0); } +.drawer-head { min-height: 74px; display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 14px; border-bottom: 1px solid var(--line); } +.drawer-head span { display: block; margin-bottom: 4px; color: var(--accent); font-size: 10px; } +.drawer-head h2 { margin: 0; font-size: 17px; } +#drawerBody { overflow: auto; padding: 14px; } +.drawer-progress { display: grid; grid-template-columns: 1fr auto; gap: 8px; align-items: center; } +.drawer-progress > span { grid-column: 1 / -1; color: var(--muted); } +.drawer-section { margin-top: 10px; padding: 11px; border: 1px solid var(--line-soft); border-radius: 5px; background: var(--surface-3); } +.drawer-section h3 { margin: 0 0 8px; font-size: 13px; } +.task-facts { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; } +.task-facts div { display: grid; gap: 3px; } +.task-facts span { color: var(--muted); font-size: 10px; } +.report { margin-top: 7px; padding: 10px; border: 1px solid #675328; border-radius: 5px; background: #211b0e; } +.report.ok { border-color: #285b45; background: #10231b; } +.report.bad { border-color: #6f3635; background: #271416; } +.report p { margin: 6px 0 0; color: var(--muted); line-height: 1.5; } +.technical-details { margin-top: 10px; color: var(--muted); } +.technical-details summary { cursor: pointer; font-weight: 700; } +pre { max-width: 100%; overflow: auto; margin: 8px 0 0; padding: 10px; border: 1px solid var(--line-soft); border-radius: 4px; background: #04090d; color: #cfe0e4; font: 11px/1.55 ui-monospace, SFMono-Regular, Menlo, monospace; } + +dialog { width: min(700px, calc(100vw - 28px)); max-height: calc(100vh - 32px); overflow: auto; padding: 0; border: 1px solid var(--line); border-radius: var(--radius); background: transparent; color: var(--text); } +dialog::backdrop { background: rgba(0, 0, 0, .66); backdrop-filter: blur(3px); } +.dialog-card { margin: 0; padding-bottom: 1px; background: var(--surface); box-shadow: var(--shadow); } +menu { display: flex; justify-content: flex-end; gap: 8px; margin: 0; padding: 13px; } +.user-form { margin: 0 13px 13px; } +.user-list { display: grid; margin: 0 13px 13px; border-top: 1px solid var(--line-soft); } +.user-row { min-height: 54px; display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 8px; } +.user-row > div:first-child { min-width: 0; display: grid; gap: 3px; } +.user-row span { color: var(--muted); font-size: 10px; } +.upload-progress { position: relative; height: 34px; overflow: hidden; display: grid; place-items: center; border: 1px solid var(--line); border-radius: 5px; background: #07121a; } +.upload-progress span { position: absolute; inset: 0 auto 0 0; width: 0; background: #0e4b44; transition: width 160ms ease; } +.upload-progress strong { position: relative; font-size: 11px; } +.toast { position: fixed; right: 18px; bottom: 18px; z-index: 90; max-width: min(420px, calc(100vw - 36px)); padding: 10px 12px; border: 1px solid #286b62; border-radius: 5px; background: #0b1a22; color: var(--text); box-shadow: var(--shadow); opacity: 0; transform: translateY(14px); pointer-events: none; transition: opacity 150ms ease, transform 150ms ease; } +.toast.show { opacity: 1; transform: none; } +.viewer-mode [data-open-import], .viewer-mode .bulk-bar button, .viewer-mode [data-delete-dataset], .viewer-mode [data-delete-scene], .viewer-mode [data-delete-model] { display: none; } + +@media (max-width: 1180px) { + .overview-grid, .two-column, .reconstruct-grid, .scene-workbench { grid-template-columns: 1fr; } + .overview-grid .wide { grid-column: auto; } + .scene-workbench .viewer-panel { order: -1; } + .asset-grid, .model-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .selection-card-rail { grid-template-columns: repeat(3, minmax(0, 1fr)); } +} + +@media (max-width: 720px) { + body { font-size: 12px; } + .auth-shell { min-height: 100svh; grid-template-columns: 1fr; align-content: center; padding: 14px; } + .auth-brand { display: none; } + .auth-card { min-height: 0; padding: 28px 20px; } + .app-shell { display: block; } + .sidebar { position: fixed; top: auto; right: 0; bottom: 0; left: 0; z-index: 45; height: 62px; display: block; padding: 5px 7px; border-top: 1px solid var(--line); border-right: 0; } + .brand, .sidebar-actions { display: none; } + .side-nav { display: flex; height: 100%; gap: 3px; overflow-x: auto; scrollbar-width: none; } + .side-nav::-webkit-scrollbar { display: none; } + .tab { flex: 0 0 69px; min-height: 48px; padding: 6px 4px; text-align: center; font-size: 10px; } + .main-surface { padding: 11px 10px 78px; } + .page-head { min-height: 46px; align-items: start; margin-bottom: 8px; } + .page-head h1 { font-size: 20px; } + .page-head p { max-width: 250px; font-size: 10px; } + .page-head > button { flex: 0 0 auto; } + .panel-head { align-items: start; } + .journey { grid-template-columns: repeat(3, minmax(0, 1fr)); } + .metric-strip, .chart-kpis, .form-grid, .source-grid, .asset-grid, .model-grid { grid-template-columns: 1fr; } + .quality-grid { grid-template-columns: 94px minmax(0, 1fr); } + .quality-score { min-height: 94px; } + .quality-checks { grid-template-columns: 1fr; } + .viewer-panel.is-maximized { inset: 0; border-radius: 0; } + .viewer-panel.is-maximized iframe { height: calc(100vh - 78px); } + .span-2 { grid-column: auto; } + .selection-card-rail { display: flex; overflow-x: auto; } + .selection-card { flex: 0 0 145px; } + .task-row, .task-row.compact, .mini-row, .resource-row:not(:has(.row-check)), .scene-library .resource-row, .scene-list .resource-row { min-width: 0; grid-template-columns: 1fr; align-items: start; } + .task-row .row-check { justify-content: start; } + .row-actions { justify-content: flex-start; } + .resource-preview, .scene-preview-card .resource-preview { width: 100%; height: 112px; } + .scene-preview-card { grid-template-columns: 1fr; } + .viewer-panel iframe, #scenePreview { width: calc(100% - 26px); min-height: 390px; } + .task-facts { grid-template-columns: 1fr; } + .task-drawer { width: 100vw; } + .toolbar { width: 100%; } + .toolbar input, .toolbar select { width: 100%; } + .user-row { align-items: start; flex-direction: column; } } @media (prefers-reduced-motion: reduce) { - *, *::before, *::after { - transition-duration: 1ms !important; - scroll-behavior: auto !important; - } -} - -/* Prototype fidelity pass: compact product workbench matching the approved seven-page concept. */ -:root { - --bg: #050d16; - --surface: rgba(11, 25, 39, 0.94); - --surface-2: rgba(13, 31, 47, 0.96); - --line: rgba(128, 161, 181, 0.18); - --line-strong: rgba(46, 224, 198, 0.55); - --text: #eef7fb; - --muted: #91a8b8; - --accent: #2fe0c6; - --blue: #57a8ff; - --radius: 6px; - --shadow: 0 16px 42px rgba(0, 0, 0, 0.28); -} - -html, body { - background: - radial-gradient(circle at 72% 12%, rgba(47, 224, 198, 0.11), transparent 26%), - linear-gradient(180deg, #06101a 0%, #040b13 100%); -} - -.app-shell { - grid-template-columns: 132px minmax(0, 1fr); -} - -.sidebar { - padding: 12px 8px; - gap: 12px; - background: rgba(3, 10, 17, 0.96); -} - -.brand { - padding: 8px 8px 14px; -} - -.brand strong { - font-size: 15px; -} - -.brand span { - font-size: 10px; -} - -.side-nav { - gap: 5px; -} - -.tab { - min-height: 36px; - padding: 8px 10px; - border-radius: 5px; - font-size: 13px; - font-weight: 760; -} - -.main-surface { - padding: 12px 16px 22px; -} - -.page-head { - min-height: 42px; - align-items: center; - margin-bottom: 10px; -} - -.page-head h1 { - font-size: clamp(20px, 1.75vw, 28px); - line-height: 1.05; - font-weight: 900; -} - -.page-head p { - margin-top: 4px; - font-size: 12px; - line-height: 1.45; -} - -button { - min-height: 28px; - padding: 5px 9px; - border-radius: 5px; - font-size: 12px; - font-weight: 750; -} - -.primary-button { - box-shadow: 0 10px 24px rgba(47, 224, 198, 0.16); -} - -input, select, textarea { - min-height: 31px; - padding: 6px 8px; - border-radius: 5px; - font-size: 12px; -} - -label { - gap: 5px; - font-size: 11px; -} - -.panel { - border-radius: 6px; - background: - linear-gradient(145deg, rgba(15, 34, 50, 0.92), rgba(7, 18, 29, 0.93)); - box-shadow: 0 12px 34px rgba(0, 0, 0, 0.24); -} - -.panel + .panel, -.source-grid + .panel, -.two-column + .panel, -.reconstruct-grid + .panel, -.scene-workbench + .panel { - margin-top: 10px; -} - -.panel > :not(.panel-head) { - margin-left: 12px; - margin-right: 12px; -} - -.panel > :last-child { - margin-bottom: 12px; -} - -.panel-head { - min-height: 46px; - padding: 10px 12px; -} - -.panel-head h2 { - font-size: 14px; - font-weight: 860; -} - -.panel-head p { - margin-top: 3px; - font-size: 11px; -} - -.overview-grid { - grid-template-columns: minmax(520px, 1.22fr) minmax(330px, 0.78fr); - gap: 10px; - align-items: start; -} - -.overview-grid .panel { - align-self: start; -} - -.overview-grid .next-step { - height: max-content; - min-height: 0; -} - -.overview-grid .wide { - margin-top: -220px; -} - -.journey { - grid-template-columns: repeat(6, minmax(0, 1fr)); - gap: 8px; - margin-bottom: 10px; -} - -.journey div { - min-height: 54px; - padding: 9px 10px; - gap: 5px; - background: rgba(8, 20, 31, 0.7); -} - -.journey span { - font-size: 10px; -} - -.journey strong { - font-size: 12px; -} - -.metric-strip { - grid-template-columns: repeat(4, minmax(0, 1fr)); - gap: 8px; -} - -.metric-strip div, -.quality-grid div, -.chart-kpis div { - min-height: 58px; - padding: 9px; - gap: 4px; -} - -.metric-strip strong { - font-size: 23px; -} - -.source-grid, -.two-column, -.reconstruct-grid { - gap: 10px; - margin-bottom: 10px; -} - -.source-card { - min-height: 76px; - padding: 12px; -} - -.source-card strong { - font-size: 14px; -} - -.source-card span { - font-size: 11px; -} - -.bulk-bar { - min-height: 38px; - padding-top: 7px; - gap: 6px; -} - -.resource-table, -.list-stack { - gap: 0; - padding-top: 8px; -} - -.resource-row, -.task-row, -.mini-row { - min-height: 52px; - padding: 7px 8px; - border-radius: 0; - border-width: 0 0 1px; - background: rgba(7, 19, 30, 0.36); -} - -.resource-row:hover, -.task-row:hover, -.mini-row:hover { - background: rgba(16, 38, 55, 0.62); -} - -.resource-row { - grid-template-columns: 22px 58px minmax(170px, 1.1fr) minmax(120px, 0.7fr) minmax(120px, 0.7fr) minmax(230px, auto); -} - -.resource-row:not(:has(.row-check)) { - grid-template-columns: 58px minmax(170px, 1.1fr) minmax(120px, 0.7fr) minmax(120px, 0.7fr) minmax(190px, auto); -} - -.task-row { - grid-template-columns: 22px minmax(190px, 1fr) 82px minmax(170px, 0.74fr) minmax(140px, 0.72fr) minmax(260px, auto); -} - -.task-row.compact { - grid-template-columns: minmax(180px, 1fr) 78px minmax(150px, 0.8fr) minmax(180px, auto); -} - -.resource-preview { - width: 52px; - height: 38px; - padding: 0; - align-items: end; - border-radius: 5px; -} - -.resource-preview img { - width: 100%; - height: 100%; - object-fit: cover; - display: block; -} - -.resource-preview span { - margin: 5px; - font-size: 10px; -} - -.row-title { - gap: 3px; -} - -.row-title strong, -.model-card strong, -.asset-card strong { - font-size: 12px; -} - -.row-title span, -.row-meta, -small { - font-size: 11px; -} - -.row-actions { - gap: 5px; -} - -.row-actions button { - min-height: 26px; - padding: 4px 7px; - font-size: 11px; -} - -.status { - min-width: 56px; - padding: 3px 7px; - font-size: 11px; -} - -.progress { - height: 5px; -} - -.form-grid { - gap: 9px; - padding-top: 10px; -} - -.option-group { - margin-top: 10px; - padding: 9px; -} - -.quality-grid { - gap: 8px; -} - -.quality-grid strong { - margin-top: 4px; - font-size: 16px; -} - -.scene-preview-card { - grid-template-columns: 260px minmax(0, 1fr); -} - -.scene-preview-card .resource-preview { - width: 260px; - height: 148px; -} - -.scene-workbench { - grid-template-columns: minmax(240px, 0.72fr) minmax(520px, 1.45fr) minmax(300px, 0.83fr); - gap: 10px; -} - -.scene-library .resource-row, -.scene-list .resource-row { - grid-template-columns: 22px 58px minmax(128px, 1fr); - min-height: 58px; -} - -.viewer-panel iframe, -#scenePreview { - width: calc(100% - 24px); - min-height: 430px; - margin-top: 10px; - border-radius: 6px; - background: #03080e; -} - -.asset-grid { - grid-template-columns: repeat(5, minmax(120px, 1fr)); - gap: 8px; -} - -.asset-card, -.model-card { - border-radius: 6px; - padding: 10px; -} - -.model-grid { - grid-template-columns: repeat(4, minmax(180px, 1fr)); - gap: 8px; -} - -.task-drawer { - width: min(420px, 92vw); - background: rgba(7, 17, 27, 0.98); - border-left: 1px solid var(--line); -} - -.drawer-head { - padding: 12px; -} - -.toast { - border-radius: 6px; -} - -@media (max-width: 920px) { - .app-shell { - grid-template-columns: 1fr; - } - - .sidebar { - position: static; - height: auto; - } - - .side-nav { - grid-template-columns: repeat(4, minmax(0, 1fr)); - } - - .overview-grid, - .two-column, - .reconstruct-grid, - .scene-workbench { - grid-template-columns: 1fr; - } -} - -/* Prototype fidelity v3: match the approved seven-page workbench more closely. */ -body { - background: - radial-gradient(circle at 72% 14%, rgba(38, 206, 189, 0.18), transparent 28%), - radial-gradient(circle at 22% 92%, rgba(70, 136, 217, 0.13), transparent 26%), - linear-gradient(180deg, #030910 0%, #06111c 52%, #02070d 100%); -} - -.app-shell { - grid-template-columns: 150px minmax(0, 1fr); -} - -.sidebar { - border-right-color: rgba(99, 132, 154, 0.22); - background: linear-gradient(180deg, rgba(5, 16, 27, 0.98), rgba(2, 8, 15, 0.98)); -} - -.brand { - min-height: 54px; - padding: 8px 6px 12px; -} - -.brand strong { - font-size: 16px; -} - -.brand span { - color: #b6c5ce; - font-size: 11px; -} - -.tab { - min-height: 40px; - padding: 9px 11px; - color: #d8e4eb; - border-radius: 5px; - font-size: 13px; -} - -.tab.active { - border-color: rgba(42, 217, 195, 0.48); - background: linear-gradient(90deg, rgba(42, 217, 195, 0.21), rgba(33, 83, 113, 0.08)); -} - -.main-surface { - padding: 10px 12px 18px; -} - -.page-head { - min-height: 38px; - margin-bottom: 8px; -} - -.page-head h1 { - font-size: 23px; - line-height: 1.05; -} - -.page-head p { - max-width: 780px; - font-size: 12px; -} - -.panel { - border-color: rgba(97, 130, 153, 0.24); - background: - linear-gradient(145deg, rgba(14, 34, 51, 0.92), rgba(7, 19, 31, 0.94)), - linear-gradient(90deg, rgba(47, 224, 198, 0.05), transparent); -} - -.panel-head { - min-height: 42px; - padding: 8px 10px; -} - -.panel-head h2 { - font-size: 13px; -} - -.panel-head p { - color: #9fb2c0; - font-size: 11px; -} - -.overview-grid { - grid-template-columns: minmax(360px, 0.78fr) minmax(330px, 0.72fr) minmax(330px, 0.7fr); - grid-template-rows: auto auto; - grid-template-areas: - "journey scene tasks" - "next scene model"; - gap: 10px; -} - -.journey-panel { - grid-area: journey; -} - -.next-step { - grid-area: next; -} - -.scene-hero-panel { - grid-area: scene; -} - -.overview-grid > .panel:nth-of-type(3) { - grid-area: tasks; -} - -.overview-grid > .panel:nth-of-type(5) { - grid-area: model; -} - -.overview-grid .wide { - grid-column: auto; - margin-top: 0; -} - -.journey { - grid-template-columns: 1fr; - gap: 0; - margin: 0; - padding: 8px 0 0; -} - -.journey div { - min-height: 42px; - grid-template-columns: 52px 1fr; - align-items: center; - align-content: center; - border-width: 0 0 1px; - border-radius: 0; - background: transparent; -} - -.journey span { - color: var(--accent); -} - -.metric-strip { - grid-template-columns: repeat(2, minmax(0, 1fr)); -} - -.metric-strip div { - min-height: 54px; -} - -.scene-preview-card { - grid-template-columns: 1fr; -} - -.scene-preview-card .resource-preview { - width: 100%; - height: clamp(220px, 28vh, 340px); -} - -.scene-preview-card p { - margin: 6px 0; -} - -.table-shell { - overflow: hidden; - border: 1px solid rgba(97, 130, 153, 0.16); - border-radius: 6px; - background: rgba(4, 13, 22, 0.36); -} - -.table-header { - display: grid; - align-items: center; - gap: 12px; - min-height: 32px; - padding: 0 8px; - border-bottom: 1px solid rgba(97, 130, 153, 0.18); - color: #8fa5b6; - font-size: 11px; - font-weight: 780; -} - -.dataset-header { - grid-template-columns: 22px 58px minmax(170px, 1.1fr) minmax(120px, 0.7fr) minmax(120px, 0.7fr) minmax(230px, auto); -} - -.dataset-shell .resource-table { - padding-top: 0; -} - -.selection-card-rail { - display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); - gap: 8px; - padding-top: 10px; -} - -.selection-card { - min-height: 104px; - display: grid; - grid-template-columns: 72px minmax(0, 1fr); - grid-template-rows: 1fr auto; - gap: 7px 9px; - align-items: center; - padding: 8px; - text-align: left; - border-color: rgba(97, 130, 153, 0.22); - background: rgba(5, 15, 25, 0.74); -} - -.selection-card.active { - border-color: rgba(47, 224, 198, 0.78); - box-shadow: inset 0 0 0 1px rgba(47, 224, 198, 0.28), 0 0 0 2px rgba(47, 224, 198, 0.08); -} - -.selection-thumb { - grid-row: 1 / 3; - width: 72px; - height: 72px; - display: grid; - place-items: center; - overflow: hidden; - border: 1px solid rgba(47, 224, 198, 0.22); - border-radius: 5px; - background: - linear-gradient(135deg, rgba(47, 224, 198, 0.13), transparent), - #071621; -} - -.selection-thumb img { - width: 100%; - height: 100%; - object-fit: cover; -} - -.selection-thumb span { - color: #dffaf6; - font-size: 11px; - font-weight: 850; -} - -.selection-card strong { - align-self: end; - overflow: hidden; - color: #f1fbff; - font-size: 12px; - text-overflow: ellipsis; - white-space: nowrap; -} - -.selection-card em { - align-self: start; - overflow: hidden; - color: #91a8b8; - font-size: 11px; - font-style: normal; - text-overflow: ellipsis; - white-space: nowrap; -} - -.reconstruct-grid { - grid-template-columns: minmax(560px, 1.22fr) minmax(300px, 0.78fr); -} - -.preflight-panel { - min-height: 100%; -} - -.quality-grid { - grid-template-columns: 140px minmax(0, 1fr); - align-items: stretch; -} - -.quality-score { - min-height: 150px; - display: grid; - place-items: center; - align-content: center; - gap: 5px; - border: 1px solid rgba(47, 224, 198, 0.36); - border-radius: 6px; - background: - radial-gradient(circle, rgba(47, 224, 198, 0.22) 0 34%, transparent 36%), - rgba(5, 15, 25, 0.66); -} - -.quality-score.warn { - border-color: rgba(255, 195, 90, 0.48); - background: - radial-gradient(circle, rgba(255, 195, 90, 0.18) 0 34%, transparent 36%), - rgba(5, 15, 25, 0.66); -} - -.quality-score b { - font-size: 38px; - line-height: 1; -} - -.quality-score span { - color: #a9bac6; - font-size: 12px; - font-weight: 820; -} - -.quality-checks { - display: grid; - gap: 8px; -} - -.quality-checks div { - min-height: 0; -} - -.quality-checks .ok { - border-color: rgba(142, 232, 107, 0.24); -} - -.quality-checks .warn { - border-color: rgba(255, 195, 90, 0.38); -} - -.scene-workbench { - grid-template-columns: minmax(245px, 0.74fr) minmax(560px, 1.58fr) minmax(300px, 0.84fr); -} - -.viewer-panel iframe, -#scenePreview { - min-height: 470px; -} - -.asset-grid { - grid-template-columns: repeat(6, minmax(120px, 1fr)); -} - -.task-drawer.open { - width: min(430px, 92vw); -} - -@media (max-width: 1280px) { - .overview-grid { - grid-template-columns: minmax(360px, 1fr) minmax(330px, 0.86fr); - grid-template-rows: auto; - min-height: auto; - grid-template-areas: - "journey tasks" - "next model" - "scene scene"; - } - - .selection-card-rail { - grid-template-columns: repeat(2, minmax(0, 1fr)); - } - - .asset-grid { - grid-template-columns: repeat(4, minmax(120px, 1fr)); - } -} - -@media (max-width: 920px) { - .app-shell { - grid-template-columns: 1fr; - } - - .overview-grid { - grid-template-columns: 1fr; - grid-template-rows: auto; - min-height: auto; - grid-template-areas: none; - } - - .journey-panel, - .next-step, - .scene-hero-panel, - .overview-grid > .panel:nth-of-type(3), - .overview-grid > .panel:nth-of-type(5) { - grid-area: auto; - } - - .dataset-header { - display: none; - } - - .selection-card-rail, - .quality-grid, - .source-grid, - .reconstruct-grid, - .scene-workbench { - grid-template-columns: 1fr; - } + *, *::before, *::after { scroll-behavior: auto !important; transition-duration: 1ms !important; animation-duration: 1ms !important; } } diff --git a/src/real2sim_platform/storage.py b/src/real2sim_platform/storage.py new file mode 100644 index 0000000..5eb6b9c --- /dev/null +++ b/src/real2sim_platform/storage.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import hashlib +import os +import re +from pathlib import Path, PurePosixPath +from typing import Any, Iterable + +from .config import PlatformConfig + + +ALLOWED_AREAS = {"uploads", "datasets", "scenes", "models", "runs", "backups"} +_SAFE_COMPONENT = re.compile(r"[^a-zA-Z0-9._-]+") + + +class StorageUnavailable(RuntimeError): + pass + + +class StorageBoundaryError(ValueError): + pass + + +class BosStorage: + def __init__(self, config: PlatformConfig) -> None: + self.config = config + self.bucket = config.bos_bucket.strip() + self.prefix = normalize_prefix(config.bos_prefix) + self._client = None + + @property + def configured(self) -> bool: + return bool(self.bucket and self.config.bos_access_key_id and self.config.bos_secret_access_key) + + def health(self) -> dict[str, Any]: + result = {"configured": self.configured, "bucket": self.bucket, "prefix": self.prefix, "endpoint": self.config.bos_endpoint} + if not self.configured: + result["ready"] = False + result["reason"] = "BOS credentials or bucket are not configured" + return result + try: + self.list_objects(area="uploads", relative_prefix="", max_keys=1) + result["ready"] = True + except Exception as exc: + result["ready"] = False + result["reason"] = str(exc) + return result + + def object_key(self, area: str, relative_path: str) -> str: + if area not in ALLOWED_AREAS: + raise StorageBoundaryError(f"unsupported BOS area: {area}") + relative = normalize_relative_path(relative_path) + return f"{self.prefix}/{area}/{relative}" if relative else f"{self.prefix}/{area}/" + + def uri_for_key(self, key: str) -> str: + checked = self.assert_owned_key(key) + return f"bos://{self.bucket}/{checked}" + + def assert_owned_key(self, key: str) -> str: + normalized = str(PurePosixPath(str(key).strip().lstrip("/"))) + owned_root = f"{self.prefix}/" + if not normalized.startswith(owned_root) or normalized == self.prefix: + raise StorageBoundaryError("object key is outside the Real2Sim BOS prefix") + area = normalized[len(owned_root):].split("/", 1)[0] + if area not in ALLOWED_AREAS: + raise StorageBoundaryError("object key is outside an approved Real2Sim area") + if ".." in PurePosixPath(normalized).parts: + raise StorageBoundaryError("parent traversal is not allowed") + return normalized + + def upload_file(self, local_path: Path, *, area: str, relative_path: str, metadata: dict[str, str] | None = None) -> dict[str, Any]: + if not local_path.is_file(): + raise FileNotFoundError(local_path) + key = self.object_key(area, relative_path) + client = self._get_client() + kwargs: dict[str, Any] = {} + if metadata: + kwargs["user_metadata"] = {str(k): str(v) for k, v in metadata.items()} + client.put_object_from_file(self.bucket, key, str(local_path), **kwargs) + return { + "key": key, + "storage_uri": self.uri_for_key(key), + "size_bytes": local_path.stat().st_size, + "checksum": sha256_file(local_path), + } + + def download_file(self, storage_uri: str, target_path: Path) -> Path: + key = self.key_from_uri(storage_uri) + target_path.parent.mkdir(parents=True, exist_ok=True) + self._get_client().get_object_to_file(self.bucket, key, str(target_path)) + return target_path + + def list_objects(self, *, area: str, relative_prefix: str = "", max_keys: int = 1000) -> list[dict[str, Any]]: + prefix = self.object_key(area, relative_prefix) + response = self._get_client().list_objects(self.bucket, prefix=prefix, max_keys=max(1, min(1000, int(max_keys)))) + contents: Iterable[Any] = getattr(response, "contents", None) or [] + return [ + { + "key": self.assert_owned_key(str(getattr(item, "key", ""))), + "storage_uri": self.uri_for_key(str(getattr(item, "key", ""))), + "size_bytes": int(getattr(item, "size", 0) or 0), + "etag": str(getattr(item, "e_tag", getattr(item, "etag", "")) or "").strip('"'), + "last_modified": str(getattr(item, "last_modified", "") or ""), + } + for item in contents + if getattr(item, "key", None) + ] + + def delete_object(self, storage_uri: str) -> None: + key = self.key_from_uri(storage_uri) + self._get_client().delete_object(self.bucket, key) + + def key_from_uri(self, storage_uri: str) -> str: + prefix = f"bos://{self.bucket}/" + if not storage_uri.startswith(prefix): + raise StorageBoundaryError("BOS URI bucket does not match configured bucket") + return self.assert_owned_key(storage_uri[len(prefix):]) + + def _get_client(self): + if not self.configured: + raise StorageUnavailable("BOS storage is not configured") + if self._client is not None: + return self._client + try: + from baidubce.auth.bce_credentials import BceCredentials + from baidubce.bce_client_configuration import BceClientConfiguration + from baidubce.services.bos.bos_client import BosClient + except ImportError as exc: + raise StorageUnavailable("bce-python-sdk is not installed") from exc + credentials = BceCredentials(self.config.bos_access_key_id, self.config.bos_secret_access_key) + configuration = BceClientConfiguration(credentials=credentials, endpoint=self.config.bos_endpoint) + self._client = BosClient(configuration) + return self._client + + +def normalize_prefix(value: str) -> str: + parts = PurePosixPath(str(value).strip().strip("/")) + if not str(parts) or str(parts) == "." or ".." in parts.parts: + raise StorageBoundaryError("BOS prefix must be a non-empty relative path") + return str(parts) + + +def normalize_relative_path(value: str) -> str: + raw = str(value or "").replace("\\", "/").strip().lstrip("/") + if not raw: + return "" + path = PurePosixPath(raw) + if path.is_absolute() or ".." in path.parts: + raise StorageBoundaryError("BOS path traversal is not allowed") + return str(path) + + +def safe_object_name(filename: str) -> str: + name = Path(filename).name.strip() + cleaned = _SAFE_COMPONENT.sub("-", name).strip(".-") + if not cleaned: + raise ValueError("filename is empty after sanitization") + return cleaned[:180] + + +def sha256_file(path: Path, chunk_size: int = 8 * 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(chunk_size): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_write_part(path: Path, chunks: Iterable[bytes]) -> tuple[int, str]: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + digest = hashlib.sha256() + size = 0 + with temporary.open("wb") as handle: + for chunk in chunks: + handle.write(chunk) + digest.update(chunk) + size += len(chunk) + handle.flush() + os.fsync(handle.fileno()) + temporary.replace(path) + return size, digest.hexdigest() diff --git a/src/real2sim_platform/store.py b/src/real2sim_platform/store.py index 506f23c..15c0f9a 100644 --- a/src/real2sim_platform/store.py +++ b/src/real2sim_platform/store.py @@ -28,13 +28,16 @@ class PlatformStore: manifest_path: Path | None = None, metrics: dict[str, Any] | None = None, tags: list[str] | None = None, + storage_uri: str = "", + checksum: str = "", + cache_state: str = "local", ) -> dict[str, Any]: item_id = item_id or uuid.uuid4().hex now = _now() with self._connect() as db: db.execute( - "insert into datasets(id,name,path,scan_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)", - (item_id, name, str(path), _json(scan), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now), + "insert into datasets(id,name,path,scan_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (item_id, name, str(path), _json(scan), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now), ) return self.get_dataset(item_id) @@ -65,13 +68,16 @@ class PlatformStore: manifest_path: Path | None = None, metrics: dict[str, Any] | None = None, tags: list[str] | None = None, + storage_uri: str = "", + checksum: str = "", + cache_state: str = "local", ) -> dict[str, Any]: item_id = item_id or uuid.uuid4().hex now = _now() with self._connect() as db: db.execute( - "insert into scenes(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)", - (item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now), + "insert into scenes(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now), ) return self.get_scene(item_id) @@ -125,13 +131,16 @@ class PlatformStore: manifest_path: Path | None = None, metrics: dict[str, Any] | None = None, tags: list[str] | None = None, + storage_uri: str = "", + checksum: str = "", + cache_state: str = "local", ) -> dict[str, Any]: item_id = item_id or uuid.uuid4().hex now = _now() with self._connect() as db: db.execute( - "insert into models(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)", - (item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now), + "insert into models(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now), ) return self.get_model(item_id) @@ -148,6 +157,18 @@ class PlatformStore: def delete_model(self, item_id: str) -> None: self._delete("models", item_id) + def update_entity_storage(self, table: str, item_id: str, *, storage_uri: str, checksum: str = "", cache_state: str = "local") -> dict[str, Any]: + if table not in {"datasets", "scenes", "models"}: + raise ValueError("unsupported entity table") + mapper = {"datasets": self._dataset_row, "scenes": self._scene_row, "models": self._model_row}[table] + self._get(table, item_id, mapper) + with self._connect() as db: + db.execute( + f"update {table} set storage_uri=?,checksum=?,cache_state=?,updated_at=? where id=?", + (storage_uri, checksum, cache_state, _now(), item_id), + ) + return self._get(table, item_id, mapper) + def create_task(self, *, task_type: str, params: dict[str, Any], run_dir: Path) -> dict[str, Any]: item_id = uuid.uuid4().hex now = _now() @@ -164,7 +185,33 @@ class PlatformStore: def get_task(self, item_id: str) -> dict[str, Any]: return self._get("tasks", item_id, self._task_row) + def claim_next_queued_task(self, worker_id: str) -> dict[str, Any] | None: + db = self._connect() + try: + db.execute("begin immediate") + row = db.execute("select * from tasks where status='queued' order by created_at limit 1").fetchone() + if row is None: + db.commit() + return None + now = _now() + cursor = db.execute( + "update tasks set status='running',worker_id=?,claimed_at=?,attempt=attempt+1,progress=max(progress,1),stage='启动中',message='GPU worker 已领取任务',updated_at=? where id=? and status='queued'", + (worker_id, now, now, row["id"]), + ) + if cursor.rowcount != 1: + db.rollback() + return None + claimed = db.execute("select * from tasks where id=?", (row["id"],)).fetchone() + db.commit() + return self._task_row(claimed) + except Exception: + db.rollback() + raise + finally: + db.close() + def next_queued_task(self) -> dict[str, Any] | None: + """Compatibility helper for older callers; new workers must claim atomically.""" with self._connect() as db: row = db.execute("select * from tasks where status='queued' order by created_at limit 1").fetchone() return self._task_row(row) if row else None @@ -258,6 +305,153 @@ class PlatformStore: ) return self.get_artifact(item_id) + def create_user(self, *, username: str, password_hash: str, role: str, must_change_password: bool = True) -> dict[str, Any]: + item_id = uuid.uuid4().hex + now = _now() + with self._connect() as db: + db.execute( + "insert into users(id,username,password_hash,role,disabled,must_change_password,created_at,updated_at) values(?,?,?,?,0,?,?,?)", + (item_id, username, password_hash, role, int(must_change_password), now, now), + ) + return self.get_user(item_id) + + def count_users(self) -> int: + with self._connect() as db: + return int(db.execute("select count(*) from users").fetchone()[0]) + + def list_users(self) -> list[dict[str, Any]]: + with self._connect() as db: + rows = db.execute("select * from users order by created_at").fetchall() + return [self._user_row(row) for row in rows] + + def get_user(self, user_id: str) -> dict[str, Any]: + return self._get("users", user_id, self._user_row) + + def get_user_by_username(self, username: str) -> dict[str, Any] | None: + with self._connect() as db: + row = db.execute("select * from users where lower(username)=lower(?)", (username,)).fetchone() + return self._user_row(row, include_password=True) if row else None + + def update_user( + self, + user_id: str, + *, + role: str | None = None, + disabled: bool | None = None, + password_hash: str | None = None, + must_change_password: bool | None = None, + last_login: float | None = None, + ) -> dict[str, Any]: + self.get_user(user_id) + fields: list[str] = [] + values: list[Any] = [] + for name, value in (("role", role), ("password_hash", password_hash), ("last_login", last_login)): + if value is not None: + fields.append(f"{name}=?") + values.append(value) + for name, value in (("disabled", disabled), ("must_change_password", must_change_password)): + if value is not None: + fields.append(f"{name}=?") + values.append(int(value)) + if fields: + fields.append("updated_at=?") + values.extend([_now(), user_id]) + with self._connect() as db: + db.execute(f"update users set {','.join(fields)} where id=?", values) + return self.get_user(user_id) + + def delete_user(self, user_id: str) -> None: + self._delete("users", user_id) + + def create_session(self, *, token_hash: str, user_id: str, expires_at: float) -> None: + now = _now() + with self._connect() as db: + db.execute( + "insert into sessions(token_hash,user_id,expires_at,created_at,last_seen) values(?,?,?,?,?)", + (token_hash, user_id, expires_at, now, now), + ) + + def get_session_user(self, token_hash: str) -> dict[str, Any] | None: + now = _now() + with self._connect() as db: + row = db.execute( + "select users.* from sessions join users on users.id=sessions.user_id where sessions.token_hash=? and sessions.expires_at>? and users.disabled=0", + (token_hash, now), + ).fetchone() + if row: + db.execute("update sessions set last_seen=? where token_hash=?", (now, token_hash)) + db.execute("delete from sessions where expires_at<=?", (now,)) + return self._user_row(row) if row else None + + def delete_session(self, token_hash: str) -> None: + with self._connect() as db: + db.execute("delete from sessions where token_hash=?", (token_hash,)) + + def delete_user_sessions(self, user_id: str) -> None: + with self._connect() as db: + db.execute("delete from sessions where user_id=?", (user_id,)) + + def create_upload( + self, + *, + owner_user_id: str | None, + filename: str, + content_type: str, + total_size: int, + chunk_size: int, + total_parts: int, + checksum_expected: str, + local_dir: Path, + item_id: str | None = None, + ) -> dict[str, Any]: + item_id = item_id or uuid.uuid4().hex + now = _now() + with self._connect() as db: + db.execute( + "insert into uploads(id,owner_user_id,filename,content_type,total_size,chunk_size,total_parts,received_parts_json,checksum_expected,checksum_actual,status,local_dir,storage_uri,error,created_at,updated_at) values(?,?,?,?,?,?,?,'[]',?,?,'uploading',?,'','',?,?)", + (item_id, owner_user_id, filename, content_type, total_size, chunk_size, total_parts, checksum_expected, "", str(local_dir), now, now), + ) + return self.get_upload(item_id) + + def get_upload(self, upload_id: str) -> dict[str, Any]: + return self._get("uploads", upload_id, self._upload_row) + + def record_upload_part(self, upload_id: str, part_number: int) -> dict[str, Any]: + upload = self.get_upload(upload_id) + parts = sorted(set([*upload["received_parts"], int(part_number)])) + with self._connect() as db: + db.execute("update uploads set received_parts_json=?,updated_at=? where id=?", (_json(parts), _now(), upload_id)) + return self.get_upload(upload_id) + + def complete_upload(self, upload_id: str, *, checksum_actual: str, storage_uri: str) -> dict[str, Any]: + with self._connect() as db: + db.execute( + "update uploads set status='completed',checksum_actual=?,storage_uri=?,error='',updated_at=? where id=?", + (checksum_actual, storage_uri, _now(), upload_id), + ) + return self.get_upload(upload_id) + + def fail_upload(self, upload_id: str, error: str) -> dict[str, Any]: + with self._connect() as db: + db.execute("update uploads set status='failed',error=?,updated_at=? where id=?", (error, _now(), upload_id)) + return self.get_upload(upload_id) + + def delete_upload(self, upload_id: str) -> None: + self._delete("uploads", upload_id) + + def update_worker_heartbeat(self, worker_id: str, payload: dict[str, Any]) -> None: + now = _now() + with self._connect() as db: + db.execute( + "insert into worker_heartbeats(worker_id,payload_json,updated_at) values(?,?,?) on conflict(worker_id) do update set payload_json=excluded.payload_json,updated_at=excluded.updated_at", + (worker_id, _json(payload), now), + ) + + def list_worker_heartbeats(self) -> list[dict[str, Any]]: + with self._connect() as db: + rows = db.execute("select * from worker_heartbeats order by updated_at desc").fetchall() + return [{"worker_id": row["worker_id"], "payload": json.loads(row["payload_json"] or "{}"), "updated_at": row["updated_at"]} for row in rows] + def list_task_artifacts(self, task_id: str) -> list[dict[str, Any]]: with self._connect() as db: rows = db.execute("select * from artifacts where task_id=? order by created_at", (task_id,)).fetchall() @@ -268,6 +462,8 @@ class PlatformStore: def _init(self) -> None: with self._connect() as db: + db.execute("pragma journal_mode=WAL") + db.execute("pragma synchronous=NORMAL") db.execute("create table if not exists datasets(id text primary key,name text,path text,scan_json text,created_at real,updated_at real)") db.execute("create table if not exists scenes(id text primary key,name text,path text,metadata_json text,created_at real,updated_at real)") db.execute("create table if not exists models(id text primary key,name text,path text,metadata_json text,created_at real,updated_at real)") @@ -277,6 +473,16 @@ class PlatformStore: "run_dir text,error text,cancel_requested integer not null default 0,created_at real,updated_at real)" ) db.execute("create table if not exists artifacts(id text primary key,task_id text,name text,path text,kind text,created_at real)") + db.execute( + "create table if not exists users(id text primary key,username text not null unique,password_hash text not null,role text not null,disabled integer not null default 0,must_change_password integer not null default 1,last_login real,created_at real,updated_at real)" + ) + db.execute( + "create table if not exists sessions(token_hash text primary key,user_id text not null,expires_at real not null,created_at real,last_seen real,foreign key(user_id) references users(id) on delete cascade)" + ) + db.execute( + "create table if not exists uploads(id text primary key,owner_user_id text,filename text not null,content_type text,total_size integer not null,chunk_size integer not null,total_parts integer not null,received_parts_json text not null default '[]',checksum_expected text,checksum_actual text,status text not null,local_dir text not null,storage_uri text,error text,created_at real,updated_at real)" + ) + db.execute("create table if not exists worker_heartbeats(worker_id text primary key,payload_json text not null,updated_at real not null)") for table in ["datasets", "scenes", "models"]: self._ensure_column(db, table, "source_type", "text not null default 'local'") self._ensure_column(db, table, "source_uri", "text not null default ''") @@ -285,14 +491,24 @@ class PlatformStore: self._ensure_column(db, table, "manifest_path", "text not null default ''") self._ensure_column(db, table, "metrics_json", "text not null default '{}'") self._ensure_column(db, table, "tags_json", "text not null default '[]'") + self._ensure_column(db, table, "storage_uri", "text not null default ''") + self._ensure_column(db, table, "checksum", "text not null default ''") + self._ensure_column(db, table, "cache_state", "text not null default 'local'") self._ensure_column(db, "tasks", "pid", "integer") self._ensure_column(db, "tasks", "progress", "integer not null default 0") self._ensure_column(db, "tasks", "stage", "text not null default ''") self._ensure_column(db, "tasks", "message", "text not null default ''") + self._ensure_column(db, "tasks", "worker_id", "text not null default ''") + self._ensure_column(db, "tasks", "claimed_at", "real") + self._ensure_column(db, "tasks", "attempt", "integer not null default 0") + db.execute("create index if not exists idx_tasks_status_created on tasks(status,created_at)") + db.execute("create index if not exists idx_sessions_user on sessions(user_id)") def _connect(self) -> sqlite3.Connection: - db = sqlite3.connect(self.db_path) + db = sqlite3.connect(self.db_path, timeout=30) db.row_factory = sqlite3.Row + db.execute("pragma busy_timeout=5000") + db.execute("pragma foreign_keys=on") return db def _ensure_column(self, db: sqlite3.Connection, table: str, column: str, spec: str) -> None: @@ -359,6 +575,9 @@ class PlatformStore: "manifest_path": row["manifest_path"], "metrics": json.loads(row["metrics_json"] or "{}"), "tags": json.loads(row["tags_json"] or "[]"), + "storage_uri": row["storage_uri"], + "checksum": row["checksum"], + "cache_state": row["cache_state"], "created_at": row["created_at"], "updated_at": row["updated_at"], } @@ -377,6 +596,9 @@ class PlatformStore: "message": row["message"] or "", "cancel_requested": bool(row["cancel_requested"]), "pid": row["pid"], + "worker_id": row["worker_id"], + "claimed_at": row["claimed_at"], + "attempt": int(row["attempt"] or 0), "created_at": row["created_at"], "updated_at": row["updated_at"], } @@ -384,6 +606,41 @@ class PlatformStore: def _artifact_row(self, row: sqlite3.Row) -> dict[str, Any]: return {"id": row["id"], "task_id": row["task_id"], "name": row["name"], "path": row["path"], "kind": row["kind"], "created_at": row["created_at"]} + def _user_row(self, row: sqlite3.Row, *, include_password: bool = False) -> dict[str, Any]: + item = { + "id": row["id"], + "username": row["username"], + "role": row["role"], + "disabled": bool(row["disabled"]), + "must_change_password": bool(row["must_change_password"]), + "last_login": row["last_login"], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + if include_password: + item["password_hash"] = row["password_hash"] + return item + + def _upload_row(self, row: sqlite3.Row) -> dict[str, Any]: + return { + "id": row["id"], + "owner_user_id": row["owner_user_id"], + "filename": row["filename"], + "content_type": row["content_type"] or "application/octet-stream", + "total_size": int(row["total_size"]), + "chunk_size": int(row["chunk_size"]), + "total_parts": int(row["total_parts"]), + "received_parts": json.loads(row["received_parts_json"] or "[]"), + "checksum_expected": row["checksum_expected"] or "", + "checksum_actual": row["checksum_actual"] or "", + "status": row["status"], + "local_dir": row["local_dir"], + "storage_uri": row["storage_uri"] or "", + "error": row["error"] or "", + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + def _json(value: Any) -> str: return json.dumps(value, ensure_ascii=False) diff --git a/src/real2sim_platform/worker.py b/src/real2sim_platform/worker.py index 767884d..133fe87 100644 --- a/src/real2sim_platform/worker.py +++ b/src/real2sim_platform/worker.py @@ -7,7 +7,7 @@ from typing import Any from pathlib import Path -from .adapters import TaskCanceled, execute_task, index_artifacts_for_task +from .adapters import TaskCanceled, execute_task, index_artifacts_for_task, run_doctor, sync_task_outputs from .config import PlatformConfig from .store import PlatformStore @@ -18,40 +18,65 @@ class TaskWorker: self.config = config self._stop = threading.Event() self._thread: threading.Thread | None = None + self._last_heartbeat = 0.0 def start(self) -> None: if self._thread and self._thread.is_alive(): return + self._stop.clear() self.store.recover_interrupted_tasks() + self._heartbeat(force=True) self._thread = threading.Thread(target=self._loop, name="real2sim-platform-worker", daemon=True) self._thread.start() def stop(self) -> None: self._stop.set() + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=5) def _loop(self) -> None: while not self._stop.is_set(): - task = self.store.next_queued_task() + self._heartbeat() + task = self.store.claim_next_queued_task(self.config.worker_id) if task is None: time.sleep(0.5) continue self._run(task) + def _heartbeat(self, *, force: bool = False) -> None: + now = time.time() + if not force and now - self._last_heartbeat < 10: + return + report = run_doctor(self.config) + self.store.update_worker_heartbeat( + self.config.worker_id, + { + "ready": bool(report.get("ok")), + "checks": report.get("checks", {}), + "warnings": report.get("warnings", []), + "pid": __import__("os").getpid(), + }, + ) + self._last_heartbeat = now + def _run(self, task: dict[str, Any]) -> None: if self.store.get_task(task["id"])["cancel_requested"]: self.store.update_task(task["id"], status="canceled", error="task canceled before start", progress=0, stage="已取消", message="任务启动前已取消") return - self.store.update_task(task["id"], status="running", progress=5, stage="启动中", message="任务已进入执行队列") + self.store.update_task(task["id"], status="running", progress=5, stage="启动中", message=f"{self.config.worker_id} 正在初始化任务") try: result = execute_task(self.config, self.store, task) + result = sync_task_outputs(self.config, self.store, task, result) except TaskCanceled as exc: self._write_trace(task, exc) index_artifacts_for_task(self.store, task["id"], Path(task["run_dir"])) + self._persist_failed_run(task) self.store.update_task(task["id"], status="canceled", error=str(exc), progress=0, stage="已取消", message=str(exc)) return except Exception as exc: self._write_trace(task, exc) index_artifacts_for_task(self.store, task["id"], Path(task["run_dir"])) + self._persist_failed_run(task) self.store.update_task(task["id"], status="failed", error=str(exc), stage="执行失败", message=str(exc)) return status = "canceled" if self.store.get_task(task["id"])["cancel_requested"] else "succeeded" @@ -65,3 +90,11 @@ class TaskWorker: with log_path.open("a", encoding="utf-8") as handle: handle.write("\nERROR: " + str(exc) + "\n") handle.write(traceback.format_exc() + "\n") + + def _persist_failed_run(self, task: dict[str, Any]) -> None: + try: + sync_task_outputs(self.config, self.store, task, {}) + except Exception as exc: + log_path = Path(task["run_dir"]) / "logs" / "task.log" + with log_path.open("a", encoding="utf-8") as handle: + handle.write(f"\nfailed to persist terminal task artifacts: {exc}\n") diff --git a/src/real2sim_platform/worker_main.py b/src/real2sim_platform/worker_main.py new file mode 100644 index 0000000..9d68218 --- /dev/null +++ b/src/real2sim_platform/worker_main.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import argparse +import signal +import time +from pathlib import Path + +from .config import ensure_workspace, load_config +from .store import PlatformStore +from .worker import TaskWorker + + +def main() -> int: + parser = argparse.ArgumentParser(description="Real2Sim serial GPU worker") + parser.add_argument("--config", type=Path) + args = parser.parse_args() + config = load_config(args.config) + ensure_workspace(config) + store = PlatformStore(config.db_path) + worker = TaskWorker(store, config) + stopping = False + + def stop(*_args) -> None: + nonlocal stopping + stopping = True + worker.stop() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + worker.start() + while not stopping: + time.sleep(1) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_platform.py b/tests/test_platform.py index 5c9c224..d9d213b 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -215,11 +215,14 @@ def test_scene_asset_append_updates_manifest_and_artifact(tmp_path: Path) -> Non assert response.status_code == 200, response.text payload = response.json() assert payload["asset"]["name"] == "fixture metadata" - manifest = client.get(f"/api/scenes/{scene['id']}/manifest").json() + assert payload["scene"]["id"] != scene["id"] + assert payload["parent_scene_id"] == scene["id"] + manifest = client.get(f"/api/scenes/{payload['scene']['id']}/manifest").json() added = [item for item in manifest["assets"] if item["name"] == "fixture metadata"] assert added assert added[0]["role"] == "simulator_import" assert added[0]["transform"]["translation"] == [1, 2, 3] + assert manifest["revision"]["parent_scene_id"] == scene["id"] assert client.get(added[0]["url"]).status_code == 200 diff --git a/tests/test_production_platform.py b/tests/test_production_platform.py new file mode 100644 index 0000000..957a34c --- /dev/null +++ b/tests/test_production_platform.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import concurrent.futures +from pathlib import Path + +import h5py +import pytest +from fastapi.testclient import TestClient + +from real2sim_platform.config import PlatformConfig +from real2sim_platform.rm65_controller import collect +from real2sim_platform.server import create_app +from real2sim_platform.storage import BosStorage, StorageBoundaryError, normalize_relative_path +from real2sim_platform.store import PlatformStore + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_auth_first_login_rbac_and_user_crud(tmp_path: Path) -> None: + config = PlatformConfig( + app_root=ROOT, + workspace_dir=tmp_path / "workspace", + worker_mode="api", + auth_enabled=True, + bootstrap_admin_password="Admin!Password123", + ) + with TestClient(create_app(config)) as client: + assert client.get("/api/datasets").status_code == 401 + login = client.post("/api/auth/login", json={"username": "admin", "password": "Admin!Password123"}) + assert login.status_code == 200 + assert login.json()["user"]["must_change_password"] is True + assert client.get("/api/datasets").status_code == 428 + changed = client.post( + "/api/auth/change-password", + json={"current_password": "Admin!Password123", "new_password": "Changed!Password456"}, + ) + assert changed.status_code == 200 + assert client.get("/api/datasets").status_code == 401 + assert client.post("/api/auth/login", json={"username": "admin", "password": "Changed!Password456"}).status_code == 200 + viewer = client.post( + "/api/users", + json={"username": "viewer.user", "password": "Viewer!Password123", "role": "viewer"}, + ) + assert viewer.status_code == 200 + assert viewer.json()["role"] == "viewer" + assert client.patch(f"/api/users/{viewer.json()['id']}", json={"disabled": True}).json()["disabled"] is True + + viewer_config = PlatformConfig( + app_root=ROOT, + workspace_dir=config.workspace_dir, + worker_mode="api", + auth_enabled=True, + bootstrap_admin_password="unused-Password!123", + ) + store = PlatformStore(viewer_config.db_path) + viewer_id = next(item["id"] for item in store.list_users() if item["username"] == "viewer.user") + store.update_user(viewer_id, disabled=False) + with TestClient(create_app(viewer_config)) as viewer_client: + assert viewer_client.post("/api/auth/login", json={"username": "viewer.user", "password": "Viewer!Password123"}).status_code == 200 + assert viewer_client.get("/api/auth/me").status_code == 200 + assert viewer_client.post("/api/auth/change-password", json={"current_password": "Viewer!Password123", "new_password": "Viewer!Changed456"}).status_code == 200 + assert viewer_client.post("/api/auth/login", json={"username": "viewer.user", "password": "Viewer!Changed456"}).status_code == 200 + assert viewer_client.get("/api/datasets").status_code == 200 + assert viewer_client.post("/api/tasks", json={"type": "renderer_benchmark", "params": {}}).status_code == 403 + + +def test_bos_prefix_boundary_is_enforced(tmp_path: Path) -> None: + config = PlatformConfig( + workspace_dir=tmp_path, + bos_bucket="testxyz", + bos_prefix="real2sim-poc", + bos_access_key_id="configured-but-not-used", + bos_secret_access_key="configured-but-not-used", + ) + storage = BosStorage(config) + assert storage.object_key("datasets", "team/a.json") == "real2sim-poc/datasets/team/a.json" + assert storage.key_from_uri("bos://testxyz/real2sim-poc/models/m1/card.json") == "real2sim-poc/models/m1/card.json" + with pytest.raises(StorageBoundaryError): + storage.object_key("datasets", "../../droboticflow/secret") + with pytest.raises(StorageBoundaryError): + storage.assert_owned_key("droboticflow/workspaces/data") + with pytest.raises(StorageBoundaryError): + storage.key_from_uri("bos://another-bucket/real2sim-poc/datasets/a") + with pytest.raises(StorageBoundaryError): + normalize_relative_path("../outside") + + +def test_atomic_task_claim_has_no_duplicates(tmp_path: Path) -> None: + store = PlatformStore(tmp_path / "state.db") + for index in range(24): + store.create_task(task_type="renderer_benchmark", params={"index": index}, run_dir=tmp_path / f"run-{index}") + + def claim(worker: int): + task = store.claim_next_queued_task(f"worker-{worker}") + return task["id"] if task else None + + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor: + claimed = [item for item in executor.map(claim, range(24)) if item] + assert len(claimed) == 24 + assert len(set(claimed)) == 24 + assert all(task["status"] == "running" and task["attempt"] == 1 for task in store.list_tasks()) + + +def test_api_capabilities_follow_live_worker_heartbeat(tmp_path: Path) -> None: + config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api") + store = PlatformStore(config.db_path) + store.update_worker_heartbeat( + "worker-a800", + { + "ready": True, + "checks": { + "ffmpeg": True, + "colmap": True, + "anysplat_root": True, + "anysplat_python": True, + "anysplat_pinned": True, + "mujoco_python": True, + "motrixsim_python": True, + "discoverse_act": True, + "torch_cuda": True, + "act_import": True, + "rm65_controller": True, + }, + }, + ) + with TestClient(create_app(config)) as client: + capabilities = client.get("/api/capabilities").json() + assert capabilities["worker"]["connected"] is True + assert capabilities["runtimes"]["anysplat"]["ready"] is True + assert capabilities["runtimes"]["discoverse_act"]["ready"] is True + assert capabilities["runtimes"]["rm65_controller"]["ready"] is True + + +def test_login_rate_limit_blocks_repeated_failures(tmp_path: Path) -> None: + config = PlatformConfig( + app_root=ROOT, + workspace_dir=tmp_path / "workspace", + worker_mode="api", + auth_enabled=True, + bootstrap_admin_password="Admin!Password123", + ) + with TestClient(create_app(config)) as client: + for _ in range(8): + assert client.post("/api/auth/login", json={"username": "admin", "password": "wrong-password"}).status_code == 401 + assert client.post("/api/auth/login", json={"username": "admin", "password": "wrong-password"}).status_code == 429 + + +def test_production_task_gates_reject_fake_routes(tmp_path: Path) -> None: + config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api", production_mode=True) + client = TestClient(create_app(config)) + dry_run = client.post( + "/api/tasks", + json={"type": "scene_reconstruction", "params": {"input_path": str(tmp_path), "dry_run": True}}, + ) + assert dry_run.status_code == 400 + legacy = client.post( + "/api/tasks", + json={"type": "training", "params": {"dataset_path": str(tmp_path), "recipe": "bc_v2"}}, + ) + assert legacy.status_code == 400 + fake_expert = client.post( + "/api/tasks", + json={"type": "expert_collection", "params": {"scene_path": str(tmp_path), "policy_preset": "pick_and_place_v1"}}, + ) + assert fake_expert.status_code == 400 + + +@pytest.mark.integration +def test_real_rm65_physical_collection_writes_act_hdf5(tmp_path: Path) -> None: + scene = ROOT / "workspace/motrixsim_assets/parlour_rm65_pick_place/simulator_assets/rm65_table_pick_place.xml" + if not scene.is_file(): + pytest.skip("real imported RM65 scene asset is not present") + manifest = collect(scene, tmp_path / "collection", required_successes=1, max_attempts=2, max_steps=240, seed=7, width=160, height=120) + assert manifest["successful_episodes"] == 1 + assert manifest["episodes"][0]["lift_height"] >= 0.10 + assert manifest["episodes"][0]["place_error"] <= 0.05 + episode = tmp_path / "collection/episode_0.hdf5" + with h5py.File(episode, "r") as root: + assert root["/observations/qpos"].shape[1] == 7 + assert root["/action"].shape[1] == 7 + assert root["/observations/images/overview"].shape[0] > 0 + assert root["/observations/contact"].shape[1] == 2 + assert bool(root.attrs["success"]) is True + assert (tmp_path / "collection/episode_0.mp4").stat().st_size > 0 diff --git a/vendor/gs_playground/LICENSE b/vendor/gs_playground/LICENSE new file mode 100644 index 0000000..716ca82 --- /dev/null +++ b/vendor/gs_playground/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Yufei Jia (贾宇飞) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/gs_playground/UPSTREAM_COMMIT b/vendor/gs_playground/UPSTREAM_COMMIT new file mode 100644 index 0000000..f99216c --- /dev/null +++ b/vendor/gs_playground/UPSTREAM_COMMIT @@ -0,0 +1 @@ +e10805930b2b652f5608c41f671c3460b47c0c72 diff --git a/vendor/gs_playground/real2sim/__init__.py b/vendor/gs_playground/real2sim/__init__.py new file mode 100644 index 0000000..85624f8 --- /dev/null +++ b/vendor/gs_playground/real2sim/__init__.py @@ -0,0 +1,9 @@ +"""Real2Sim pipeline tools for GS-Playground.""" + +from .errors import PipelineError, MissingStageCommand, StageArtifactError + +__all__ = [ + "PipelineError", + "MissingStageCommand", + "StageArtifactError", +] diff --git a/vendor/gs_playground/real2sim/__main__.py b/vendor/gs_playground/real2sim/__main__.py new file mode 100644 index 0000000..a049ad7 --- /dev/null +++ b/vendor/gs_playground/real2sim/__main__.py @@ -0,0 +1,5 @@ +from .cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/real2sim/builtin.py b/vendor/gs_playground/real2sim/builtin.py new file mode 100644 index 0000000..fe59fdf --- /dev/null +++ b/vendor/gs_playground/real2sim/builtin.py @@ -0,0 +1,1023 @@ +"""Built-in single-image Real2Sim baseline. + +This is a real, deterministic image-to-assets implementation designed to make +the public preview repository usable before the paper authors release their +GroundingDINO/SAM/LaMa/SAM-3D/AnySplat stack. + +It does not claim the visual quality of AnySplat/SAM-3D. It produces valid +sim-ready assets from a single RGB image: instance masks, an inpainted +background, 3DGS PLY files, OBJ meshes, object poses, camera metadata, and a +manifest consumable by the GS-Playground package writer. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from .contracts import Pose, Real2SimAsset, Real2SimManifest, load_instances, write_json +from .image_io import read_gray, read_rgb, write_gray, write_rgb + + +PPM_SUFFIXES = {".ppm"} +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".ppm", ".pgm", ".webp"} + + +@dataclass(frozen=True) +class BuiltinOptions: + max_scene_points: int = 80000 + max_object_points: int = 20000 + max_objects: int = 6 + min_component_area_ratio: float = 0.0025 + stage_min_component_area_ratio: float = 0.0005 + focal_scale: float = 0.85 + prefer_stage_artifacts: bool = False + external_depth_weight: float = 0.0 + color_enhance_strength: float = 0.0 + scene_point_scale: float = 0.012 + object_point_scale: float = 0.012 + opacity_logit: float = 4.0 + multiview_yaw_scale: float = 0.06 + multiview_lateral_step: float = 0.006 + mask_close_passes: int = 1 + mask_dilate_passes: int = 0 + max_mask_overlap: float = 0.75 + + @classmethod + def from_config(cls, config: dict[str, Any] | None) -> "BuiltinOptions": + raw = (config or {}).get("builtin", {}) + if not isinstance(raw, dict): + raw = {} + return cls( + max_scene_points=int(raw.get("max_scene_points", 80000)), + max_object_points=int(raw.get("max_object_points", 20000)), + max_objects=int(raw.get("max_objects", 6)), + min_component_area_ratio=float(raw.get("min_component_area_ratio", 0.0025)), + stage_min_component_area_ratio=float(raw.get("stage_min_component_area_ratio", 0.0005)), + focal_scale=float(raw.get("focal_scale", 0.85)), + prefer_stage_artifacts=bool(raw.get("prefer_stage_artifacts", False)), + external_depth_weight=_clamp01(float(raw.get("external_depth_weight", 0.0))), + color_enhance_strength=_clamp01(float(raw.get("color_enhance_strength", 0.0))), + scene_point_scale=max(0.001, float(raw.get("scene_point_scale", 0.012))), + object_point_scale=max(0.001, float(raw.get("object_point_scale", 0.012))), + opacity_logit=float(raw.get("opacity_logit", 4.0)), + multiview_yaw_scale=max(0.0, min(1.0, float(raw.get("multiview_yaw_scale", 0.06)))), + multiview_lateral_step=max(0.0, float(raw.get("multiview_lateral_step", 0.006))), + mask_close_passes=max(0, int(raw.get("mask_close_passes", 1))), + mask_dilate_passes=max(0, int(raw.get("mask_dilate_passes", 0))), + max_mask_overlap=_clamp01(float(raw.get("max_mask_overlap", 0.75))), + ) + + +def run_builtin_real2sim( + capture, + context: dict[str, Any], + prompt: str, + config: dict[str, Any] | None = None, + *, + include_objects: bool = True, +) -> Real2SimManifest: + options = BuiltinOptions.from_config(config) + rgb = read_rgb(Path(context["image"])) + height, width = rgb.shape[:2] + focal = options.focal_scale * max(width, height) + intrinsics = { + "width": width, + "height": height, + "fx": focal, + "fy": focal, + "cx": (width - 1) / 2.0, + "cy": (height - 1) / 2.0, + } + + depth = estimate_pseudo_depth(rgb) + if options.prefer_stage_artifacts: + depth = fuse_depth_from_artifact(depth, Path(context["depth_map"]), rgb.shape[:2], weight=options.external_depth_weight) + + component_records = [] + using_stage_components = False + if include_objects: + if options.prefer_stage_artifacts: + component_records = load_stage_component_records(context, rgb.shape[:2]) + using_stage_components = bool(component_records) + if not component_records: + foreground = estimate_foreground_mask(rgb) + components = select_components(foreground, options.max_objects, options.min_component_area_ratio) + if not components: + components = [central_component(height, width)] + component_records = [(f"object_{idx:02d}", f"object_{idx:02d}", mask) for idx, mask in enumerate(components)] + write_builtin_detection_artifacts(component_records, context) + min_area_ratio = options.stage_min_component_area_ratio if using_stage_components else options.min_component_area_ratio + component_records = refine_component_records(component_records, options, rgb.shape[:2], min_area_ratio=min_area_ratio) + target_count = prompt_target_object_count(prompt) + if target_count is not None and len(component_records) >= target_count: + component_records = select_tabletop_subject_records(component_records, rgb.shape[:2], target_count, rgb=rgb) + write_builtin_detection_artifacts(component_records, context) + + all_object_mask = np.zeros((height, width), dtype=bool) + for _, _, mask in component_records: + all_object_mask |= mask + background_image = Path(context["background_image"]) + if not include_objects: + background = rgb + background_image = background_image.with_suffix(".ppm") + write_rgb(background_image, background) + elif options.prefer_stage_artifacts and background_image.exists(): + background = read_rgb(background_image) + if background.shape[:2] != (height, width): + background = inpaint_with_border_color(rgb, all_object_mask) + background_image = background_image.with_suffix(".ppm") + write_rgb(background_image, background) + else: + background = inpaint_with_border_color(rgb, all_object_mask) + background_image = background_image.with_suffix(".ppm") + write_rgb(background_image, background) + depth_map = Path(context["depth_map"]).with_suffix(".pgm") + context["background_image"] = background_image + context["depth_map"] = depth_map + write_gray(depth_map, normalize(depth)) + + scene_dir = Path(context["scene_reconstruction_dir"]) + scene_dir.mkdir(parents=True, exist_ok=True) + scene_ply = Path(context["scene_ply"]) + scene_mesh = Path(context["scene_mesh"]) + scene_rgb = enhance_colors(background, options.color_enhance_strength) + object_rgb = enhance_colors(rgb, options.color_enhance_strength) + points, colors, camera_views = reconstruct_scene_points(capture, scene_rgb, depth, intrinsics, options) + write_3dgs_ply(scene_ply, points, colors, point_scale=options.scene_point_scale, opacity_logit=options.opacity_logit) + write_depth_mesh_obj(scene_mesh, scene_rgb, depth, intrinsics, grid=96) + + cameras_json = Path(context["cameras_json"]) + write_json( + cameras_json, + { + "intrinsics": intrinsics, + "extrinsics": { + "camera_to_world": [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ] + }, + "extrinsic": [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ], + "views": camera_views, + "note": "Built-in pseudo-depth camera metadata. Replace with AnySplat calibration when available.", + }, + ) + + object_assets: list[Real2SimAsset] = [] + objects_dir = Path(context["objects_dir"]) + objects_dir.mkdir(parents=True, exist_ok=True) + for idx, (instance_id, label, mask) in enumerate(component_records if include_objects else []): + name = safe_name(instance_id) or f"object_{idx:02d}" + object_dir = objects_dir / name + object_dir.mkdir(parents=True, exist_ok=True) + object_points, object_colors = backproject(object_rgb, depth, intrinsics, mask=mask, max_points=options.max_object_points) + if object_points.size == 0: + continue + center = object_points.mean(axis=0) + local_points = object_points - center + extent = np.maximum(object_points.max(axis=0) - object_points.min(axis=0), np.array([0.04, 0.04, 0.04], dtype=np.float32)) + object_ply = object_dir / "gaussian.ply" + object_mesh = object_dir / "mesh.obj" + object_pose = object_dir / "pose.json" + visual_points, visual_colors = object_billboard_points( + object_rgb, + mask, + extent, + label=label, + max_points=options.max_object_points, + ) + if visual_points.size == 0: + visual_points, visual_colors = local_points, object_colors + write_3dgs_ply(object_ply, visual_points, visual_colors, point_scale=options.object_point_scale, opacity_logit=options.opacity_logit) + write_box_obj(object_mesh, extent) + pose = Pose(position=tuple(float(x) for x in center), quat_wxyz=(1.0, 0.0, 0.0, 0.0), scale=(1.0, 1.0, 1.0)) + write_json(object_pose, pose.to_json()) + object_assets.append( + Real2SimAsset( + name=name, + label=label, + gaussian_ply=object_ply, + mesh_obj=object_mesh, + pose=pose, + dynamic=True, + source_instance_id=instance_id, + ) + ) + + return Real2SimManifest( + source=capture.source, + prompt=prompt, + scene=Real2SimAsset(name="scene", label="background_scene", gaussian_ply=scene_ply, mesh_obj=scene_mesh), + objects=tuple(object_assets), + camera_artifacts={ + "source_image": capture.image, + "depth_map": depth_map, + "cameras_json": cameras_json, + "background_image": background_image, + }, + metadata={ + "pipeline": "GS-Playground built-in Real2Sim baseline", + "capture_kind": capture.kind, + "backend": "builtin", + "builtin_scene_fusion": "synthetic_multiview" if len(camera_views) > 1 else "single_view", + "builtin_scene_view_count": len(camera_views), + "scene_generation_only": not include_objects, + "limitations": [ + "Pseudo-depth is estimated deterministically from RGB images.", + "Synthetic multiview fusion uses approximate canonical view yaw when real SfM/AnySplat poses are unavailable.", + "Instance masks use saliency and connected components unless external GroundingDINO/SAM commands are configured.", + "Prompt-aware tabletop subject filtering keeps the main requested assets separate from background scene clutter.", + "Object meshes are collision-ready boxes derived from reconstructed point extents.", + ], + }, + ) + + +def reconstruct_scene_points( + capture, + scene_rgb: np.ndarray, + depth: np.ndarray, + intrinsics: dict[str, float], + options: BuiltinOptions, +) -> tuple[np.ndarray, np.ndarray, list[dict[str, Any]]]: + frame_paths = capture_frame_paths(capture) + if len(frame_paths) <= 1: + mask = np.ones(depth.shape, dtype=bool) + points, colors = backproject(scene_rgb, depth, intrinsics, mask=mask, max_points=options.max_scene_points) + return points, colors, [camera_view_record(frame_paths[0] if frame_paths else Path(capture.image), 0, 0.0)] + + per_view_limit = max(1, options.max_scene_points // len(frame_paths)) + all_points: list[np.ndarray] = [] + all_colors: list[np.ndarray] = [] + camera_views: list[dict[str, Any]] = [] + for index, frame_path in enumerate(frame_paths): + frame_rgb = enhance_colors(read_rgb(frame_path), options.color_enhance_strength) + if frame_rgb.shape[:2] != depth.shape: + frame_rgb = resize_rgb_nearest(frame_rgb, depth.shape) + frame_depth = estimate_pseudo_depth(frame_rgb) + points, colors = backproject( + frame_rgb, + frame_depth, + intrinsics, + mask=np.ones(depth.shape, dtype=bool), + max_points=per_view_limit, + ) + yaw_degrees = canonical_multiview_yaw(index, len(frame_paths)) + points = transform_multiview_points(points, yaw_degrees, index, len(frame_paths), options) + all_points.append(points) + all_colors.append(colors) + camera_views.append(camera_view_record(frame_path, index, yaw_degrees)) + + points = np.concatenate(all_points, axis=0) if all_points else np.zeros((0, 3), dtype=np.float32) + colors = np.concatenate(all_colors, axis=0) if all_colors else np.zeros((0, 3), dtype=np.float32) + if len(points) > options.max_scene_points: + idx = deterministic_point_sample(points, options.max_scene_points) + points = points[idx] + colors = colors[idx] + return points.astype(np.float32), colors.astype(np.float32), camera_views + + +def capture_frame_paths(capture) -> list[Path]: + frames_dir = getattr(capture, "frames_dir", None) + if frames_dir: + frames = sorted(path for path in Path(frames_dir).iterdir() if path.suffix.lower() in IMAGE_SUFFIXES) + if frames: + return frames + return [Path(capture.image)] + + +def canonical_multiview_yaw(index: int, total: int) -> float: + canonical = [0.0, -45.0, -90.0, -135.0, 180.0, 135.0, 90.0, -25.0] + if total == 8 and index < len(canonical): + return canonical[index] + if total <= 1: + return 0.0 + return -180.0 + (360.0 * index / total) + + +def transform_multiview_points(points: np.ndarray, yaw_degrees: float, index: int, total: int, options: BuiltinOptions) -> np.ndarray: + if points.size == 0: + return points + centered = points.copy() + yaw = math.radians(yaw_degrees * options.multiview_yaw_scale) + cos_y = math.cos(yaw) + sin_y = math.sin(yaw) + x = centered[:, 0].copy() + y = centered[:, 1].copy() + centered[:, 0] = (cos_y * x) - (sin_y * y) + centered[:, 1] = (sin_y * x) + (cos_y * y) + if total > 1: + centered[:, 0] += (index - (total - 1) / 2.0) * options.multiview_lateral_step + centered[:, 2] += 0.018 * math.sin((index / max(total - 1, 1)) * math.pi) + return centered.astype(np.float32) + + +def camera_view_record(path: Path, index: int, yaw_degrees: float) -> dict[str, Any]: + yaw = math.radians(yaw_degrees) + radius = 1.6 + return { + "index": index, + "image": path.as_posix(), + "approx_yaw_degrees": yaw_degrees, + "camera_to_world": [ + [math.cos(yaw), -math.sin(yaw), 0.0, radius * math.sin(yaw)], + [math.sin(yaw), math.cos(yaw), 0.0, -radius * math.cos(yaw)], + [0.0, 0.0, 1.0, 0.55], + [0.0, 0.0, 0.0, 1.0], + ], + } + + +def resize_rgb_nearest(rgb: np.ndarray, shape: tuple[int, int]) -> np.ndarray: + h, w = shape + ys = np.linspace(0, rgb.shape[0] - 1, h).astype(np.int32) + xs = np.linspace(0, rgb.shape[1] - 1, w).astype(np.int32) + return rgb[ys[:, None], xs[None, :], :].astype(np.float32) + + +def deterministic_point_sample(points: np.ndarray, max_points: int) -> np.ndarray: + scaled = np.round(points.astype(np.float64) * 100000.0).astype(np.int64) + ranks = ( + scaled[:, 0].astype(np.uint64) * np.uint64(73856093) + ^ scaled[:, 1].astype(np.uint64) * np.uint64(19349663) + ^ scaled[:, 2].astype(np.uint64) * np.uint64(83492791) + ) + choice = np.argpartition(ranks, max_points - 1)[:max_points] + return choice[np.argsort(ranks[choice])] + + +def load_stage_component_records(context: dict[str, Any], shape: tuple[int, int]) -> list[tuple[str, str, np.ndarray]]: + instances_json = Path(context["instances_json"]) + if not instances_json.exists(): + return [] + records = [] + for instance in load_instances(instances_json): + if instance.mask_path is None or not instance.mask_path.exists(): + continue + mask = read_gray(instance.mask_path) > 0.5 + if mask.shape != shape: + continue + records.append((safe_name(instance.instance_id), instance.label, mask)) + return records + + +def refine_component_records( + records: list[tuple[str, str, np.ndarray]], + options: BuiltinOptions, + shape: tuple[int, int], + *, + min_area_ratio: float | None = None, +) -> list[tuple[str, str, np.ndarray]]: + h, w = shape + area_ratio = options.min_component_area_ratio if min_area_ratio is None else min_area_ratio + min_area = max(16, int(h * w * area_ratio)) + occupied = np.zeros(shape, dtype=bool) + refined: list[tuple[str, str, np.ndarray]] = [] + for instance_id, label, raw_mask in records: + mask = np.asarray(raw_mask, dtype=bool) + if mask.shape != shape: + continue + if options.mask_close_passes: + mask = erode(dilate(mask, options.mask_close_passes), options.mask_close_passes) + if options.mask_dilate_passes: + mask = dilate(mask, options.mask_dilate_passes) + area = int(mask.sum()) + if area < min_area: + continue + overlap = int((mask & occupied).sum()) / float(area) + if overlap > options.max_mask_overlap: + continue + mask = mask & ~occupied + if int(mask.sum()) < min_area: + continue + refined.append((instance_id, label, mask)) + occupied |= mask + if len(refined) >= options.max_objects: + break + return refined + + +def prompt_target_object_count(prompt: str) -> int | None: + lowered = prompt.lower() + bottle_terms = ("bottle", "bottles", "瓶", "水瓶", "葫芦") + if not any(term in lowered for term in bottle_terms): + return None + if any(term in lowered for term in ("two", "2", "pair", "both", "两个", "两", "一对")): + return 2 + return None + + +def select_tabletop_subject_records( + records: list[tuple[str, str, np.ndarray]], + shape: tuple[int, int], + target_count: int, + *, + rgb: np.ndarray | None = None, +) -> list[tuple[str, str, np.ndarray]]: + h, w = shape + scored: list[tuple[float, tuple[str, str, np.ndarray]]] = [] + for record in records: + _instance_id, _label, mask = record + x0, y0, x1, y1 = mask_bbox(mask) + width = max(x1 - x0, 0) + height = max(y1 - y0, 0) + area = int(mask.sum()) + if area <= 0: + continue + bottom_ratio = y1 / max(h, 1) + height_ratio = height / max(h, 1) + width_ratio = width / max(w, 1) + if bottom_ratio < 0.58: + continue + if height_ratio < 0.18 or width_ratio < 0.05: + continue + if y0 / max(h, 1) < 0.12 and bottom_ratio < 0.62: + continue + compactness = area / max(width * height, 1) + support_bonus = max(0.0, bottom_ratio - 0.58) * h * w * 0.08 + upright_bonus = min(height_ratio / max(width_ratio, 1e-4), 3.0) * h * w * 0.004 + score = area + compactness * width * height * 0.35 + support_bonus + upright_bonus + scored.append((score, record)) + + if len(scored) < target_count: + return records + + scored.sort(key=lambda item: item[0], reverse=True) + selected: list[tuple[str, str, np.ndarray]] = [] + occupied = np.zeros(shape, dtype=bool) + for _score, (instance_id, label, mask) in scored: + area = int(mask.sum()) + overlap = int((mask & occupied).sum()) / max(area, 1) + if overlap > 0.45: + continue + selected.append((instance_id, label, mask)) + occupied |= mask + if len(selected) >= target_count: + break + if len(selected) < target_count: + return records + selected.sort(key=lambda record: mask_bbox(record[2])[0]) + renamed = [] + for idx, (_instance_id, label, mask) in enumerate(selected): + semantic = "water_bottle" if idx == 0 else "gourd_bottle" if target_count == 2 else label + completed = complete_tabletop_subject_mask(mask, shape, semantic, rgb=rgb) + renamed.append((f"{idx:02d}_{semantic}", semantic, completed)) + return renamed + + +def complete_tabletop_subject_mask( + mask: np.ndarray, + shape: tuple[int, int], + semantic: str, + *, + rgb: np.ndarray | None = None, +) -> np.ndarray: + if semantic == "gourd_bottle": + return erode(dilate(mask.astype(bool), 3), 1) + if semantic == "water_bottle" and rgb is not None: + edge_mask = complete_transparent_bottle_mask_from_edges(rgb, shape) + if edge_mask is not None: + return edge_mask + + h, w = shape + x0, y0, x1, y1 = mask_bbox(mask) + width = max(x1 - x0, 1) + height = max(y1 - y0, 1) + margin_x = int(max(width * 0.28, w * 0.045)) + margin_top = int(max(height * 0.20, h * 0.035)) + margin_bottom = int(max(height * 0.30, h * 0.055)) + bx0 = max(0, x0 - margin_x) + bx1 = min(w, x1 + margin_x) + by0 = max(0, y0 - margin_top) + by1 = min(h, y1 + margin_bottom) + completed = np.zeros(shape, dtype=bool) + completed[by0:by1, bx0:bx1] = True + return erode(dilate(completed, 1), 1) + + +def complete_transparent_bottle_mask_from_edges(rgb: np.ndarray, shape: tuple[int, int]) -> np.ndarray | None: + bbox = transparent_bottle_bbox_from_edges(rgb, shape) + if bbox is None: + return None + left, top, right, bottom = bbox + completed = np.zeros(shape, dtype=bool) + completed[top:bottom, left:right] = True + return erode(dilate(completed, 1), 1) + + +def transparent_bottle_bbox_from_edges(rgb: np.ndarray, shape: tuple[int, int]) -> tuple[int, int, int, int] | None: + h, w = shape + gray = rgb @ np.array([0.299, 0.587, 0.114], dtype=np.float32) + gx = np.zeros_like(gray) + gy = np.zeros_like(gray) + gx[:, 1:-1] = np.abs(gray[:, 2:] - gray[:, :-2]) + gy[1:-1, :] = np.abs(gray[2:, :] - gray[:-2, :]) + saturation = rgb.max(axis=2) - rgb.min(axis=2) + saliency = gx + gy + saturation * 0.35 + y0 = int(h * 0.08) + y1 = int(h * 0.96) + x0 = 0 + x1 = int(w * 0.55) + roi = saliency[y0:y1, x0:x1] + if roi.size == 0: + return None + column_score = roi.sum(axis=0) + if column_score.size < 8: + return None + column_score = np.convolve(column_score, np.ones(9, dtype=np.float32) / 9.0, mode="same") + threshold = float(np.percentile(column_score, 70.0)) + active = np.where(column_score > threshold)[0] + if active.size == 0: + return None + spans: list[tuple[int, int]] = [] + start = int(active[0]) + prev = int(active[0]) + for value in active[1:].tolist(): + if value == prev + 1: + prev = int(value) + continue + spans.append((start, prev)) + start = prev = int(value) + spans.append((start, prev)) + span = max(spans, key=lambda item: (item[1] - item[0] + 1, column_score[item[0] : item[1] + 1].sum())) + left = max(0, x0 + span[0] - int(w * 0.07)) + right = min(w, x0 + span[1] + 1 + int(w * 0.18)) + top = max(0, int(h * 0.08)) + bottom = min(h, int(h * 0.98)) + if right - left < int(w * 0.16): + return None + return left, top, right, bottom + + +def write_builtin_detection_artifacts(component_records: list[tuple[str, str, np.ndarray]], context: dict[str, Any]) -> None: + detections = [] + instances = [] + masks_dir = Path(context["masks_dir"]) + masks_dir.mkdir(parents=True, exist_ok=True) + instances_json = Path(context["instances_json"]) + for instance_id, label, mask in component_records: + bbox = mask_bbox(mask) + mask_path = masks_dir / f"{safe_name(instance_id)}.pgm" + write_gray(mask_path, mask.astype(np.float32)) + detections.append({"id": instance_id, "label": label, "score": 1.0, "bbox_xyxy": list(bbox)}) + instances.append( + { + "id": instance_id, + "label": label, + "score": 1.0, + "bbox_xyxy": list(bbox), + "mask_path": mask_path.relative_to(instances_json.parent).as_posix(), + } + ) + + write_json(Path(context["detections_json"]), {"detections": detections}) + write_json(instances_json, {"instances": instances}) + + +def safe_name(value: str) -> str: + return "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in value).strip("_").lower() + + +def estimate_pseudo_depth(rgb: np.ndarray) -> np.ndarray: + h, w = rgb.shape[:2] + yy = np.linspace(0.0, 1.0, h, dtype=np.float32)[:, None] + luminance = rgb @ np.array([0.299, 0.587, 0.114], dtype=np.float32) + saturation = rgb.max(axis=2) - rgb.min(axis=2) + depth = 1.0 + 1.2 * yy + 0.25 * (1.0 - luminance) - 0.12 * saturation + return smooth(depth.astype(np.float32), passes=4) + + +def fuse_depth_from_artifact(base_depth: np.ndarray, artifact: Path, shape: tuple[int, int], *, weight: float) -> np.ndarray: + if weight <= 0.0 or not artifact.exists(): + return base_depth + external = read_gray(artifact) + if external.shape != shape: + return base_depth + + base_norm = normalize(base_depth) + external_norm = normalize(smooth(external, passes=1)) + if _correlation(1.0 - external_norm, base_norm) > _correlation(external_norm, base_norm): + external_norm = 1.0 - external_norm + + lo = float(np.percentile(base_depth, 2.0)) + hi = float(np.percentile(base_depth, 98.0)) + if hi <= lo: + return base_depth + external_depth = lo + external_norm * (hi - lo) + return smooth(((1.0 - weight) * base_depth + weight * external_depth).astype(np.float32), passes=1) + + +def estimate_foreground_mask(rgb: np.ndarray) -> np.ndarray: + h, w = rgb.shape[:2] + border = np.concatenate([rgb[:4].reshape(-1, 3), rgb[-4:].reshape(-1, 3), rgb[:, :4].reshape(-1, 3), rgb[:, -4:].reshape(-1, 3)]) + bg = np.median(border, axis=0) + color_dist = np.linalg.norm(rgb - bg[None, None, :], axis=2) + saturation = rgb.max(axis=2) - rgb.min(axis=2) + threshold = max(0.10, float(np.percentile(color_dist, 72))) + mask = (color_dist > threshold) | (saturation > 0.22) + margin_y = max(2, h // 60) + margin_x = max(2, w // 60) + mask[:margin_y, :] = False + mask[-margin_y:, :] = False + mask[:, :margin_x] = False + mask[:, -margin_x:] = False + mask = dilate(erode(mask, 1), 2) + return mask + + +def select_components(mask: np.ndarray, max_objects: int, min_area_ratio: float) -> list[np.ndarray]: + h, w = mask.shape + min_area = max(16, int(h * w * min_area_ratio)) + comps = connected_components(mask) + comps = [comp for comp in comps if int(comp.sum()) >= min_area] + comps.sort(key=lambda comp: int(comp.sum()), reverse=True) + return comps[:max_objects] + + +def connected_components(mask: np.ndarray) -> list[np.ndarray]: + h, w = mask.shape + visited = np.zeros_like(mask, dtype=bool) + components: list[np.ndarray] = [] + ys, xs = np.where(mask) + for start_y, start_x in zip(ys.tolist(), xs.tolist()): + if visited[start_y, start_x] or not mask[start_y, start_x]: + continue + comp_coords = [] + stack = [(start_y, start_x)] + visited[start_y, start_x] = True + while stack: + y, x = stack.pop() + comp_coords.append((y, x)) + for ny, nx in ((y - 1, x), (y + 1, x), (y, x - 1), (y, x + 1)): + if 0 <= ny < h and 0 <= nx < w and mask[ny, nx] and not visited[ny, nx]: + visited[ny, nx] = True + stack.append((ny, nx)) + comp = np.zeros_like(mask, dtype=bool) + cy, cx = zip(*comp_coords) + comp[np.array(cy), np.array(cx)] = True + components.append(comp) + return components + + +def central_component(height: int, width: int) -> np.ndarray: + mask = np.zeros((height, width), dtype=bool) + y0, y1 = int(height * 0.30), int(height * 0.72) + x0, x1 = int(width * 0.30), int(width * 0.70) + mask[y0:y1, x0:x1] = True + return mask + + +def mask_bbox(mask: np.ndarray) -> tuple[int, int, int, int]: + ys, xs = np.where(mask) + if ys.size == 0: + return (0, 0, 0, 0) + return (int(xs.min()), int(ys.min()), int(xs.max() + 1), int(ys.max() + 1)) + + +def inpaint_with_border_color(rgb: np.ndarray, mask: np.ndarray) -> np.ndarray: + unknown = dilate(mask, 3) + out = rgb.copy() + known = ~unknown + if not known.any(): + return smooth_rgb(out, passes=2) + fallback = np.median(rgb[known], axis=0).astype(np.float32) + h, w = mask.shape + for y in range(h): + row_unknown = unknown[y] + if not row_unknown.any(): + continue + row_known = ~row_unknown + known_x = np.where(row_known)[0] + if known_x.size == 0: + out[y, row_unknown] = fallback + continue + missing_x = np.where(row_unknown)[0] + for channel in range(3): + out[y, missing_x, channel] = np.interp( + missing_x.astype(np.float32), + known_x.astype(np.float32), + out[y, known_x, channel], + ) + unknown_after_rows = np.isnan(out).any(axis=2) + if unknown_after_rows.any(): + out[unknown_after_rows] = fallback + return smooth_rgb(out, passes=2) + + +def enhance_colors(rgb: np.ndarray, strength: float) -> np.ndarray: + if strength <= 0.0: + return rgb + flat = rgb.reshape(-1, 3) + lo = np.percentile(flat, 1.0, axis=0) + hi = np.percentile(flat, 99.0, axis=0) + span = np.maximum(hi - lo, 1e-4) + stretched = np.clip((rgb - lo[None, None, :]) / span[None, None, :], 0.0, 1.0) + blended = (1.0 - strength) * rgb + strength * stretched + + luma = (blended @ np.array([0.299, 0.587, 0.114], dtype=np.float32))[:, :, None] + saturated = luma + (blended - luma) * (1.0 + 0.45 * strength) + blur = smooth_rgb(saturated, passes=1) + sharpened = saturated + (saturated - blur) * (0.55 * strength) + return np.clip(sharpened, 0.0, 1.0).astype(np.float32) + + +def backproject( + rgb: np.ndarray, + depth: np.ndarray, + intrinsics: dict[str, float], + *, + mask: np.ndarray, + max_points: int, +) -> tuple[np.ndarray, np.ndarray]: + ys, xs = np.where(mask) + if ys.size == 0: + return np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.float32) + if ys.size > max_points: + ys, xs = sample_grid_indices(ys, xs, max_points) + z = depth[ys, xs] + fx = float(intrinsics["fx"]) + fy = float(intrinsics["fy"]) + cx = float(intrinsics["cx"]) + cy = float(intrinsics["cy"]) + xw = (xs.astype(np.float32) - cx) / fx * z + yw = z + zw = (cy - ys.astype(np.float32)) / fy * z + 0.55 + points = np.stack([xw, yw, zw], axis=1).astype(np.float32) + colors = rgb[ys, xs, :3].astype(np.float32) + return points, colors + + +def object_billboard_points( + rgb: np.ndarray, + mask: np.ndarray, + extent: np.ndarray, + *, + label: str = "", + max_points: int, +) -> tuple[np.ndarray, np.ndarray]: + semantic = label.lower() + raw_ys, raw_xs = np.where(mask) + if raw_ys.size == 0: + return np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.float32) + if "water" in semantic: + x0, y0, x1, y1 = transparent_bottle_bbox_from_edges(rgb, mask.shape) or mask_bbox(mask) + else: + x0, y0, x1, y1 = mask_bbox(mask) + width = max(float(x1 - x0), 1.0) + height = max(float(y1 - y0), 1.0) + if "water" in semantic or "gourd" in semantic: + yy, xx = np.mgrid[y0:y1, x0:x1] + ys = yy.reshape(-1) + xs = xx.reshape(-1) + else: + ys, xs = raw_ys, raw_xs + if ys.size > max_points: + ys, xs = sample_grid_indices(ys, xs, max_points) + image_height = max(float(mask.shape[0]), 1.0) + target_height = max(min((height / image_height) * 1.18, 1.15), 0.24) + target_width = max(target_height * width / height, 0.12) + if "water" in semantic: + target_height = max(target_height, 1.02) + target_width = max(target_width, 0.34) + elif "gourd" in semantic: + target_height = min(max(target_height, 0.62), 0.82) + target_width = max(target_width, 0.42) + x = ((xs.astype(np.float32) - (x0 + width * 0.5)) / width) * target_width + z = (((y0 + height * 0.5) - ys.astype(np.float32)) / height) * target_height + y = np.zeros_like(x, dtype=np.float32) + points = np.stack([x, y, z], axis=1).astype(np.float32) + colors = rgb[ys, xs, :3].astype(np.float32) + return points, colors + + +def sample_mask_indices(ys: np.ndarray, xs: np.ndarray, max_points: int) -> tuple[np.ndarray, np.ndarray]: + yy = ys.astype(np.uint64) + np.uint64(1) + xx = xs.astype(np.uint64) + np.uint64(1) + ranks = (yy * np.uint64(73856093)) ^ (xx * np.uint64(19349663)) + choice = np.argpartition(ranks, max_points - 1)[:max_points] + order = np.lexsort((xs[choice], ys[choice])) + idx = choice[order] + return ys[idx], xs[idx] + + +def sample_grid_indices(ys: np.ndarray, xs: np.ndarray, max_points: int) -> tuple[np.ndarray, np.ndarray]: + if ys.size <= max_points: + return ys, xs + order = np.lexsort((xs, ys)) + ordered_y = ys[order] + ordered_x = xs[order] + idx = np.linspace(0, ys.size - 1, max_points, dtype=np.int64) + return ordered_y[idx], ordered_x[idx] + + +def write_3dgs_ply( + path: Path, + points: np.ndarray, + colors: np.ndarray, + *, + point_scale: float = 0.012, + opacity_logit: float = 4.0, +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + c0 = 0.28209479177387814 + sh = (np.clip(colors, 0.0, 1.0) - 0.5) / c0 + scale = math.log(max(point_scale, 1e-5)) + with path.open("w", encoding="ascii") as f: + f.write("ply\nformat ascii 1.0\n") + f.write(f"element vertex {len(points)}\n") + for name in ("x", "y", "z", "nx", "ny", "nz", "f_dc_0", "f_dc_1", "f_dc_2"): + f.write(f"property float {name}\n") + for idx in range(45): + f.write(f"property float f_rest_{idx}\n") + for name in ("opacity", "scale_0", "scale_1", "scale_2", "rot_0", "rot_1", "rot_2", "rot_3"): + f.write(f"property float {name}\n") + f.write("end_header\n") + zeros_rest = " ".join(["0"] * 45) + for p, c in zip(points, sh): + f.write( + f"{p[0]:.7g} {p[1]:.7g} {p[2]:.7g} 0 0 0 " + f"{c[0]:.7g} {c[1]:.7g} {c[2]:.7g} {zeros_rest} " + f"{opacity_logit:.7g} {scale:.7g} {scale:.7g} {scale:.7g} 1 0 0 0\n" + ) + + +def write_depth_mesh_obj(path: Path, rgb: np.ndarray, depth: np.ndarray, intrinsics: dict[str, float], *, grid: int) -> None: + h, w = depth.shape + step_y = max(1, h // grid) + step_x = max(1, w // grid) + ys = np.arange(0, h, step_y) + xs = np.arange(0, w, step_x) + yy, xx = np.meshgrid(ys, xs, indexing="ij") + z = depth[yy, xx].astype(np.float32) + fx = float(intrinsics["fx"]) + fy = float(intrinsics["fy"]) + cx = float(intrinsics["cx"]) + cy = float(intrinsics["cy"]) + xw = (xx.astype(np.float32) - cx) / fx * z + yw = z + zw = (cy - yy.astype(np.float32)) / fy * z + 0.55 + points = np.stack([xw, yw, zw], axis=2).reshape(-1, 3) + vertex_colors = rgb[yy, xx, :3].reshape(-1, 3) + rows, cols = len(ys), len(xs) + path.parent.mkdir(parents=True, exist_ok=True) + material = path.with_suffix(".mtl") + face_records: list[tuple[int, int, int, str]] = [] + materials: dict[str, tuple[float, float, float]] = {} + for r in range(rows - 1): + for c in range(cols - 1): + a = r * cols + c + 1 + b = a + 1 + cidx = a + cols + d = cidx + 1 + color = vertex_colors[[a - 1, b - 1, cidx - 1, d - 1]].mean(axis=0) + material_name, quantized = _quantized_scene_material(path.stem, color) + materials.setdefault(material_name, quantized) + face_records.append((a, b, d, material_name)) + face_records.append((a, d, cidx, material_name)) + material.write_text( + _scene_mesh_mtl(materials), + encoding="ascii", + ) + with path.open("w", encoding="ascii") as f: + f.write(f"mtllib {material.name}\n") + f.write("o real2sim_scene_depth_mesh\n") + for p, color in zip(points, vertex_colors): + f.write(f"v {p[0]:.7g} {p[1]:.7g} {p[2]:.7g} {color[0]:.7g} {color[1]:.7g} {color[2]:.7g}\n") + for y in ys: + v = 1.0 - (float(y) / max(float(h - 1), 1.0)) + for x in xs: + u = float(x) / max(float(w - 1), 1.0) + f.write(f"vt {u:.7g} {v:.7g}\n") + current_material = None + for a, b, cidx, material_name in face_records: + if material_name != current_material: + f.write(f"usemtl {material_name}\n") + current_material = material_name + f.write(f"f {a}/{a} {b}/{b} {cidx}/{cidx}\n") + + +def _quantized_scene_material(stem: str, color: np.ndarray, *, levels: int = 18) -> tuple[str, tuple[float, float, float]]: + quantized = tuple(float(round(float(channel) * (levels - 1)) / (levels - 1)) for channel in np.clip(color, 0.0, 1.0)) + suffix = "_".join(f"{int(round(channel * 255)):02x}" for channel in quantized) + return f"{safe_name(stem)}_mat_{suffix}", quantized + + +def _scene_mesh_mtl(materials: dict[str, tuple[float, float, float]]) -> str: + lines = ["# Generated Real2Sim scene mesh material palette\n"] + for name, color in sorted(materials.items()): + r, g, b = color + lines.extend( + [ + f"newmtl {name}\n", + f"Ka {r:.6f} {g:.6f} {b:.6f}\n", + f"Kd {r:.6f} {g:.6f} {b:.6f}\n", + "Ks 0.030000 0.030000 0.030000\n", + "Ns 8.000000\n", + "d 1.000000\n", + "illum 2\n", + "\n", + ] + ) + return "".join(lines) + + +def write_box_obj(path: Path, extent: np.ndarray) -> None: + half = np.maximum(extent / 2.0, np.array([0.02, 0.02, 0.02], dtype=np.float32)) + vertices = [ + (-half[0], -half[1], -half[2]), + (half[0], -half[1], -half[2]), + (half[0], half[1], -half[2]), + (-half[0], half[1], -half[2]), + (-half[0], -half[1], half[2]), + (half[0], -half[1], half[2]), + (half[0], half[1], half[2]), + (-half[0], half[1], half[2]), + ] + faces = [(1, 2, 3), (1, 3, 4), (5, 8, 7), (5, 7, 6), (1, 5, 6), (1, 6, 2), (2, 6, 7), (2, 7, 3), (3, 7, 8), (3, 8, 4), (4, 8, 5), (4, 5, 1)] + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="ascii") as f: + f.write("o real2sim_object_box\n") + for v in vertices: + f.write(f"v {v[0]:.7g} {v[1]:.7g} {v[2]:.7g}\n") + for face in faces: + f.write(f"f {face[0]} {face[1]} {face[2]}\n") + + +def smooth(arr: np.ndarray, passes: int) -> np.ndarray: + out = arr.astype(np.float32) + for _ in range(passes): + padded = np.pad(out, 1, mode="edge") + out = ( + padded[1:-1, 1:-1] + + padded[:-2, 1:-1] + + padded[2:, 1:-1] + + padded[1:-1, :-2] + + padded[1:-1, 2:] + ) / 5.0 + return out + + +def smooth_rgb(arr: np.ndarray, passes: int) -> np.ndarray: + channels = [smooth(arr[:, :, idx], passes) for idx in range(3)] + return np.stack(channels, axis=2) + + +def erode(mask: np.ndarray, passes: int) -> np.ndarray: + out = mask.copy() + for _ in range(passes): + padded = np.pad(out, 1, mode="constant", constant_values=False) + out = ( + padded[1:-1, 1:-1] + & padded[:-2, 1:-1] + & padded[2:, 1:-1] + & padded[1:-1, :-2] + & padded[1:-1, 2:] + ) + return out + + +def dilate(mask: np.ndarray, passes: int) -> np.ndarray: + out = mask.copy() + for _ in range(passes): + padded = np.pad(out, 1, mode="constant", constant_values=False) + out = ( + padded[1:-1, 1:-1] + | padded[:-2, 1:-1] + | padded[2:, 1:-1] + | padded[1:-1, :-2] + | padded[1:-1, 2:] + ) + return out + + +def normalize(arr: np.ndarray) -> np.ndarray: + min_val = float(np.min(arr)) + max_val = float(np.max(arr)) + if max_val <= min_val: + return np.zeros_like(arr, dtype=np.float32) + return ((arr - min_val) / (max_val - min_val)).astype(np.float32) + + +def _correlation(a: np.ndarray, b: np.ndarray) -> float: + aa = a.astype(np.float32) - float(np.mean(a)) + bb = b.astype(np.float32) - float(np.mean(b)) + denom = float(np.sqrt(np.mean(aa * aa) * np.mean(bb * bb))) + if denom <= 1e-8: + return 0.0 + return float(np.mean(aa * bb) / denom) + + +def _clamp01(value: float) -> float: + return min(1.0, max(0.0, value)) diff --git a/vendor/gs_playground/real2sim/camera.py b/vendor/gs_playground/real2sim/camera.py new file mode 100644 index 0000000..378bd58 --- /dev/null +++ b/vendor/gs_playground/real2sim/camera.py @@ -0,0 +1,72 @@ +"""Camera utilities shared by Real2Sim packaging and viewers.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from .errors import StageArtifactError + + +@dataclass(frozen=True) +class SourceCamera: + position: np.ndarray + xmat: np.ndarray + fovy_degrees: float + width: int + height: int + y_up: bool = False + + +def load_anysplat_source_camera(path: Path) -> SourceCamera: + with path.open("r", encoding="utf-8") as f: + payload = json.load(f) + if not isinstance(payload, dict): + raise StageArtifactError(f"camera JSON must be an object: {path}") + + intrinsics = payload.get("intrinsics") + if not isinstance(intrinsics, dict): + raise StageArtifactError(f"camera JSON missing intrinsics: {path}") + for key in ("width", "height", "fy"): + if key not in intrinsics: + raise StageArtifactError(f"camera JSON intrinsics missing {key}: {path}") + + extrinsic_value: Any = payload.get("pred_context_pose", {}).get("extrinsic") + if extrinsic_value is None: + extrinsic_value = payload.get("extrinsic") + extrinsic = _first_matrix(extrinsic_value, rows=4, cols=4, path=path) + c2w = np.linalg.inv(extrinsic).astype(np.float32) + + width = int(intrinsics["width"]) + height = int(intrinsics["height"]) + fy = float(intrinsics["fy"]) + if width <= 0 or height <= 0 or fy <= 0: + raise StageArtifactError(f"camera JSON has invalid dimensions or focal length: {path}") + + fovy = math.degrees(2.0 * math.atan(height / (2.0 * fy))) + return SourceCamera( + position=c2w[:3, 3].astype(np.float32), + xmat=c2w[:3, :3].reshape(9).astype(np.float32), + fovy_degrees=fovy, + width=width, + height=height, + y_up=False, + ) + + +def _first_matrix(value: Any, *, rows: int, cols: int, path: Path) -> np.ndarray: + if value is None: + raise StageArtifactError(f"camera JSON missing extrinsic matrix: {path}") + arr = np.asarray(value, dtype=np.float32) + while arr.ndim > 2 and arr.shape[0] == 1: + arr = arr[0] + if arr.ndim == 3 and arr.shape[1:] == (rows, cols): + arr = arr[0] + if arr.shape != (rows, cols): + raise StageArtifactError(f"camera JSON extrinsic must have shape {(rows, cols)}, got {arr.shape}: {path}") + return arr diff --git a/vendor/gs_playground/real2sim/cli.py b/vendor/gs_playground/real2sim/cli.py new file mode 100644 index 0000000..4e5cd97 --- /dev/null +++ b/vendor/gs_playground/real2sim/cli.py @@ -0,0 +1,489 @@ +"""Command line interface for GS-Playground Real2Sim.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from .contracts import write_json +from .paper_strict import check_paper_strict, format_report, validate_paper_strict +from .pipeline import Real2SimPipeline +from .pruning import PruneOptions + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m real2sim", description="GS-Playground Real2Sim pipeline") + subparsers = parser.add_subparsers(dest="command", required=True) + + run_parser = subparsers.add_parser("run", help="Run the full image/video-to-sim-ready-scene pipeline") + run_parser.add_argument("--input", type=Path, required=True, help="Input RGB image or video") + run_parser.add_argument("--output", type=Path, required=True, help="Output package directory") + run_parser.add_argument("--work-dir", type=Path, default=None, help="Intermediate artifact directory") + run_parser.add_argument("--prompt", default="all task-relevant objects", help="Open-vocabulary detection prompt") + run_parser.add_argument("--config", type=Path, default=None, help="Real2Sim stage command config JSON") + run_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"], help="Demo robot config to emit") + run_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"], help="Real2Sim backend") + run_parser.add_argument("--dynamic-objects", action="store_true", help="Emit free joints for reconstructed objects") + run_parser.add_argument("--resume", action="store_true", help="Reuse existing work-dir artifacts") + run_parser.add_argument("--dry-run", action="store_true", help="Print/validate configured commands without requiring outputs") + run_parser.add_argument("--paper-strict", action="store_true", help="Require exact paper-equivalent external stages and runtime prerequisites") + run_parser.add_argument("--synthetic-multiview", action="store_true", help="Expand a single input image into synthetic multiview frames before scene reconstruction") + run_parser.add_argument( + "--synthetic-multiview-mode", + default="object", + choices=["object", "scene"], + help="Prompt style for synthetic multiview expansion: object turntable or scene completion", + ) + run_parser.add_argument( + "--pipeline-mode", + default="full", + choices=["full", "scene-only", "object-only"], + help="Pipeline scope: full (scene+objects), scene-only, or object-only", + ) + run_parser.add_argument("--skip-scene-reconstruction", action="store_true", help="Skip scene reconstruction in an existing work-dir run") + run_parser.add_argument( + "--skip-object-reconstruction", + action="store_true", + help="Skip object reconstruction and keep only the reconstructed scene", + ) + run_parser.add_argument("--no-prune", action="store_true", help="Skip 3DGS pruning") + run_parser.add_argument("--keep-ratio", type=float, default=None, help="Pruning keep ratio") + run_parser.set_defaults(func=_run) + + scene_parser = subparsers.add_parser("generate-scene", help="Generate a scene-only Real2Sim package") + scene_parser.add_argument("--input", type=Path, required=True, help="Input RGB image, image directory, or video") + scene_parser.add_argument("--output", type=Path, required=True, help="Output scene package directory") + scene_parser.add_argument("--work-dir", type=Path, default=None, help="Intermediate artifact directory") + scene_parser.add_argument("--prompt", default="static navigable scene", help="Scene reconstruction prompt/context") + scene_parser.add_argument("--config", type=Path, default=None, help="Real2Sim stage command config JSON") + scene_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"], help="Demo robot config to emit") + scene_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"], help="Scene generation backend") + scene_parser.add_argument("--dynamic-objects", action="store_true", help=argparse.SUPPRESS) + scene_parser.add_argument("--resume", action="store_true", help="Reuse existing work-dir artifacts") + scene_parser.add_argument("--dry-run", action="store_true", help="Print/validate configured commands without requiring outputs") + scene_parser.add_argument("--paper-strict", action="store_true", help="Require exact paper-equivalent external scene stages") + scene_parser.add_argument("--synthetic-multiview", action="store_true", help="Expand a single input image into synthetic multiview frames before scene reconstruction") + scene_parser.add_argument( + "--scene-completion", + action="store_true", + help="Alias for scene-style synthetic multiview completion when a single image has insufficient scene evidence", + ) + scene_parser.add_argument("--no-prune", action="store_true", help="Skip 3DGS pruning") + scene_parser.add_argument("--keep-ratio", type=float, default=None, help="Pruning keep ratio") + scene_parser.set_defaults(func=_generate_scene) + + package_parser = subparsers.add_parser("package-existing", help="Package existing 3DGS/mesh assets as a GS-Playground scene") + package_parser.add_argument("--source", type=Path, required=True, help="Original capture or source path") + package_parser.add_argument("--scene-ply", type=Path, required=True, help="Scene/background 3DGS PLY") + package_parser.add_argument("--scene-mesh", type=Path, default=None, help="Optional scene mesh OBJ") + package_parser.add_argument("--objects-json", type=Path, default=None, help="Optional object asset list JSON") + package_parser.add_argument("--cameras-json", type=Path, default=None, help="Optional AnySplat camera intrinsics/extrinsics JSON") + package_parser.add_argument("--depth-map", type=Path, default=None, help="Optional monocular depth artifact used for alignment") + package_parser.add_argument("--background-image", type=Path, default=None, help="Optional inpainted background image artifact") + package_parser.add_argument("--preview-image", type=Path, default=None, help="Optional source-view 3DGS preview image artifact") + package_parser.add_argument("--mesh-cleanup-report", type=Path, default=None, help="Optional scene mesh cleanup report JSON") + package_parser.add_argument("--texture-bake-report", type=Path, default=None, help="Optional COLMAP texture bake report JSON") + package_parser.add_argument("--scene-structure-report", type=Path, default=None, help="Optional scene structure report JSON") + package_parser.add_argument("--output", type=Path, required=True, help="Output package directory") + package_parser.add_argument("--prompt", default="Navigate in the reconstructed Real2Sim scene") + package_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"]) + package_parser.add_argument("--dynamic-objects", action="store_true") + package_parser.set_defaults(func=_package_existing) + + check_parser = subparsers.add_parser("check", help="Check configured external model commands and local packages") + check_parser.add_argument("--config", type=Path, default=None) + check_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"]) + check_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"]) + check_parser.add_argument("--dynamic-objects", action="store_true") + check_parser.add_argument("--paper-strict", action="store_true", help="Validate exact paper-equivalent runtime prerequisites") + check_parser.set_defaults(func=_check) + + example_parser = subparsers.add_parser("write-example-config", help="Write a config template for external model commands") + example_parser.add_argument("--output", type=Path, required=True) + example_parser.set_defaults(func=_write_example_config) + + args = parser.parse_args(argv) + return int(args.func(args)) + + +def _pipeline(args) -> Real2SimPipeline: + return Real2SimPipeline.from_config_file( + repo_root=REPO_ROOT, + config_file=getattr(args, "config", None), + robot=args.robot, + dynamic_objects=args.dynamic_objects, + dry_run=getattr(args, "dry_run", False), + backend="external" if getattr(args, "paper_strict", False) else getattr(args, "backend", "auto"), + ) + + +def _run(args) -> int: + mode = str(getattr(args, "pipeline_mode", "full")) + skip_scene = bool(getattr(args, "skip_scene_reconstruction", False)) + skip_objects = bool(getattr(args, "skip_object_reconstruction", False)) + + if mode != "full" and (skip_scene or skip_objects): + raise SystemExit("cannot combine --pipeline-mode with --skip-scene-reconstruction/--skip-object-reconstruction") + + enable_scene_reconstruction = True + enable_object_reconstruction = True + if mode == "scene-only": + enable_scene_reconstruction = True + enable_object_reconstruction = False + elif mode == "object-only": + enable_scene_reconstruction = False + enable_object_reconstruction = True + else: + enable_scene_reconstruction = not skip_scene + enable_object_reconstruction = not skip_objects + + if not enable_scene_reconstruction and not enable_object_reconstruction: + raise SystemExit("both scene and object reconstruction are disabled") + + resume = args.resume + if mode == "object-only": + resume = True + + pipeline = _pipeline(args) + paper_strict = bool(getattr(args, "paper_strict", False)) + if paper_strict and (args.no_prune or args.keep_ratio is not None): + raise SystemExit("--paper-strict cannot be combined with --no-prune or --keep-ratio; use pruning.mode=speedy_splat") + if paper_strict: + report = check_paper_strict(pipeline.config, REPO_ROOT) + for line in format_report(report): + print(line) + if args.dry_run: + return 0 if report["ok"] else 1 + validate_paper_strict(pipeline.config, REPO_ROOT) + if args.dry_run: + status = pipeline.check() + required = {"grounding_dino", "sam2", "instance_refinement", "lama", "anysplat", "sam3d", "pose_alignment"} + if args.backend in {"auto", "builtin", "hybrid"} and status["builtin_ready"]: + print("builtin: ready (numpy)") + if args.backend == "builtin": + return 0 + for stage, info in status["stages"].items(): + state = "configured" if info["configured"] else "missing" + print(f"{stage}: {state} ({info['env']})") + external_ready = all(status["stages"][stage]["configured"] for stage in required) + return 0 if external_ready or (args.backend in {"auto", "hybrid"} and status["builtin_ready"]) else 1 + + prune = None + if args.no_prune or args.keep_ratio is not None: + keep_ratio = args.keep_ratio if args.keep_ratio is not None else 0.3 + prune = PruneOptions(enabled=not args.no_prune, keep_ratio=keep_ratio) + result = pipeline.run( + input_path=args.input, + output_dir=args.output, + work_dir=args.work_dir, + prompt=args.prompt, + resume=resume, + prune_options=prune, + enable_scene_reconstruction=enable_scene_reconstruction, + enable_object_reconstruction=enable_object_reconstruction, + synthetic_multiview=args.synthetic_multiview, + synthetic_multiview_mode=getattr(args, "synthetic_multiview_mode", "object"), + ) + print(f"Real2Sim package: {result.package.root}") + print(f"Scene XML: {result.package.scene_xml}") + print(f"Demo config: {result.package.config_json}") + print(f"Manifest: {result.package.manifest_json}") + return 0 + + +def _generate_scene(args) -> int: + pipeline = _pipeline(args) + paper_strict = bool(getattr(args, "paper_strict", False)) + if paper_strict and (args.no_prune or args.keep_ratio is not None): + raise SystemExit("--paper-strict cannot be combined with --no-prune or --keep-ratio; use pruning.mode=speedy_splat") + if paper_strict: + report = check_paper_strict(pipeline.config, REPO_ROOT) + for line in format_report(report): + print(line) + if args.dry_run: + return 0 if report["ok"] else 1 + validate_paper_strict(pipeline.config, REPO_ROOT) + if args.dry_run: + status = pipeline.check() + if args.backend in {"auto", "builtin", "hybrid"} and status["builtin_ready"]: + print("builtin: ready (numpy)") + return 0 if args.backend == "builtin" else 0 + required = {"grounding_dino", "sam2", "instance_refinement", "lama", "anysplat"} + for stage, info in status["stages"].items(): + state = "configured" if info["configured"] else "missing" + print(f"{stage}: {state} ({info['env']})") + return 0 if all(status["stages"][stage]["configured"] for stage in required) else 1 + + prune = None + if args.no_prune or args.keep_ratio is not None: + keep_ratio = args.keep_ratio if args.keep_ratio is not None else 0.3 + prune = PruneOptions(enabled=not args.no_prune, keep_ratio=keep_ratio) + result = pipeline.run( + input_path=args.input, + output_dir=args.output, + work_dir=args.work_dir, + prompt=args.prompt, + resume=args.resume, + prune_options=prune, + enable_scene_reconstruction=True, + enable_object_reconstruction=False, + synthetic_multiview=args.synthetic_multiview or getattr(args, "scene_completion", False), + synthetic_multiview_mode="scene", + ) + print(f"Real2Sim scene package: {result.package.root}") + print(f"Scene PLY: {result.manifest.scene.gaussian_ply}") + print(f"Scene XML: {result.package.scene_xml}") + print(f"Demo config: {result.package.config_json}") + print(f"Manifest: {result.package.manifest_json}") + return 0 + + +def _package_existing(args) -> int: + result = _pipeline(args).package_existing( + source=args.source, + scene_ply=args.scene_ply, + scene_mesh=args.scene_mesh, + objects_json=args.objects_json, + output_dir=args.output, + prompt=args.prompt, + cameras_json=args.cameras_json, + depth_map=args.depth_map, + background_image=args.background_image, + preview_image=args.preview_image, + mesh_cleanup_report=args.mesh_cleanup_report, + texture_bake_report=args.texture_bake_report, + scene_structure_report=args.scene_structure_report, + ) + print(f"Real2Sim package: {result.root}") + print(f"Scene XML: {result.scene_xml}") + print(f"Demo config: {result.config_json}") + print(f"Manifest: {result.manifest_json}") + return 0 + + +def _check(args) -> int: + status = _pipeline(args).check() + paper_strict = bool(getattr(args, "paper_strict", False)) + if paper_strict: + status["paper_strict"] = check_paper_strict(_pipeline(args).config, REPO_ROOT) + write_json(Path("/tmp/gsplayground_real2sim_check.json"), status) + for stage, info in status["stages"].items(): + state = "configured" if info["configured"] else "missing" + print(f"{stage}: {state} ({info['env']})") + for module, ok in status["python_packages"].items(): + prefix = "controller_python." if paper_strict else "" + print(f"{prefix}{module}: {'ok' if ok else 'missing'}") + if paper_strict: + for module in status["paper_strict"].get("python_modules", []): + stage = module.get("stage", "unknown") + name = module.get("name", "unknown") + state = "ok" if module.get("ok") else "missing" + print(f"paper_strict.{stage}.{name}: {state}") + for line in format_report(status["paper_strict"]): + print(line) + print("Wrote /tmp/gsplayground_real2sim_check.json") + return 0 if not paper_strict or status["paper_strict"]["ok"] else 1 + + +def _write_example_config(args) -> int: + write_json(args.output, example_config()) + print(f"Wrote {args.output}") + return 0 + + +def example_config() -> dict: + return { + "stages": { + "synthetic_multiview": { + "command": [ + "{python}", + "/path/to/synthetic_multiview_expand.py", + "--image", + "{image}", + "--output-dir", + "{synthetic_multiview_dir}", + "--frames-dir", + "{synthetic_frames_dir}", + "--prompt", + "{synthetic_multiview_prompt}", + "--size", + "1536x1024", + ] + }, + "grounding_dino": { + "command": [ + "{python}", + "/path/to/grounding_dino_detect.py", + "--image", + "{image}", + "--prompt", + "{prompt}", + "--output", + "{detections_json}", + ] + }, + "sam2": { + "command": [ + "{python}", + "/path/to/sam2_segment.py", + "--image", + "{image}", + "--detections", + "{detections_json}", + "--masks-dir", + "{masks_dir}", + "--output", + "{instances_json}", + ] + }, + "instance_refinement": { + "command": [ + "{python}", + "/path/to/instance_refine.py", + "--instances", + "{instances_json}", + "--output", + "{instances_json}", + ] + }, + "lama": { + "command": [ + "{python}", + "/path/to/lama_inpaint.py", + "--image", + "{image}", + "--instances", + "{instances_json}", + "--output", + "{background_image}", + "--prompt", + "{prompt}", + "--work-dir", + "{work_dir}", + "--redetect-command-env", + "REAL2SIM_GROUNDING_DINO_COMMAND", + "--segment-command-env", + "REAL2SIM_SAM2_COMMAND", + "--require-redetect-loop", + ] + }, + "depth_estimation": { + "command": [ + "{python}", + "/path/to/depth_anything_depth.py", + "--image", + "{background_image}", + "--output", + "{depth_map}", + "--model", + "depth-anything/Depth-Anything-V2-Small-hf", + ] + }, + "anysplat": { + "command": [ + "{python}", + "/path/to/anysplat_scene.py", + "--image", + "{background_image}", + "--frames-dir", + "{frames_dir}", + "--output-dir", + "{scene_reconstruction_dir}", + "--ply", + "{scene_ply}", + "--mesh", + "{scene_mesh}", + "--cameras", + "{cameras_json}", + "--depth", + "{depth_map}", + "--max-views", + "8", + ] + }, + "sam3d": { + "command": [ + "{python}", + "/path/to/sam3d_object.py", + "--image", + "{image}", + "--mask", + "{mask}", + "--instance", + "{instance_json}", + "--output-dir", + "{object_dir}", + "--ply", + "{object_ply}", + "--mesh", + "{object_mesh}", + "--pose", + "{object_pose}", + "--cameras", + "{cameras_json}", + "--match-camera-preprocess", + "--object-centric-preprocess", + "--object-preprocess-size", + "896", + "--object-crop-margin", + "0.12", + ] + }, + "pose_alignment": { + "command": [ + "{python}", + "/path/to/refine_pose_alignment.py", + "--assets", + "{aligned_assets_json}", + "--instances", + "{instances_json}", + "--depth", + "{depth_map}", + "--cameras", + "{cameras_json}", + "--work-dir", + "{work_dir}", + "--output", + "{aligned_assets_json}", + ] + }, + }, + "builtin": { + "max_scene_points": 180000, + "max_object_points": 50000, + "max_objects": 10, + "min_component_area_ratio": 0.003, + "stage_min_component_area_ratio": 0.0005, + "focal_scale": 0.9, + "external_depth_weight": 0.65, + "color_enhance_strength": 0.35, + "scene_point_scale": 0.018, + "object_point_scale": 0.014, + "opacity_logit": 4.5, + "mask_close_passes": 2, + "mask_dilate_passes": 1, + "max_mask_overlap": 0.6, + }, + "pruning": { + "enabled": True, + "keep_ratio": 0.85, + "mode": "speedy_splat", + "external_command": [ + "{python}", + "scripts/real2sim_external/speedy_splat_prune.py", + "--input", + "{input}", + "--output", + "{output}", + "--image", + "{source_image}", + "--cameras", + "{cameras_json}", + "--keep-ratio", + "{keep_ratio}", + ], + }, + } diff --git a/vendor/gs_playground/real2sim/commands.py b/vendor/gs_playground/real2sim/commands.py new file mode 100644 index 0000000..f374eff --- /dev/null +++ b/vendor/gs_playground/real2sim/commands.py @@ -0,0 +1,97 @@ +"""External command execution for Real2Sim stages. + +The paper pipeline relies on large model systems that are not vendored in this +preview repository. This module keeps those integrations explicit: each stage +declares the command it needs and the artifacts it must produce. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Union + +from .errors import MissingStageCommand, PipelineError + + +CommandValue = Union[str, Sequence[str]] + + +@dataclass(frozen=True) +class StageCommand: + name: str + env_var: str + description: str + required: bool = True + + +class _SafeFormatDict(dict[str, str]): + def __missing__(self, key: str) -> str: + raise KeyError(f"unknown command placeholder: {key}") + + +def _format_arg(value: str, context: Mapping[str, Any]) -> str: + flattened = {key: str(val) for key, val in context.items()} + return value.format_map(_SafeFormatDict(flattened)) + + +def resolve_command(stage: StageCommand, config: Mapping[str, Any]) -> CommandValue | None: + stages = config.get("stages", {}) + stage_config = stages.get(stage.name, {}) if isinstance(stages, Mapping) else {} + configured = stage_config.get("command") if isinstance(stage_config, Mapping) else None + env_command = os.environ.get(stage.env_var) + command = command_preferring_env(configured, env_command) + if command: + return command + + if stage.required: + raise MissingStageCommand( + f"{stage.name} is required but no command was configured. " + f"Set stages.{stage.name}.command in the Real2Sim config or export {stage.env_var}. " + f"Expected stage: {stage.description}" + ) + return None + + +def command_preferring_env(configured: CommandValue | None, env_command: str | None) -> CommandValue | None: + if env_command: + return env_command + return configured + + +def command_contains_placeholder(command: Any) -> bool: + if command is None: + return False + parts = command if isinstance(command, (list, tuple)) else [str(command)] + return any("/path/to/" in str(part) for part in parts) + + +def format_command(command: CommandValue, context: Mapping[str, Any]) -> list[str]: + if isinstance(command, str): + return [_format_arg(arg, context) for arg in shlex.split(command)] + return [_format_arg(str(arg), context) for arg in command] + + +def run_command( + stage: StageCommand, + command: CommandValue, + context: Mapping[str, Any], + *, + cwd: Path, + dry_run: bool = False, +) -> list[str]: + argv = format_command(command, context) + if not argv: + raise PipelineError(f"{stage.name} command is empty") + + if dry_run: + return argv + + completed = subprocess.run(argv, cwd=cwd, check=False) + if completed.returncode != 0: + raise PipelineError(f"{stage.name} command failed with exit code {completed.returncode}: {argv}") + return argv diff --git a/vendor/gs_playground/real2sim/contracts.py b/vendor/gs_playground/real2sim/contracts.py new file mode 100644 index 0000000..fbee7bb --- /dev/null +++ b/vendor/gs_playground/real2sim/contracts.py @@ -0,0 +1,185 @@ +"""Artifact contracts for Real2Sim stage boundaries.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from .errors import StageArtifactError + + +def read_json(path: Path) -> Any: + if not path.exists(): + raise StageArtifactError(f"missing JSON artifact: {path}") + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def write_json(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + + +def require_file(path: Path, description: str) -> Path: + if not path.is_file(): + raise StageArtifactError(f"missing {description}: {path}") + return path + + +def require_dir(path: Path, description: str) -> Path: + if not path.is_dir(): + raise StageArtifactError(f"missing {description}: {path}") + return path + + +@dataclass(frozen=True) +class Pose: + position: tuple[float, float, float] = (0.0, 0.0, 0.0) + quat_wxyz: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0) + scale: tuple[float, float, float] = (1.0, 1.0, 1.0) + + @classmethod + def from_json(cls, path: Path) -> "Pose": + data = read_json(path) + if not isinstance(data, Mapping): + raise StageArtifactError(f"pose artifact must be a JSON object: {path}") + + pos = data.get("position", data.get("translation", [0.0, 0.0, 0.0])) + quat = data.get("quat_wxyz", data.get("quaternion_wxyz")) + if quat is None and "quaternion_xyzw" in data: + qx, qy, qz, qw = _float_list(data["quaternion_xyzw"], 4, "quaternion_xyzw", path) + quat = [qw, qx, qy, qz] + if quat is None: + quat = [1.0, 0.0, 0.0, 0.0] + + scale_raw = data.get("scale", [1.0, 1.0, 1.0]) + if isinstance(scale_raw, (int, float)): + scale = (float(scale_raw), float(scale_raw), float(scale_raw)) + else: + scale = tuple(_float_list(scale_raw, 3, "scale", path)) + + return cls( + position=tuple(_float_list(pos, 3, "position", path)), + quat_wxyz=tuple(_float_list(quat, 4, "quat_wxyz", path)), + scale=scale, + ) + + def to_json(self) -> dict[str, list[float]]: + return { + "position": list(self.position), + "quat_wxyz": list(self.quat_wxyz), + "scale": list(self.scale), + } + + +@dataclass(frozen=True) +class Instance: + instance_id: str + label: str + score: float | None = None + bbox_xyxy: tuple[float, float, float, float] | None = None + mask_path: Path | None = None + + @classmethod + def from_mapping(cls, raw: Mapping[str, Any], base_dir: Path) -> "Instance": + raw_id = raw.get("id", raw.get("instance_id", raw.get("name"))) + label = raw.get("label", raw.get("class", raw_id)) + if raw_id is None or label is None: + raise StageArtifactError(f"instance entry requires id/name and label: {raw}") + bbox = raw.get("bbox_xyxy", raw.get("bbox")) + mask = raw.get("mask_path", raw.get("mask")) + return cls( + instance_id=str(raw_id), + label=str(label), + score=float(raw["score"]) if raw.get("score") is not None else None, + bbox_xyxy=tuple(_float_list(bbox, 4, "bbox", base_dir)) if bbox is not None else None, + mask_path=(base_dir / str(mask)).resolve() if mask else None, + ) + + +@dataclass(frozen=True) +class Real2SimAsset: + name: str + label: str + gaussian_ply: Path + mesh_obj: Path | None = None + pose: Pose = field(default_factory=Pose) + dynamic: bool = False + source_instance_id: str | None = None + + def to_json(self, root: Path) -> dict[str, Any]: + return { + "name": self.name, + "label": self.label, + "gaussian_ply": _relative_or_absolute(self.gaussian_ply, root), + "mesh_obj": _relative_or_absolute(self.mesh_obj, root) if self.mesh_obj else None, + "pose": self.pose.to_json(), + "dynamic": self.dynamic, + "source_instance_id": self.source_instance_id, + } + + +@dataclass(frozen=True) +class Real2SimManifest: + source: Path + prompt: str + scene: Real2SimAsset + objects: tuple[Real2SimAsset, ...] = () + camera_artifacts: dict[str, Path] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) + + def to_json(self, root: Path) -> dict[str, Any]: + return { + "source": _relative_or_absolute(self.source, root), + "prompt": self.prompt, + "scene": self.scene.to_json(root), + "objects": [obj.to_json(root) for obj in self.objects], + "camera_artifacts": { + key: _relative_or_absolute(value, root) + for key, value in sorted(self.camera_artifacts.items()) + }, + "metadata": self.metadata, + } + + +def load_instances(path: Path) -> list[Instance]: + data = read_json(path) + entries = data.get("instances", data) if isinstance(data, Mapping) else data + if not isinstance(entries, list): + raise StageArtifactError(f"instances artifact must be a list or object with instances list: {path}") + return [Instance.from_mapping(entry, path.parent) for entry in entries] + + +def validate_detections(path: Path) -> None: + data = read_json(path) + entries = data.get("detections", data) if isinstance(data, Mapping) else data + if not isinstance(entries, list): + raise StageArtifactError(f"detections artifact must be a list or object with detections list: {path}") + for entry in entries: + if not isinstance(entry, Mapping): + raise StageArtifactError(f"detection entries must be JSON objects: {path}") + if "bbox" not in entry and "bbox_xyxy" not in entry: + raise StageArtifactError(f"detection entry missing bbox/bbox_xyxy: {entry}") + + +def _float_list(value: Any, size: int, field_name: str, path: Path) -> list[float]: + if not isinstance(value, Iterable) or isinstance(value, (str, bytes)): + raise StageArtifactError(f"{field_name} must be a list with {size} values in {path}") + vals = [float(item) for item in value] + if len(vals) != size: + raise StageArtifactError(f"{field_name} must have {size} values in {path}, got {len(vals)}") + return vals + + +def _relative_or_absolute(path: Path | None, root: Path) -> str | None: + if path is None: + return None + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + return path.resolve().as_posix() diff --git a/vendor/gs_playground/real2sim/errors.py b/vendor/gs_playground/real2sim/errors.py new file mode 100644 index 0000000..5d091fc --- /dev/null +++ b/vendor/gs_playground/real2sim/errors.py @@ -0,0 +1,15 @@ +"""Pipeline-specific exceptions.""" + +from __future__ import annotations + + +class PipelineError(RuntimeError): + """Base class for Real2Sim pipeline failures.""" + + +class MissingStageCommand(PipelineError): + """Raised when a required external model stage is not configured.""" + + +class StageArtifactError(PipelineError): + """Raised when a stage does not produce the expected artifacts.""" diff --git a/vendor/gs_playground/real2sim/image_io.py b/vendor/gs_playground/real2sim/image_io.py new file mode 100644 index 0000000..43fa0e7 --- /dev/null +++ b/vendor/gs_playground/real2sim/image_io.py @@ -0,0 +1,140 @@ +"""Small image I/O helpers used by the built-in Real2Sim baseline.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from .errors import PipelineError + + +def read_rgb(path: Path) -> np.ndarray: + suffix = path.suffix.lower() + if suffix == ".ppm": + return _read_ppm(path) + + try: + import matplotlib.image as mpimg + except Exception as exc: # pragma: no cover - depends on runtime env + raise PipelineError( + f"reading {suffix or 'this image'} requires matplotlib in this environment. " + "Use a .ppm input or run through the project uv environment." + ) from exc + + arr = np.asarray(mpimg.imread(path), dtype=np.float32) + if arr.ndim == 2: + arr = np.repeat(arr[:, :, None], 3, axis=2) + if arr.shape[-1] == 4: + alpha = arr[:, :, 3:4] + arr = arr[:, :, :3] * alpha + (1.0 - alpha) + else: + arr = arr[:, :, :3] + if arr.dtype.kind in {"u", "i"} or arr.max() > 1.5: + arr = arr / 255.0 + return np.clip(arr, 0.0, 1.0).astype(np.float32) + + +def read_gray(path: Path) -> np.ndarray: + suffix = path.suffix.lower() + if suffix == ".pgm": + return _read_pgm(path) + + try: + import matplotlib.image as mpimg + except Exception as exc: # pragma: no cover - depends on runtime env + raise PipelineError( + f"reading {suffix or 'this mask'} requires matplotlib in this environment. " + "Use a .pgm mask or run through the project uv environment." + ) from exc + + arr = np.asarray(mpimg.imread(path), dtype=np.float32) + if arr.ndim == 3: + arr = arr[:, :, :3].mean(axis=2) + if arr.dtype.kind in {"u", "i"} or arr.max() > 1.5: + arr = arr / 255.0 + return np.clip(arr, 0.0, 1.0).astype(np.float32) + + +def write_rgb(path: Path, image: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + arr = np.clip(image, 0.0, 1.0) + if path.suffix.lower() == ".ppm": + _write_ppm(path, arr) + return + try: + import matplotlib.image as mpimg + + mpimg.imsave(path, arr) + except Exception: + _write_ppm(path, arr) + + +def write_gray(path: Path, image: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + arr = np.clip(image, 0.0, 1.0) + if path.suffix.lower() == ".pgm": + _write_pgm(path, arr) + return + try: + import matplotlib.image as mpimg + + mpimg.imsave(path, arr, cmap="gray") + except Exception: + _write_pgm(path, arr) + + +def _read_ppm(path: Path) -> np.ndarray: + with path.open("rb") as f: + magic = f.readline().strip() + if magic != b"P6": + raise PipelineError(f"only binary P6 PPM input is supported without matplotlib: {path}") + dims = _next_ppm_token_line(f) + width, height = [int(x) for x in dims.split()] + maxval = int(_next_ppm_token_line(f)) + if maxval <= 0 or maxval > 255: + raise PipelineError(f"unsupported PPM max value {maxval}: {path}") + data = np.frombuffer(f.read(width * height * 3), dtype=np.uint8) + if data.size != width * height * 3: + raise PipelineError(f"truncated PPM input: {path}") + return data.reshape(height, width, 3).astype(np.float32) / float(maxval) + + +def _read_pgm(path: Path) -> np.ndarray: + with path.open("rb") as f: + magic = f.readline().strip() + if magic != b"P5": + raise PipelineError(f"only binary P5 PGM input is supported without matplotlib: {path}") + dims = _next_ppm_token_line(f) + width, height = [int(x) for x in dims.split()] + maxval = int(_next_ppm_token_line(f)) + if maxval <= 0 or maxval > 255: + raise PipelineError(f"unsupported PGM max value {maxval}: {path}") + data = np.frombuffer(f.read(width * height), dtype=np.uint8) + if data.size != width * height: + raise PipelineError(f"truncated PGM input: {path}") + return data.reshape(height, width).astype(np.float32) / float(maxval) + + +def _write_ppm(path: Path, image: np.ndarray) -> None: + arr = (np.clip(image, 0.0, 1.0) * 255.0 + 0.5).astype(np.uint8) + with path.open("wb") as f: + f.write(f"P6\n{arr.shape[1]} {arr.shape[0]}\n255\n".encode("ascii")) + f.write(arr[:, :, :3].tobytes()) + + +def _write_pgm(path: Path, image: np.ndarray) -> None: + arr = (np.clip(image, 0.0, 1.0) * 255.0 + 0.5).astype(np.uint8) + with path.open("wb") as f: + f.write(f"P5\n{arr.shape[1]} {arr.shape[0]}\n255\n".encode("ascii")) + f.write(arr.tobytes()) + + +def _next_ppm_token_line(f) -> str: + while True: + line = f.readline() + if not line: + raise PipelineError("unexpected EOF while reading PPM header") + line = line.strip() + if line and not line.startswith(b"#"): + return line.decode("ascii") diff --git a/vendor/gs_playground/real2sim/packaging.py b/vendor/gs_playground/real2sim/packaging.py new file mode 100644 index 0000000..566eefe --- /dev/null +++ b/vendor/gs_playground/real2sim/packaging.py @@ -0,0 +1,3145 @@ +"""GS-Playground package writer for Real2Sim assets.""" + +from __future__ import annotations + +import os +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from math import atan2, cos, pi, sin, sqrt +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from .contracts import Real2SimAsset, Real2SimManifest, write_json +from .errors import StageArtifactError + + +MAX_MOTRIX_PREVIEW_MESH_FACES = 15_000 +DEFAULT_MOTRIX_PREVIEW_PROXY_TEXTURE_SIZE = 1024 +DEFAULT_SCENE_SURFEL_GEOMS = 9_000 +DEFAULT_OBJECT_SURFEL_GEOMS = 3_000 + + +@dataclass(frozen=True) +class PackageResult: + root: Path + scene_xml: Path + config_json: Path + manifest_json: Path + + +@dataclass(frozen=True) +class _ObjFace: + tokens: tuple[str, ...] + vertices: tuple[int, ...] + texcoords: tuple[int | None, ...] + normals: tuple[int | None, ...] + material: str | None + + +@dataclass(frozen=True) +class _ProxyPartMaterial: + key: str + color: tuple[float, float, float] + map_texture: bool + specular: float + shininess: float + alpha: float = 1.0 + + +class PackageWriter: + def __init__( + self, + *, + robot: str = "go2", + dynamic_objects: bool = False, + include_scene_mesh: bool = False, + ) -> None: + self.robot = robot + self.dynamic_objects = dynamic_objects + self.include_scene_mesh = include_scene_mesh + + def write(self, manifest: Real2SimManifest, output_dir: Path) -> PackageResult: + root = output_dir.resolve() + gs_dir = root / "3dgs" + mesh_dir = root / "meshes" + mjcf_dir = root / "mjcf" + config_dir = root / "configs" + artifact_dir = root / "artifacts" + for directory in (gs_dir, mesh_dir, mjcf_dir, config_dir, artifact_dir): + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir(parents=True, exist_ok=True) + + packaged_scene = self._copy_asset( + manifest.scene, + gs_dir, + mesh_dir, + "scene", + include_mesh=self.include_scene_mesh, + allow_preview_proxy=False, + ) + packaged_objects = tuple( + self._copy_asset(asset, gs_dir, mesh_dir, f"object_{idx:03d}_{asset.name}") + for idx, asset in enumerate(manifest.objects) + ) + packaged_camera_artifacts = self._copy_camera_artifacts(manifest.camera_artifacts, artifact_dir) + packaged_manifest = Real2SimManifest( + source=manifest.source, + prompt=manifest.prompt, + scene=packaged_scene, + objects=packaged_objects, + camera_artifacts=packaged_camera_artifacts, + metadata={ + **manifest.metadata, + "package_format": "gs-playground-real2sim-v1", + "robot": self.robot, + "motrix_preview_mesh_mode": motrix_preview_mesh_mode(), + "motrix_preview_mesh_face_limit": motrix_preview_mesh_face_limit(), + "motrix_preview_proxy_profile": motrix_preview_proxy_profile(), + "motrix_preview_proxy_quality": motrix_preview_proxy_quality(), + "motrix_preview_proxy_texture_size": motrix_preview_proxy_texture_size(), + "motrix_scene_visual_mode": scene_visual_mode(), + "motrix_scene_surfel_limit": scene_surfel_geom_limit(), + "motrix_scene_surfel_saturation": scene_surfel_saturation(), + "motrix_scene_surfel_contrast": scene_surfel_contrast(), + "motrix_scene_surfel_highlight_cap": scene_surfel_highlight_cap(), + "motrix_object_surfel_limit": object_surfel_geom_limit(), + "motrix_visual_maxgeom": motrix_visual_maxgeom(len(packaged_objects)), + }, + ) + + scene_xml = mjcf_dir / "scene.xml" + self._write_mjcf(packaged_manifest, scene_xml, mesh_dir) + + config_json = config_dir / f"{self.robot}_real2sim.json" + write_json( + config_json, + { + "robot": self.robot, + "scene": _relative_posix(scene_xml, config_dir), + "scene_gaussians": self._scene_gaussian_config(packaged_manifest, config_dir), + "gaussian_links": self._gaussian_link_config(packaged_manifest), + "camera_artifacts": self._camera_artifact_config(packaged_manifest, config_dir), + "assets_dir": "assets", + "robot_gs_dir": "", + "save_dir": _relative_posix(root / "data", config_dir), + "prompt": manifest.prompt or "Navigate in the reconstructed Real2Sim scene", + "auto_reset": True, + }, + ) + + manifest_json = root / "package_manifest.json" + write_json(manifest_json, packaged_manifest.to_json(root)) + self._write_motrix_import(root, packaged_manifest, scene_xml, config_json) + return PackageResult(root=root, scene_xml=scene_xml, config_json=config_json, manifest_json=manifest_json) + + def _write_motrix_import( + self, + root: Path, + manifest: Real2SimManifest, + scene_xml: Path, + config_json: Path, + ) -> None: + preview = manifest.camera_artifacts.get("source_gs_preview") or manifest.camera_artifacts.get("background_image") + has_source_camera_preview = ( + manifest.camera_artifacts.get("cameras_json") is not None + and manifest.camera_artifacts.get("source_gs_preview") is not None + ) + scene_visual_mode_value = str(manifest.metadata.get("motrix_scene_visual_mode") or "") + prefer_mjcf_scene = scene_visual_mode_value in {"mesh", "hybrid"} + recommended_camera = "overview" if prefer_mjcf_scene else ("source_3dgs_preview" if has_source_camera_preview else "overview") + viewer_camera = "closeup" if (has_source_camera_preview and not prefer_mjcf_scene) else "overview" + viewer_args = [ + "--package", + f"/path/to/{root.name}", + "--camera", + viewer_camera, + "--gs-width", + "900", + "--gs-height", + "700", + "--static-gs-preview", + "--show-source-gs", + ] + if has_source_camera_preview and not prefer_mjcf_scene: + viewer_args.append("--source-gs-only") + import_manifest = { + "format": "gs-playground-real2sim-motrix-import-v1", + "import_type": "mjcf_scene_with_3dgs", + "scene_xml": _relative_posix(scene_xml, root), + "config": _relative_posix(config_json, root), + "scene_gaussians": {"scene": _relative_posix(manifest.scene.gaussian_ply, root)}, + "camera_artifacts": { + key: _relative_posix(path, root) + for key, path in sorted(manifest.camera_artifacts.items()) + }, + "preview": _relative_posix(preview, root) if preview is not None else None, + "recommended_camera": recommended_camera, + "viewer_args": viewer_args, + "wuying_archive_launch": ( + "scripts/real2sim_wuying_launch_package.sh " + f"--archive /path/to/{root.name}.tar.gz " + f"--dest /tmp/{root.name}_import --with-gs" + ), + "verification": { + "scene_ply_vertices": count_ply_vertices(manifest.scene.gaussian_ply), + "object_count": len(manifest.objects), + "pipeline": manifest.metadata.get("pipeline"), + "recommended_preview": recommended_camera, + }, + } + if import_manifest["preview"] is None: + import_manifest.pop("preview") + + write_json(root / "motrix_import.json", import_manifest) + (root / "MOTRIX_IMPORT.md").write_text( + "\n".join( + [ + "# MotrixSim Import", + "", + "Required entrypoints:", + "", + "- MJCF scene: `mjcf/scene.xml`", + "- Viewer config: `configs/*_real2sim.json`", + "- Scene Gaussian asset: `3dgs/scene.ply`", + "- Import manifest: `motrix_import.json`", + "", + "Recommended Wuying launch:", + "", + "```bash", + import_manifest["wuying_archive_launch"], + "```", + "", + "Direct viewer arguments:", + "", + "```bash", + " ".join(import_manifest["viewer_args"]), + "```", + "", + "Verification:", + "", + f"- scene vertices: {import_manifest['verification']['scene_ply_vertices']}", + f"- object count: {import_manifest['verification']['object_count']}", + "", + ] + ), + encoding="utf-8", + ) + + def _copy_asset( + self, + asset: Real2SimAsset, + gs_dir: Path, + mesh_dir: Path, + prefix: str, + *, + include_mesh: bool = True, + allow_preview_proxy: bool = True, + ) -> Real2SimAsset: + safe = _slug(prefix) + ply_dst = gs_dir / f"{safe}.ply" + shutil.copy2(asset.gaussian_ply, ply_dst) + + mesh_dst = None + if include_mesh and asset.mesh_obj is not None: + mesh_dst = mesh_dir / f"{safe}.obj" + shutil.copy2(asset.mesh_obj, mesh_dst) + copy_obj_material_sidecars(asset.mesh_obj, mesh_dst) + clean_obj_floating_top_artifacts(mesh_dst) + materialize_obj_materials(mesh_dst) + auto_expose_obj_diffuse_textures(mesh_dst) + write_obj_preview_lod(mesh_dst, max_faces=motrix_preview_mesh_face_limit(), allow_proxy=allow_preview_proxy) + + return Real2SimAsset( + name=safe, + label=asset.label, + gaussian_ply=ply_dst, + mesh_obj=mesh_dst, + pose=asset.pose, + dynamic=asset.dynamic, + source_instance_id=asset.source_instance_id, + ) + + def _copy_camera_artifacts(self, camera_artifacts: dict[str, Path], artifact_dir: Path) -> dict[str, Path]: + copied: dict[str, Path] = {} + for key, path in sorted(camera_artifacts.items()): + src = Path(path) + if not src.is_file(): + raise StageArtifactError(f"missing camera artifact {key}: {src}") + suffix = src.suffix + dst = artifact_dir / f"{_slug(key)}{suffix}" + shutil.copy2(src, dst) + copied[key] = dst + return copied + + def _scene_gaussian_config(self, manifest: Real2SimManifest, config_dir: Path) -> dict[str, str]: + gaussians = {"scene": _relative_posix(manifest.scene.gaussian_ply, config_dir)} + for asset in manifest.objects: + gaussians[asset.name] = _relative_posix(asset.gaussian_ply, config_dir) + return gaussians + + def _gaussian_link_config(self, manifest: Real2SimManifest) -> dict[str, str]: + return { + asset.name: asset.name + for asset in manifest.objects + if asset.dynamic or self.dynamic_objects + } + + def _camera_artifact_config(self, manifest: Real2SimManifest, config_dir: Path) -> dict[str, str]: + return { + key: _relative_posix(path, config_dir) + for key, path in sorted(manifest.camera_artifacts.items()) + } + + def _write_mjcf(self, manifest: Real2SimManifest, scene_xml: Path, mesh_dir: Path) -> None: + root = ET.Element("mujoco", {"model": "real2sim_scene"}) + ET.SubElement(root, "option", {"gravity": "0 0 -9.81"}) + ET.SubElement(root, "compiler", {"meshdir": "../meshes", "texturedir": "../meshes"}) + visual = ET.SubElement(root, "visual") + ET.SubElement( + visual, + "headlight", + { + "active": "1", + "ambient": "0.42 0.42 0.42", + "diffuse": "0.78 0.78 0.78", + "specular": "0.18 0.18 0.18", + }, + ) + asset_node = ET.SubElement(root, "asset") + world = ET.SubElement(root, "worldbody") + ET.SubElement( + world, + "light", + { + "name": "key_light", + "pos": "1.5 -2 2.8", + "dir": "-0.45 0.55 -1", + "diffuse": "0.9 0.88 0.82", + "specular": "0.22 0.22 0.22", + }, + ) + ET.SubElement( + world, + "light", + { + "name": "fill_light", + "pos": "-2 -1 1.6", + "dir": "0.8 0.35 -0.45", + "diffuse": "0.55 0.58 0.62", + "specular": "0.08 0.08 0.08", + }, + ) + ET.SubElement( + world, + "light", + { + "name": "top_light", + "pos": "0 0 3", + "dir": "0 0 -1", + "diffuse": "0.55 0.55 0.55", + "specular": "0.12 0.12 0.12", + }, + ) + ET.SubElement(world, "camera", {"name": "overview", **manifest_overview_camera_attrs(manifest)}) + ET.SubElement(world, "camera", {"name": "closeup", **manifest_closeup_camera_attrs(manifest)}) + if manifest.scene.mesh_obj is None: + ET.SubElement( + world, + "geom", + { + "name": "fallback_ground", + "type": "plane", + "pos": "0 0 -0.01", + "size": "5 5 0.01", + "rgba": "0.35 0.35 0.35 1", + "group": "3", + }, + ) + + completion_material = add_scene_completion_material(asset_node, mesh_dir) + completion_backdrop_mesh = add_scene_completion_backdrop_mesh(asset_node, mesh_dir, completion_material) + + self._add_asset_mesh(asset_node, manifest.scene, mesh_dir) + for obj in manifest.objects: + self._add_asset_mesh(asset_node, obj, mesh_dir) + + scene_front_y = scene_preview_front_y(manifest.scene.mesh_obj) + + self._add_body( + world, + manifest.scene, + mesh_dir, + is_scene=True, + scene_completion_material=completion_material, + scene_completion_backdrop_mesh=completion_backdrop_mesh, + ) + for obj in manifest.objects: + self._add_body(world, obj, mesh_dir, is_scene=False, object_preview_y=scene_front_y) + + _indent_xml(root) + tree = ET.ElementTree(root) + scene_xml.parent.mkdir(parents=True, exist_ok=True) + tree.write(scene_xml, encoding="utf-8", xml_declaration=True) + + def _add_asset_mesh(self, asset_node: ET.Element, asset: Real2SimAsset, mesh_dir: Path) -> None: + if asset.mesh_obj is None: + return + attrs = { + "name": f"{asset.name}_mesh", + "file": asset.mesh_obj.resolve().relative_to(mesh_dir.resolve()).as_posix(), + } + if not _is_unit_scale(asset.pose.scale): + attrs["scale"] = _vec(asset.pose.scale) + ET.SubElement(asset_node, "mesh", attrs) + material_attrs = { + "name": f"{asset.name}_material", + "rgba": _rgba(obj_average_color(asset.mesh_obj)), + "specular": "0.15", + "shininess": "0.25", + } + texture = obj_diffuse_texture(asset.mesh_obj) + if texture is not None: + texture_name = f"{asset.name}_texture" + ET.SubElement( + asset_node, + "texture", + { + "name": texture_name, + "type": "2d", + "file": texture.resolve().relative_to(mesh_dir.resolve()).as_posix(), + }, + ) + material_attrs["texture"] = texture_name + material_attrs["rgba"] = "1 1 1 1" + ET.SubElement(asset_node, "material", material_attrs) + + def _add_body( + self, + world: ET.Element, + asset: Real2SimAsset, + mesh_dir: Path, + *, + is_scene: bool, + object_preview_y: float | None = None, + scene_completion_material: str | None = None, + scene_completion_backdrop_mesh: str | None = None, + ) -> None: + body_attrs = { + "name": asset.name, + "pos": _vec(asset.pose.position), + "quat": _vec(asset.pose.quat_wxyz), + } + is_dynamic_object = not is_scene and (asset.dynamic or self.dynamic_objects) + bounds = None + if asset.mesh_obj is not None: + bounds = ( + object_dynamic_collision_bounds(asset, preview_world_y=object_preview_y) + if is_dynamic_object + else obj_bounds(asset.mesh_obj, scale=asset.pose.scale) + ) + if is_dynamic_object: + body_attrs["gravcomp"] = "1" + body = ET.SubElement(world, "body", body_attrs) + if is_dynamic_object: + mass, diaginertia = dynamic_object_inertial(asset, bounds) + ET.SubElement(body, "joint", {"name": f"{asset.name}_freejoint", "type": "free", "damping": "5"}) + ET.SubElement(body, "inertial", {"pos": "0 0 0", "mass": f"{mass:.8g}", "diaginertia": _vec(diaginertia)}) + + if asset.mesh_obj is not None: + visual_mode = scene_visual_mode() if is_scene else "surfels" + color_surfels = 0 + if not is_scene or visual_mode in {"surfels", "hybrid"}: + color_surfels = ( + add_scene_color_surfel_geoms(body, asset) + if is_scene + else add_object_color_surfel_geoms(body, asset, preview_world_y=object_preview_y) + ) + if color_surfels == 0 or visual_mode in {"mesh", "hybrid"}: + self._add_mesh_visual_geom(body, asset, is_scene=is_scene) + if is_scene: + add_scene_completion_geoms(body) + add_scene_completion_backdrop_geom( + body, + mesh_name=scene_completion_backdrop_mesh, + material_name=scene_completion_material, + ) + if bounds is not None and not is_scene: + center, half_size = bounds + ET.SubElement( + body, + "geom", + { + "name": f"{asset.name}_collision_box", + "type": "box", + "pos": _vec(center), + "size": _vec(half_size), + "rgba": "0 1 0 0", + "group": "3", + "contype": "1", + "conaffinity": "1", + "friction": "0.8 0.01 0.001", + }, + ) + elif not is_scene: + raise StageArtifactError(f"object asset requires a mesh for collision packaging: {asset.name}") + + def _add_mesh_visual_geom(self, body: ET.Element, asset: Real2SimAsset, *, is_scene: bool) -> None: + geom_attrs = { + "name": f"{asset.name}_visual", + "type": "mesh", + "mesh": f"{asset.name}_mesh", + "group": "2", + "contype": "0", + "conaffinity": "0", + } + if asset.mesh_obj is not None and ( + not is_scene + or obj_diffuse_texture(asset.mesh_obj) is not None + or not obj_has_material_library(asset.mesh_obj) + ): + geom_attrs["material"] = f"{asset.name}_material" + ET.SubElement(body, "geom", geom_attrs) + + +def obj_bounds( + path: Path, + *, + scale: tuple[float, float, float] = (1.0, 1.0, 1.0), +) -> tuple[tuple[float, float, float], tuple[float, float, float]] | None: + vertices: list[tuple[float, float, float]] = [] + with path.open("r", encoding="utf-8", errors="ignore") as f: + for line in f: + if line.startswith("v "): + parts = line.split() + if len(parts) >= 4: + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + if not vertices: + return None + + mins = [min(v[i] for v in vertices) for i in range(3)] + maxs = [max(v[i] for v in vertices) for i in range(3)] + center = tuple(((mins[i] + maxs[i]) / 2.0) * scale[i] for i in range(3)) + half = tuple(max(((maxs[i] - mins[i]) / 2.0) * abs(scale[i]), 1e-4) for i in range(3)) + return center, half + + +def object_dynamic_collision_bounds( + asset: Real2SimAsset, + *, + preview_world_y: float | None = None, + max_half_extent: float = 0.12, +) -> tuple[tuple[float, float, float], tuple[float, float, float]] | None: + bounds = obj_bounds(asset.mesh_obj, scale=asset.pose.scale) if asset.mesh_obj is not None else None + if preview_world_y is None: + return clamp_collision_bounds(bounds, max_half_extent=max_half_extent) + + points = read_gaussian_points(asset.gaussian_ply, max_points=20_000) + if not points: + return clamp_collision_bounds(bounds, max_half_extent=max_half_extent) + xs = [point[0] * asset.pose.scale[0] for point in points] + ys = [point[1] * asset.pose.scale[1] for point in points] + zs = [point[2] * asset.pose.scale[2] for point in points] + center = ( + (min(xs) + max(xs)) * 0.5, + preview_world_y - asset.pose.position[1], + (min(zs) + max(zs)) * 0.5, + ) + half = ( + max((max(xs) - min(xs)) * 0.5, 1e-4), + max((max(ys) - min(ys)) * 0.5 * 0.08, 1e-4), + max((max(zs) - min(zs)) * 0.5, 1e-4), + ) + return clamp_collision_bounds((center, half), max_half_extent=max_half_extent) + + +def clamp_collision_bounds( + bounds: tuple[tuple[float, float, float], tuple[float, float, float]] | None, + *, + max_half_extent: float, + min_half_extent: float = 0.015, +) -> tuple[tuple[float, float, float], tuple[float, float, float]] | None: + if bounds is None: + return None + center, half = bounds + clamped_half = tuple(min(max(abs(value), min_half_extent), max_half_extent) for value in half) + return center, clamped_half + + +def dynamic_object_inertial( + asset: Real2SimAsset, + bounds: tuple[tuple[float, float, float], tuple[float, float, float]] | None, +) -> tuple[float, tuple[float, float, float]]: + half = bounds[1] if bounds is not None else (0.05, 0.05, 0.05) + dimensions = tuple(max(2.0 * abs(value), 0.03) for value in half) + mass = dynamic_object_mass(asset, dimensions) + x, y, z = dimensions + inertia = ( + max(mass * (y * y + z * z) / 12.0, 1e-5), + max(mass * (x * x + z * z) / 12.0, 1e-5), + max(mass * (x * x + y * y) / 12.0, 1e-5), + ) + return mass, inertia + + +def dynamic_object_mass(asset: Real2SimAsset, dimensions: tuple[float, float, float]) -> float: + semantic = f"{asset.label} {asset.name}".lower() + if any(token in semantic for token in ("bottle", "can", "cup", "mug")): + return 0.2 + if any(token in semantic for token in ("book", "box", "cube", "block")): + return 0.35 + volume = max(dimensions[0] * dimensions[1] * dimensions[2], 1e-6) + return min(max(volume * 250.0, 0.05), 1.0) + + +def scene_preview_front_y(scene_mesh: Path | None) -> float | None: + bounds = obj_bounds(scene_mesh) if scene_mesh is not None else None + if bounds is None: + return None + center, half = bounds + return center[1] - half[1] - 0.06 + + +def count_ply_vertices(path: Path) -> int | None: + try: + with path.open("r", encoding="ascii", errors="ignore") as handle: + for line in handle: + if line.startswith("element vertex "): + return int(line.split()[-1]) + if line.strip() == "end_header": + break + except OSError: + return None + return None + + +def overview_camera_attrs(scene_mesh: Path | None) -> dict[str, str]: + bounds = obj_bounds(scene_mesh) if scene_mesh is not None else None + return overview_camera_attrs_from_bounds(bounds) + + +def manifest_overview_camera_attrs(manifest: Real2SimManifest) -> dict[str, str]: + bounds = manifest_world_bounds(manifest) + return overview_camera_attrs_from_bounds(bounds) + + +def manifest_closeup_camera_attrs(manifest: Real2SimManifest) -> dict[str, str]: + bounds = manifest_world_bounds(manifest) + return closeup_camera_attrs_from_bounds(bounds) + + +def manifest_world_bounds(manifest: Real2SimManifest) -> tuple[tuple[float, float, float], tuple[float, float, float]] | None: + mins: list[tuple[float, float, float]] = [] + maxs: list[tuple[float, float, float]] = [] + for asset in (manifest.scene, *manifest.objects): + if asset.mesh_obj is None: + continue + bounds = obj_bounds(asset.mesh_obj, scale=asset.pose.scale) + if bounds is None: + continue + center, half = bounds + world_center = tuple(center[idx] + asset.pose.position[idx] for idx in range(3)) + mins.append(tuple(world_center[idx] - half[idx] for idx in range(3))) + maxs.append(tuple(world_center[idx] + half[idx] for idx in range(3))) + if not mins: + return None + world_min = tuple(min(values[idx] for values in mins) for idx in range(3)) + world_max = tuple(max(values[idx] for values in maxs) for idx in range(3)) + center = tuple((world_min[idx] + world_max[idx]) * 0.5 for idx in range(3)) + half = tuple(max((world_max[idx] - world_min[idx]) * 0.5, 1e-4) for idx in range(3)) + return center, half + + +def overview_camera_attrs_from_bounds(bounds: tuple[tuple[float, float, float], tuple[float, float, float]] | None) -> dict[str, str]: + if bounds is None: + return { + "pos": "1.5 -2.5 1.4", + "xyaxes": "0.866 0.5 0 -0.25 0.433 0.866", + } + center, half = bounds + radius = max(max(half[0], half[2]), 0.35) + target = (center[0], center[1], center[2]) + pos = ( + center[0], + center[1] - max(radius * 2.4, half[1] + 1.35), + center[2], + ) + x_axis, y_axis = source_view_xyaxes(pos, target) + return {"pos": _vec(pos), "xyaxes": f"{_vec(x_axis)} {_vec(y_axis)}"} + + +def closeup_camera_attrs_from_bounds(bounds: tuple[tuple[float, float, float], tuple[float, float, float]] | None) -> dict[str, str]: + if bounds is None: + return { + "pos": "0 -0.75 1", + "xyaxes": "1 0 0 0 0 1", + } + center, half = bounds + radius = max(max(half[0], half[2]), 0.35) + target = (center[0], center[1], center[2]) + pos = ( + center[0], + center[1] - max(radius * 1.15, half[1] + 0.55, 0.55), + center[2], + ) + x_axis, y_axis = source_view_xyaxes(pos, target) + return {"pos": _vec(pos), "xyaxes": f"{_vec(x_axis)} {_vec(y_axis)}"} + + +def source_view_xyaxes( + pos: tuple[float, float, float], + target: tuple[float, float, float], +) -> tuple[tuple[float, float, float], tuple[float, float, float]]: + forward = _normalise_vec(tuple(target[idx] - pos[idx] for idx in range(3)), fallback=(0.0, 1.0, 0.0)) + x_axis = _normalise_vec(_cross(forward, (0.0, 0.0, 1.0)), fallback=(1.0, 0.0, 0.0)) + y_axis = _normalise_vec(_cross(x_axis, forward), fallback=(0.0, 0.0, 1.0)) + return x_axis, y_axis + + +def look_at_xyaxes( + pos: tuple[float, float, float], + target: tuple[float, float, float], +) -> tuple[tuple[float, float, float], tuple[float, float, float]]: + forward = _normalise_vec(tuple(target[idx] - pos[idx] for idx in range(3)), fallback=(0.0, 1.0, -0.35)) + z_axis = tuple(-value for value in forward) + up = (0.0, 0.0, 1.0) + x_axis = _normalise_vec(_cross(up, z_axis), fallback=(1.0, 0.0, 0.0)) + y_axis = _normalise_vec(_cross(z_axis, x_axis), fallback=(0.0, 0.0, 1.0)) + return x_axis, y_axis + + +def _normalise_vec( + values: tuple[float, float, float], + *, + fallback: tuple[float, float, float], +) -> tuple[float, float, float]: + norm = sqrt(sum(value * value for value in values)) + if norm < 1e-8: + return fallback + return tuple(value / norm for value in values) + + +def _cross( + a: tuple[float, float, float], + b: tuple[float, float, float], +) -> tuple[float, float, float]: + return ( + a[1] * b[2] - a[2] * b[1], + a[2] * b[0] - a[0] * b[2], + a[0] * b[1] - a[1] * b[0], + ) + + +def obj_average_color(path: Path, default: tuple[float, float, float] = (0.72, 0.72, 0.72)) -> tuple[float, float, float]: + count = 0 + sums = [0.0, 0.0, 0.0] + with path.open("r", encoding="utf-8", errors="ignore") as f: + for line in f: + if not line.startswith("v "): + continue + parts = line.split() + if len(parts) < 7: + continue + color = _normalise_rgb(parts[4:7]) + for idx, channel in enumerate(color): + sums[idx] += channel + count += 1 + if count == 0: + return default + return tuple(channel_sum / count for channel_sum in sums) + + +def obj_has_material_library(path: Path) -> bool: + with path.open("r", encoding="utf-8", errors="ignore") as f: + return any(line.startswith("mtllib ") for line in f) + + +def add_scene_color_surfel_geoms( + body: ET.Element, + asset: Real2SimAsset, + *, + max_points: int | None = None, +) -> int: + max_points = scene_surfel_geom_limit() if max_points is None else max_points + mesh_obj = asset.mesh_obj + if mesh_obj is None: + points = [] + else: + points = read_obj_vertex_color_points(mesh_obj, max_points=max_points) + if not points: + points = read_obj_texture_color_points(mesh_obj, max_points=max_points) + if not points: + points = read_gaussian_points(asset.gaussian_ply, max_points=max_points) + return add_color_surfel_geoms(body, "scene_color_surfel", points, max_points=max_points) + + +def read_obj_vertex_color_points( + mesh_obj: Path, + *, + max_points: int, +) -> list[tuple[float, float, float, float, float, float]]: + color_mesh = mesh_obj.with_name(f"{mesh_obj.stem}_highres.obj") + if not color_mesh.is_file(): + color_mesh = mesh_obj + vertices: list[tuple[float, float, float, float, float, float]] = [] + with color_mesh.open("r", encoding="utf-8", errors="ignore") as f: + for line in f: + if not line.startswith("v "): + continue + parts = line.split() + if len(parts) < 7: + continue + x, y, z = (float(parts[idx]) for idx in range(1, 4)) + r, g, b = _normalise_rgb(parts[4:7]) + vertices.append((x, y, z, r, g, b)) + if not vertices: + return [] + stride = max(1, int(len(vertices) / max_points)) + return vertices[::stride][:max_points] + + +def read_obj_texture_color_points( + mesh_obj: Path, + *, + max_points: int, +) -> list[tuple[float, float, float, float, float, float]]: + texture = obj_diffuse_texture(mesh_obj) + if texture is None: + return [] + try: + from PIL import Image + except ImportError: + return [] + vertices: list[tuple[float, float, float]] = [] + texcoords: list[tuple[float, float]] = [] + uv_sums: dict[int, list[float]] = {} + uv_counts: dict[int, int] = {} + try: + with mesh_obj.open("r", encoding="utf-8", errors="ignore") as handle: + for line in handle: + if line.startswith("v "): + parts = line.split() + if len(parts) >= 4: + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + elif line.startswith("vt "): + parts = line.split() + if len(parts) >= 3: + texcoords.append((float(parts[1]), float(parts[2]))) + elif line.startswith("f "): + for token in line.split()[1:]: + vertex_index, texcoord_index = parse_obj_vertex_texcoord_token( + token, + vertex_count=len(vertices), + texcoord_count=len(texcoords), + ) + if vertex_index is None or texcoord_index is None: + continue + if vertex_index < 0 or vertex_index >= len(vertices): + continue + if texcoord_index < 0 or texcoord_index >= len(texcoords): + continue + u, v = texcoords[texcoord_index] + if vertex_index not in uv_sums: + uv_sums[vertex_index] = [0.0, 0.0] + uv_counts[vertex_index] = 0 + uv_sums[vertex_index][0] += u + uv_sums[vertex_index][1] += v + uv_counts[vertex_index] += 1 + except (OSError, ValueError): + return [] + if not uv_sums: + return [] + try: + image = Image.open(texture).convert("RGB") + except OSError: + return [] + width, height = image.size + pixels = image.load() + vertex_indices = sorted(uv_sums) + stride = max(1, int(len(vertex_indices) / max_points)) + points: list[tuple[float, float, float, float, float, float]] = [] + for vertex_index in vertex_indices[::stride]: + if len(points) >= max_points: + break + count = max(uv_counts.get(vertex_index, 0), 1) + u = uv_sums[vertex_index][0] / count + v = uv_sums[vertex_index][1] / count + x, y, z = vertices[vertex_index] + px = min(max(int(round((u % 1.0) * (width - 1))), 0), width - 1) + py = min(max(int(round((1.0 - (v % 1.0)) * (height - 1))), 0), height - 1) + r, g, b = pixels[px, py] + points.append((x, y, z, r / 255.0, g / 255.0, b / 255.0)) + return points + + +def parse_obj_vertex_texcoord_token( + token: str, + *, + vertex_count: int, + texcoord_count: int, +) -> tuple[int | None, int | None]: + parts = token.split("/") + if not parts or not parts[0]: + return None, None + try: + vertex_index = int(parts[0]) + except ValueError: + return None, None + texcoord_index = None + if len(parts) >= 2 and parts[1]: + try: + texcoord_index = int(parts[1]) + except ValueError: + texcoord_index = None + return resolve_obj_index(vertex_index, vertex_count), resolve_obj_index(texcoord_index, texcoord_count) + + +def resolve_obj_index(index: int | None, count: int) -> int | None: + if index is None: + return None + if index > 0: + return index - 1 + if index < 0: + return count + index + return None + + +def add_color_surfel_geoms( + body: ET.Element, + prefix: str, + points: list[tuple[float, float, float, float, float, float]], + *, + max_points: int, +) -> int: + if not points: + return 0 + sampled = points[:max_points] + xs = [point[0] for point in sampled] + ys = [point[1] for point in sampled] + zs = [point[2] for point in sampled] + span = max(max(xs) - min(xs), max(ys) - min(ys), max(zs) - min(zs), 1e-3) + radius = max(min(span / 180.0, 0.018), 0.003) * scene_surfel_radius_scale() + size = (radius * 1.2, max(radius * 0.45, 0.0015), radius * 1.2) + for idx, (x, y, z, r, g, b) in enumerate(sampled): + r, g, b = motrix_scene_surfel_color((r, g, b)) + ET.SubElement( + body, + "geom", + { + "name": f"{prefix}_{idx:04d}", + "type": "box", + "pos": _vec((x, y, z)), + "size": _vec(size), + "rgba": _rgba((r, g, b), 1.0), + "group": "2", + "contype": "0", + "conaffinity": "0", + }, + ) + return len(sampled) + + +def add_object_color_surfel_geoms( + body: ET.Element, + asset: Real2SimAsset, + *, + max_points: int | None = None, + preview_world_y: float | None = None, +) -> int: + max_points = object_surfel_geom_limit() if max_points is None else max_points + points = read_gaussian_points(asset.gaussian_ply, max_points=max_points) + if len(points) < 50: + return 0 + xs = [point[0] for point in points] + ys = [point[1] for point in points] + zs = [point[2] for point in points] + span = max(max(xs) - min(xs), max(ys) - min(ys), max(zs) - min(zs), 1e-3) + center_y = (min(ys) + max(ys)) * 0.5 + radius = max(min(span / 80.0, 0.018), 0.004) + size = (radius * 1.25, radius * 1.25, radius * 1.25) + for idx, (x, y, z, r, g, b) in enumerate(points): + r, g, b = motrix_scene_surfel_color((r, g, b)) + if preview_world_y is not None: + y = (y - center_y) * 0.08 + (preview_world_y - asset.pose.position[1]) + ET.SubElement( + body, + "geom", + { + "name": f"{asset.name}_surfel_{idx:04d}", + "type": "box", + "pos": _vec((x, y, z)), + "size": _vec(size), + "rgba": _rgba((r, g, b), 1.0), + "group": "2", + "contype": "0", + "conaffinity": "0", + }, + ) + return len(points) + + +def read_gaussian_points(path: Path, *, max_points: int) -> list[tuple[float, float, float, float, float, float]]: + try: + with path.open("rb") as f: + header_bytes: list[bytes] = [] + while True: + line = f.readline() + if not line: + return [] + header_bytes.append(line) + if line.strip() == b"end_header": + break + header = [line.decode("ascii", errors="ignore").strip() for line in header_bytes] + if not header or header[0] != "ply": + return [] + is_ascii = "format ascii 1.0" in header + is_binary_little = "format binary_little_endian 1.0" in header + if not is_ascii and not is_binary_little: + return [] + vertex_count = 0 + properties: list[tuple[str, str]] = [] + reading_vertex = False + for line in header: + parts = line.split() + if len(parts) >= 3 and parts[:2] == ["element", "vertex"]: + vertex_count = int(parts[2]) + reading_vertex = True + continue + if len(parts) >= 3 and parts[0] == "element" and parts[1] != "vertex": + reading_vertex = False + if reading_vertex and len(parts) == 3 and parts[0] == "property": + properties.append((parts[1], parts[2])) + names = [name for _kind, name in properties] + indices = {name: idx for idx, name in enumerate(names)} + required = ("x", "y", "z", "f_dc_0", "f_dc_1", "f_dc_2") + if not all(name in indices for name in required): + return [] + stride = max(1, vertex_count // max(max_points, 1)) + points: list[tuple[float, float, float, float, float, float]] = [] + sh_c0 = 0.28209479177387814 + if is_ascii: + for line_idx, raw_line in enumerate(f): + if line_idx % stride != 0: + continue + parts = raw_line.decode("ascii", errors="ignore").split() + if len(parts) < len(properties): + continue + points.append(_gaussian_point_from_values(parts, indices, sh_c0)) + if len(points) >= max_points: + break + else: + readers, record_size = binary_ply_vertex_readers(properties) + if record_size <= 0: + return [] + for vertex_idx in range(vertex_count): + record = f.read(record_size) + if len(record) != record_size: + break + if vertex_idx % stride != 0: + continue + values = [reader(record) for reader in readers] + points.append(_gaussian_point_from_values(values, indices, sh_c0)) + if len(points) >= max_points: + break + return points + except (OSError, ValueError, struct.error): + return [] + + +def _gaussian_point_from_values( + values: list[object], + indices: dict[str, int], + sh_c0: float, +) -> tuple[float, float, float, float, float, float]: + x = float(values[indices["x"]]) + y = float(values[indices["y"]]) + z = float(values[indices["z"]]) + r = float(values[indices["f_dc_0"]]) * sh_c0 + 0.5 + g = float(values[indices["f_dc_1"]]) * sh_c0 + 0.5 + b = float(values[indices["f_dc_2"]]) * sh_c0 + 0.5 + return (x, y, z, r, g, b) + + +def binary_ply_vertex_readers(properties: list[tuple[str, str]]): + offsets: list[tuple[str, int, int]] = [] + offset = 0 + sizes = { + "char": 1, + "int8": 1, + "uchar": 1, + "uint8": 1, + "short": 2, + "int16": 2, + "ushort": 2, + "uint16": 2, + "int": 4, + "int32": 4, + "uint": 4, + "uint32": 4, + "float": 4, + "float32": 4, + "double": 8, + "float64": 8, + } + formats = { + "char": "b", + "int8": "b", + "uchar": "B", + "uint8": "B", + "short": "h", + "int16": "h", + "ushort": "H", + "uint16": "H", + "int": "i", + "int32": "i", + "uint": "I", + "uint32": "I", + "float": "f", + "float32": "f", + "double": "d", + "float64": "d", + } + for kind, _name in properties: + if kind not in sizes: + return [], 0 + offsets.append((formats[kind], offset, sizes[kind])) + offset += sizes[kind] + + def make_reader(fmt: str, start: int, size: int): + return lambda record: struct.unpack_from("<" + fmt, record, start)[0] + + return [make_reader(fmt, start, size) for fmt, start, size in offsets], offset + + +def motrix_scene_surfel_color(color: tuple[float, float, float]) -> tuple[float, float, float]: + """Keep single-image scene surfels legible under MotrixSim preview lights.""" + + r, g, b = (min(max(float(channel), 0.0), 1.0) for channel in color) + luminance = 0.2126 * r + 0.7152 * g + 0.0722 * b + saturation = scene_surfel_saturation() + contrast = scene_surfel_contrast() + channels = [] + for value in (r, g, b): + saturated = luminance + (value - luminance) * saturation + contrasted = 0.5 + (saturated - 0.5) * contrast + compressed = min(contrasted, scene_surfel_highlight_cap()) + channels.append(min(max(compressed, 0.0), 1.0)) + return tuple(channels) + + +def scene_surfel_geom_limit() -> int: + raw = os.environ.get("REAL2SIM_MOTRIX_SCENE_SURFELS") + if raw is None: + return DEFAULT_SCENE_SURFEL_GEOMS + try: + value = int(raw) + except ValueError: + return DEFAULT_SCENE_SURFEL_GEOMS + return min(max(value, 1_000), 30_000) + + +def object_surfel_geom_limit() -> int: + raw = os.environ.get("REAL2SIM_MOTRIX_OBJECT_SURFELS") + if raw is None: + return DEFAULT_OBJECT_SURFEL_GEOMS + try: + value = int(raw) + except ValueError: + return DEFAULT_OBJECT_SURFEL_GEOMS + return min(max(value, 250), 12_000) + + +def motrix_visual_maxgeom(object_count: int = 0) -> int: + raw = os.environ.get("REAL2SIM_MOTRIX_MAXGEOM") + if raw is not None: + try: + return min(max(int(raw), 10_000), 200_000) + except ValueError: + pass + scene_mode = scene_visual_mode() + scene_geoms = scene_surfel_geom_limit() if scene_mode in {"surfels", "hybrid"} else 0 + object_geoms = max(int(object_count), 0) * object_surfel_geom_limit() + mesh_and_completion_margin = 2_500 + return min(max((scene_geoms + object_geoms + mesh_and_completion_margin) * 2, 10_000), 200_000) + + +def scene_visual_mode() -> str: + raw = os.environ.get("REAL2SIM_MOTRIX_SCENE_VISUAL", "surfels").strip().lower() + if raw in { + "mesh", + "surface", + "continuous", + "textured_mesh", + "textured-mesh", + "photogrammetry", + "high_fidelity", + "high-fidelity", + }: + return "mesh" + if raw in {"hybrid", "mesh+surfels", "surfels+mesh"}: + return "hybrid" + return "surfels" + + +def add_scene_completion_material(asset_node: ET.Element, mesh_dir: Path) -> str | None: + texture_path = scene_completion_texture_path() + if texture_path is None: + return None + suffix = texture_path.suffix.lower() + if suffix not in {".png", ".jpg", ".jpeg"}: + return None + mesh_dir.mkdir(parents=True, exist_ok=True) + dest = mesh_dir / "scene_completion_texture.png" + try: + if suffix == ".png": + shutil.copy2(texture_path, dest) + else: + from PIL import Image + + Image.open(texture_path).convert("RGB").save(dest) + except OSError: + return None + except ImportError: + return None + ET.SubElement( + asset_node, + "texture", + { + "name": "scene_completion_texture", + "type": "2d", + "file": dest.name, + }, + ) + ET.SubElement( + asset_node, + "material", + { + "name": "scene_completion_photo_material", + "texture": "scene_completion_texture", + "rgba": "1 1 1 0.88", + "specular": "0.02", + "shininess": "0.05", + }, + ) + return "scene_completion_photo_material" + + +def scene_completion_texture_path() -> Path | None: + raw = os.environ.get("REAL2SIM_SCENE_COMPLETION_TEXTURE") + if not raw: + return None + path = Path(raw) + if not path.is_file() or path.stat().st_size <= 0: + return None + return path + + +def add_scene_completion_backdrop_mesh( + asset_node: ET.Element, + mesh_dir: Path, + material_name: str | None, +) -> str | None: + if material_name is None: + return None + report = scene_completion_report() + if report is None: + return None + plane = scene_completion_back_plane(report) + if plane is None: + return None + mesh_dir.mkdir(parents=True, exist_ok=True) + mesh_path = mesh_dir / "scene_completion_backdrop.obj" + vertices = plane + mesh_path.write_text( + "\n".join( + [ + "o real2sim_scene_completion_backdrop", + f"v {vertices[0][0]:.8f} {vertices[0][1]:.8f} {vertices[0][2]:.8f}", + f"v {vertices[1][0]:.8f} {vertices[1][1]:.8f} {vertices[1][2]:.8f}", + f"v {vertices[2][0]:.8f} {vertices[2][1]:.8f} {vertices[2][2]:.8f}", + f"v {vertices[3][0]:.8f} {vertices[3][1]:.8f} {vertices[3][2]:.8f}", + "vt 0.00000000 0.00000000", + "vt 1.00000000 0.00000000", + "vt 1.00000000 1.00000000", + "vt 0.00000000 1.00000000", + "f 1/1 2/2 3/3", + "f 1/1 3/3 4/4", + "", + ] + ), + encoding="utf-8", + ) + ET.SubElement( + asset_node, + "mesh", + { + "name": "scene_completion_backdrop_mesh", + "file": mesh_path.name, + "inertia": "shell", + }, + ) + return "scene_completion_backdrop_mesh" + + +def scene_completion_back_plane(report: dict[str, object]) -> list[tuple[float, float, float]] | None: + low = report.get("bounds_low") + high = report.get("bounds_high") + if not (isinstance(low, list) and isinstance(high, list) and len(low) == 3 and len(high) == 3): + return None + try: + bounds_low = tuple(float(value) for value in low) + bounds_high = tuple(float(value) for value in high) + except (TypeError, ValueError): + return None + up_axis = str(report.get("up_axis") or "z").lower() + if up_axis not in {"x", "y", "z"}: + up_axis = "z" + axes = {"x": 0, "y": 1, "z": 2} + up = axes[up_axis] + horizontal = [axis for axis in (0, 1, 2) if axis != up] + u_axis, depth_axis = horizontal + span = [max(bounds_high[idx] - bounds_low[idx], 1e-4) for idx in range(3)] + offset = max(max(span) / 1000.0, 0.005) + depth = bounds_low[depth_axis] - offset + backdrop_margin = scene_completion_backdrop_margin() + u_margin = span[u_axis] * backdrop_margin + up_margin = span[up] * backdrop_margin + u_low = bounds_low[u_axis] - u_margin + u_high = bounds_high[u_axis] + u_margin + up_low = bounds_low[up] - up_margin + up_high = bounds_high[up] + up_margin + + def vertex(u: float, v: float) -> tuple[float, float, float]: + coords = [0.0, 0.0, 0.0] + coords[u_axis] = u + coords[depth_axis] = depth + coords[up] = v + return (coords[0], coords[1], coords[2]) + + return [ + vertex(u_low, up_low), + vertex(u_high, up_low), + vertex(u_high, up_high), + vertex(u_low, up_high), + ] + + +def scene_completion_backdrop_margin() -> float: + raw = os.environ.get("REAL2SIM_SCENE_COMPLETION_BACKDROP_MARGIN") + if raw is None: + return 0.0 + try: + value = float(raw) + except ValueError: + return 0.0 + return min(max(value, 0.0), 2.0) + + +def add_scene_completion_backdrop_geom( + body: ET.Element, + *, + mesh_name: str | None, + material_name: str | None, +) -> int: + if mesh_name is None or material_name is None: + return 0 + ET.SubElement( + body, + "geom", + { + "name": "scene_completion_photo_backdrop", + "type": "mesh", + "mesh": mesh_name, + "material": material_name, + "group": "2", + "contype": "0", + "conaffinity": "0", + }, + ) + return 1 + + +def add_scene_completion_geoms(body: ET.Element) -> int: + report = scene_completion_report() + if report is None: + return 0 + low = report.get("bounds_low") + high = report.get("bounds_high") + planes = report.get("planes") + if not (isinstance(low, list) and isinstance(high, list) and len(low) == 3 and len(high) == 3): + return 0 + if not isinstance(planes, list): + return 0 + try: + bounds_low = tuple(float(value) for value in low) + bounds_high = tuple(float(value) for value in high) + except (TypeError, ValueError): + return 0 + up_axis = str(report.get("up_axis") or "z").lower() + if up_axis not in {"x", "y", "z"}: + up_axis = "z" + axes = {"x": 0, "y": 1, "z": 2} + up = axes[up_axis] + horizontal = [axis for axis in (0, 1, 2) if axis != up] + a0, a1 = horizontal + span = [max(bounds_high[idx] - bounds_low[idx], 1e-4) for idx in range(3)] + thickness = max(max(span) / 500.0, 0.01) + added = 0 + for raw_name in planes: + name = str(raw_name) + pos = [(bounds_low[idx] + bounds_high[idx]) * 0.5 for idx in range(3)] + size = [span[idx] * 0.5 for idx in range(3)] + rgba = (0.68, 0.65, 0.59, 0.06) + if name == "floor": + pos[up] = bounds_low[up] + size[up] = thickness + rgba = (0.42, 0.41, 0.39, 0.12) + elif name == "ceiling": + pos[up] = bounds_high[up] + size[up] = thickness + rgba = (0.72, 0.70, 0.66, 0.04) + elif name == "left": + pos[a0] = bounds_low[a0] + size[a0] = thickness + elif name == "right": + pos[a0] = bounds_high[a0] + size[a0] = thickness + elif name == "back": + pos[a1] = bounds_low[a1] + size[a1] = thickness + elif name == "front": + pos[a1] = bounds_high[a1] + size[a1] = thickness + else: + continue + ET.SubElement( + body, + "geom", + { + "name": f"scene_completion_{name}", + "type": "box", + "pos": _vec(tuple(pos)), + "size": _vec(tuple(size)), + "rgba": _rgba(rgba[:3], rgba[3]), + "group": "2", + "contype": "1", + "conaffinity": "1", + }, + ) + added += 1 + return added + + +def scene_completion_report() -> dict[str, object] | None: + path = os.environ.get("REAL2SIM_SCENE_COMPLETION_REPORT") + if not path: + return None + report_path = Path(path) + if not report_path.is_file(): + return None + try: + import json + + data = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return data if isinstance(data, dict) else None + + +def scene_surfel_saturation() -> float: + return _env_float("REAL2SIM_MOTRIX_SCENE_SURFEL_SATURATION", 1.35, minimum=0.2, maximum=3.0) + + +def scene_surfel_radius_scale() -> float: + return _env_float("REAL2SIM_MOTRIX_SCENE_SURFEL_RADIUS_SCALE", 1.0, minimum=0.2, maximum=4.0) + + +def scene_surfel_contrast() -> float: + return _env_float("REAL2SIM_MOTRIX_SCENE_SURFEL_CONTRAST", 1.12, minimum=0.2, maximum=3.0) + + +def scene_surfel_highlight_cap() -> float: + return _env_float("REAL2SIM_MOTRIX_SCENE_SURFEL_HIGHLIGHT_CAP", 0.88, minimum=0.45, maximum=1.0) + + +def _env_float(name: str, default: float, *, minimum: float, maximum: float) -> float: + raw = os.environ.get(name) + if raw is None: + return default + try: + value = float(raw) + except ValueError: + return default + return min(max(value, minimum), maximum) + + +def obj_diffuse_texture(path: Path) -> Path | None: + obj_dir = path.parent + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + if not line.startswith("mtllib "): + continue + for raw_mtl in line.split()[1:]: + texture = mtl_diffuse_texture(obj_dir / raw_mtl) + if texture is not None: + return texture + return None + + +def mtl_diffuse_texture(path: Path) -> Path | None: + if not path.is_file(): + return None + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = line.split() + if len(parts) < 2 or parts[0] != "map_Kd": + continue + texture = (path.parent / parts[-1]).resolve() + if texture.is_file(): + return texture + return None + + +def clean_obj_floating_top_artifacts( + path: Path, + *, + min_component_count: int = 50, + min_faces: int = 100, + top_ratio: float = 0.82, + tip_ratio: float = 0.92, + flat_y_ratio: float = 0.12, + broad_ratio: float = 0.45, + tip_broad_ratio: float = 0.25, + desaturated_top_ratio: float = 0.88, + desaturated_flat_y_ratio: float = 0.09, + desaturated_broad_ratio: float = 0.08, + preserve_saturated_top_saturation: float = 95.0, +) -> dict[str, int]: + """Remove disconnected, wide, flat top sheets from SAM-3D OBJ exports. + + Transparent objects often produce thousands of tiny OBJ components. A common + failure mode is a broad, very flat sheet near the top of the local object + frame. It shows up as a floating "cloth" in MotrixSim after texture baking. + The cleanup is deliberately gated on high component count so normal OBJ + assets with intentional flat tops are left unchanged. + """ + + raw_lines = path.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True) + parsed = _parse_obj(raw_lines) + vertices = parsed["vertices"] + texcoords = parsed["texcoords"] + normals = parsed["normals"] + faces: list[_ObjFace] = parsed["faces"] + if not vertices or not faces: + return {"removed_components": 0, "removed_faces": 0, "kept_faces": len(faces)} + + components = _obj_face_components(faces, len(vertices)) + if len(components) < min_component_count: + return {"removed_components": 0, "removed_faces": 0, "kept_faces": len(faces)} + + used_vertices = sorted({vertex for face in faces for vertex in face.vertices}) + bounds = _vertex_bounds(vertices, used_vertices) + if bounds is None: + return {"removed_components": 0, "removed_faces": 0, "kept_faces": len(faces)} + mins, maxs = bounds + global_ext = tuple(max(maxs[idx] - mins[idx], 1e-8) for idx in range(3)) + texture_image = _load_obj_diffuse_texture(path) if texcoords else None + + removed_roots: set[int] = set() + for root, face_indices in components.items(): + if len(face_indices) < min_faces: + continue + component_vertices = sorted({vertex for face_index in face_indices for vertex in faces[face_index].vertices}) + component_bounds = _vertex_bounds(vertices, component_vertices) + if component_bounds is None: + continue + comp_mins, comp_maxs = component_bounds + comp_ext = tuple(comp_maxs[idx] - comp_mins[idx] for idx in range(3)) + center_y = (comp_mins[1] + comp_maxs[1]) * 0.5 + top_position = (center_y - mins[1]) / global_ext[1] + flatness = comp_ext[1] / global_ext[1] + broadness = max(comp_ext[0] / global_ext[0], comp_ext[2] / global_ext[2]) + broad_top_sheet = top_position >= top_ratio and flatness <= flat_y_ratio and broadness >= broad_ratio + high_tip_sheet = top_position >= tip_ratio and flatness <= flat_y_ratio and broadness >= tip_broad_ratio + desaturated_top_sheet = False + saturated_top_part = False + texture_stats = None + if texture_image is not None: + texture_stats = _component_texture_stats(texture_image, texcoords, faces, face_indices) + if texture_stats is not None: + _, saturation = texture_stats + saturated_top_part = saturation >= preserve_saturated_top_saturation + if ( + texture_image is not None + and top_position >= desaturated_top_ratio + and flatness <= desaturated_flat_y_ratio + and broadness >= desaturated_broad_ratio + ): + if texture_stats is not None: + brightness, saturation = texture_stats + desaturated_top_sheet = 30.0 <= brightness <= 155.0 and saturation <= 80.0 + if (not saturated_top_part and (broad_top_sheet or high_tip_sheet)) or desaturated_top_sheet: + removed_roots.add(root) + + if not removed_roots: + return {"removed_components": 0, "removed_faces": 0, "kept_faces": len(faces)} + + kept_faces = [face for idx, face in enumerate(faces) if parsed["face_roots"][idx] not in removed_roots] + _write_compacted_obj(path, parsed["header"], vertices, texcoords, normals, kept_faces) + removed_faces = len(faces) - len(kept_faces) + return {"removed_components": len(removed_roots), "removed_faces": removed_faces, "kept_faces": len(kept_faces)} + + +def _load_obj_diffuse_texture(path: Path): + texture = obj_diffuse_texture(path) + if texture is None: + return None + try: + from PIL import Image + + return Image.open(texture).convert("RGB") + except Exception: + return None + + +def _component_texture_stats( + image, + texcoords: list[str], + faces: list[_ObjFace], + face_indices: list[int], + *, + sample_limit: int = 300, +) -> tuple[float, float] | None: + pixels: list[tuple[int, int, int]] = [] + width, height = image.size + step = max(len(face_indices) // sample_limit, 1) + for face_index in face_indices[::step]: + face = faces[face_index] + for texcoord in face.texcoords: + if texcoord is None or texcoord <= 0 or texcoord > len(texcoords): + continue + parts = texcoords[texcoord - 1].split() + if len(parts) < 3: + continue + u = float(parts[1]) + v = float(parts[2]) + x = min(max(int(u * (width - 1)), 0), width - 1) + y = min(max(int((1.0 - v) * (height - 1)), 0), height - 1) + pixels.append(image.getpixel((x, y))) + if not pixels: + return None + channels = tuple(sum(pixel[idx] for pixel in pixels) / len(pixels) for idx in range(3)) + brightness = sum(channels) / 3.0 + saturation = max(channels) - min(channels) + return brightness, saturation + + +def write_obj_preview_lod(path: Path, *, max_faces: int, allow_proxy: bool = True) -> dict[str, int]: + """Write a lower-face OBJ for interactive MotrixSim preview. + + The SAM-3D textured meshes remain valuable as assets, but MotrixSim's + interactive viewer stalls when every baked reconstruction triangle is loaded + directly in MJCF. This keeps a packaged ``*_highres.obj`` copy and rewrites + the MJCF-facing OBJ to a deterministic, component-aware face subset while + preserving UVs, materials, and texture sidecars. + """ + + max_faces = max(int(max_faces), 1) + raw_lines = path.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True) + parsed = _parse_obj(raw_lines) + vertices = parsed["vertices"] + texcoords = parsed["texcoords"] + normals = parsed["normals"] + faces: list[_ObjFace] = parsed["faces"] + if len(faces) <= max_faces: + ensure_obj_vertex_normals(path) + return {"original_faces": len(faces), "preview_faces": len(faces), "highres_written": 0} + + highres_path = path.with_name(f"{path.stem}_highres.obj") + if not highres_path.exists(): + shutil.copy2(path, highres_path) + + if allow_proxy and motrix_preview_mesh_mode() == "proxy" and _write_revolved_proxy_preview_lod( + path, + vertices=vertices, + texcoords=texcoords, + faces=faces, + max_faces=max_faces, + ): + ensure_obj_vertex_normals(path) + return { + "original_faces": len(faces), + "preview_faces": _count_obj_faces(path), + "highres_written": 1, + } + + if _write_pymeshlab_preview_lod(path, max_faces=max_faces): + ensure_obj_vertex_normals(path) + return { + "original_faces": len(faces), + "preview_faces": _count_obj_faces(path), + "highres_written": 1, + } + + selected = _select_lod_faces(faces, len(vertices), max_faces=max_faces) + _write_compacted_obj(path, parsed["header"], vertices, texcoords, normals, selected) + ensure_obj_vertex_normals(path) + return {"original_faces": len(faces), "preview_faces": len(selected), "highres_written": 1} + + +def motrix_preview_mesh_face_limit() -> int: + raw = os.environ.get("REAL2SIM_MOTRIX_PREVIEW_MESH_FACES") + if raw is None: + return MAX_MOTRIX_PREVIEW_MESH_FACES + try: + value = int(raw) + except ValueError: + return MAX_MOTRIX_PREVIEW_MESH_FACES + return max(value, 1_000) + + +def motrix_preview_mesh_mode() -> str: + raw = os.environ.get("REAL2SIM_MOTRIX_PREVIEW_MODE", "proxy").strip().lower() + return "lod" if raw in {"lod", "mesh", "decimate", "decimation"} else "proxy" + + +def motrix_preview_proxy_profile() -> str: + raw = os.environ.get("REAL2SIM_MOTRIX_PREVIEW_PROXY_PROFILE", "radial").strip().lower() + return "angular" if raw in {"angular", "silhouette", "non_symmetric", "asymmetric"} else "radial" + + +def motrix_preview_proxy_quality() -> str: + raw = os.environ.get("REAL2SIM_MOTRIX_PREVIEW_PROXY_QUALITY", "high").strip().lower() + if raw in {"fast", "low", "performance"}: + return "fast" + if raw in {"balanced", "medium"}: + return "balanced" + return "high" + + +def motrix_preview_proxy_texture_size() -> int: + raw = os.environ.get("REAL2SIM_MOTRIX_PREVIEW_PROXY_TEXTURE_SIZE") + if raw is None: + return DEFAULT_MOTRIX_PREVIEW_PROXY_TEXTURE_SIZE + try: + value = int(raw) + except ValueError: + return DEFAULT_MOTRIX_PREVIEW_PROXY_TEXTURE_SIZE + return min(max(value, 256), 2048) + + +def _write_revolved_proxy_preview_lod( + path: Path, + *, + vertices: list[tuple[str, tuple[float, float, float]]], + texcoords: list[str], + faces: list[_ObjFace], + max_faces: int, +) -> bool: + used_vertices = sorted({vertex for face in faces for vertex in face.vertices}) + bounds = _vertex_bounds(vertices, used_vertices) + if bounds is None: + return False + mins, maxs = bounds + ext = tuple(max(maxs[idx] - mins[idx], 1e-8) for idx in range(3)) + center_x = _quantile([vertices[index - 1][1][0] for index in used_vertices], 0.5) + center_z = _quantile([vertices[index - 1][1][2] for index in used_vertices], 0.5) + band_count, radial_segments = _proxy_resolution(max_faces) + y_min, y_max = mins[1], maxs[1] + y_span = max(y_max - y_min, 1e-8) + min_radius = max(min(ext[0], ext[2]) * 0.035, 1e-4) + if motrix_preview_proxy_profile() == "angular": + radial_profile = _angular_radial_profile( + vertices, + used_vertices, + band_count=band_count, + radial_segments=radial_segments, + center_x=center_x, + center_z=center_z, + y_min=y_min, + y_span=y_span, + min_radius=min_radius, + ) + else: + radial_profile = [ + [radius for _segment in range(radial_segments)] + for radius in _robust_radial_profile( + vertices, + used_vertices, + band_count=band_count, + center_x=center_x, + center_z=center_z, + y_min=y_min, + y_span=y_span, + min_radius=min_radius, + ) + ] + texture_name = f"{path.stem}_proxy_texture.png" + band_colors = _preview_band_colors(path, vertices, texcoords, faces, band_count, y_min, y_span) + texture_size = motrix_preview_proxy_texture_size() + if not _write_proxy_texture( + path.with_name(texture_name), + source_obj=path, + vertices=vertices, + texcoords=texcoords, + faces=faces, + width=texture_size, + height=texture_size, + center_x=center_x, + center_z=center_z, + y_min=y_min, + y_span=y_span, + fallback_band_colors=band_colors, + ): + texture_name = None + + cap_bands, cap_color = _top_red_cap_material(band_colors) + part_band_keys, part_materials = _proxy_part_materials( + band_colors, + cap_bands=cap_bands, + cap_color=cap_color, + texture_name=texture_name, + ) + vertex_records: list[tuple[float, float, float, float, float]] = [] + ring_indices: list[list[int]] = [] + for ring in range(band_count + 1): + y = y_min + y_span * (ring / band_count) + v_coord = ring / band_count + if ring == 0: + ring_radii = radial_profile[0] + elif ring == band_count: + ring_radii = radial_profile[-1] + else: + ring_radii = [ + (radial_profile[ring - 1][segment] + radial_profile[ring][segment]) * 0.5 + for segment in range(radial_segments) + ] + ring_vertices = [] + for segment in range(radial_segments + 1): + theta = 2.0 * pi * segment / radial_segments + radius = ring_radii[segment % radial_segments] + vertex_records.append( + ( + center_x + radius * cos(theta), + y, + center_z + radius * sin(theta), + segment / radial_segments, + v_coord, + ) + ) + ring_vertices.append(len(vertex_records)) + ring_indices.append(ring_vertices) + + vertex_records.append((center_x, y_min, center_z, 0.5, 0.0)) + bottom_center = len(vertex_records) + vertex_records.append((center_x, y_max, center_z, 0.5, 1.0)) + top_center = len(vertex_records) + + material_name = f"{path.stem}_proxy_baked" if texture_name is not None else f"{path.stem}_proxy_band_00" + obj_lines = [f"mtllib {path.stem}_proxy.mtl\n", f"o {_slug(path.stem)}_motrix_proxy\n"] + for x, y, z, _u, _v in vertex_records: + obj_lines.append(f"v {x:.8f} {y:.8f} {z:.8f}\n") + for _x, _y, _z, u, v in vertex_records: + obj_lines.append(f"vt {u:.8f} {v:.8f}\n") + current_material: str | None = None + for band in range(band_count): + band_key = part_band_keys[band] if band < len(part_band_keys) else None + band_material = f"{path.stem}_proxy_{band_key}" if band_key is not None else material_name + if band_material != current_material: + obj_lines.append(f"usemtl {band_material}\n") + current_material = band_material + lower = ring_indices[band] + upper = ring_indices[band + 1] + for segment in range(radial_segments): + nxt = segment + 1 + obj_lines.append(f"f {lower[segment]}/{lower[segment]} {lower[nxt]}/{lower[nxt]} {upper[nxt]}/{upper[nxt]}\n") + obj_lines.append(f"f {lower[segment]}/{lower[segment]} {upper[nxt]}/{upper[nxt]} {upper[segment]}/{upper[segment]}\n") + + if material_name != current_material: + obj_lines.append(f"usemtl {material_name}\n") + current_material = material_name + for segment in range(radial_segments): + nxt = segment + 1 + obj_lines.append(f"f {bottom_center}/{bottom_center} {ring_indices[0][nxt]}/{ring_indices[0][nxt]} {ring_indices[0][segment]}/{ring_indices[0][segment]}\n") + top_key = part_band_keys[band_count - 1] if part_band_keys else None + top_material = f"{path.stem}_proxy_{top_key}" if top_key is not None else material_name + if top_material != current_material: + obj_lines.append(f"usemtl {top_material}\n") + for segment in range(radial_segments): + nxt = segment + 1 + obj_lines.append(f"f {top_center}/{top_center} {ring_indices[-1][segment]}/{ring_indices[-1][segment]} {ring_indices[-1][nxt]}/{ring_indices[-1][nxt]}\n") + + path.write_text("".join(obj_lines), encoding="utf-8") + _write_proxy_mtl( + path.with_name(f"{path.stem}_proxy.mtl"), + path.stem, + band_colors, + texture_name=texture_name, + part_materials=part_materials, + ) + return True + + +def _proxy_resolution(max_faces: int) -> tuple[int, int]: + quality = motrix_preview_proxy_quality() + if quality == "fast": + band_count, radial_segments = 24, 32 + elif quality == "balanced": + band_count, radial_segments = 48, 64 + else: + band_count, radial_segments = 72, 96 + + minimum_bands, minimum_segments = 8, 12 + while _proxy_face_count(band_count, radial_segments) > max_faces and ( + band_count > minimum_bands or radial_segments > minimum_segments + ): + if radial_segments / max(band_count, 1) > 1.35 and radial_segments > minimum_segments: + radial_segments -= 4 + elif band_count > minimum_bands: + band_count -= 2 + elif radial_segments > minimum_segments: + radial_segments -= 2 + return band_count, radial_segments + + +def _proxy_face_count(band_count: int, radial_segments: int) -> int: + return band_count * radial_segments * 2 + radial_segments * 2 + + +def _robust_radial_profile( + vertices: list[tuple[str, tuple[float, float, float]]], + used_vertices: list[int], + *, + band_count: int, + center_x: float, + center_z: float, + y_min: float, + y_span: float, + min_radius: float, +) -> list[float]: + bands: list[list[float]] = [[] for _band in range(band_count)] + for vertex_index in used_vertices: + x, y, z = vertices[vertex_index - 1][1] + band = min(max(int(((y - y_min) / y_span) * band_count), 0), band_count - 1) + dx = x - center_x + dz = z - center_z + bands[band].append(max((dx * dx + dz * dz) ** 0.5, min_radius)) + + profile: list[float | None] = [] + for values in bands: + profile.append(max(_quantile(values, 0.74), min_radius) if values else None) + _fill_missing_profile(profile, min_radius) + smoothed = [float(value) for value in profile] + for _ in range(3): + smoothed = _smooth_profile(smoothed, min_radius) + return smoothed + + +def _angular_radial_profile( + vertices: list[tuple[str, tuple[float, float, float]]], + used_vertices: list[int], + *, + band_count: int, + radial_segments: int, + center_x: float, + center_z: float, + y_min: float, + y_span: float, + min_radius: float, +) -> list[list[float]]: + bins: list[list[list[float]]] = [[[] for _segment in range(radial_segments)] for _band in range(band_count)] + band_values: list[list[float]] = [[] for _band in range(band_count)] + for vertex_index in used_vertices: + x, y, z = vertices[vertex_index - 1][1] + band = min(max(int(((y - y_min) / y_span) * band_count), 0), band_count - 1) + dx = x - center_x + dz = z - center_z + radius = max((dx * dx + dz * dz) ** 0.5, min_radius) + theta = atan2(dz, dx) + if theta < 0.0: + theta += 2.0 * pi + segment = min(max(int((theta / (2.0 * pi)) * radial_segments), 0), radial_segments - 1) + bins[band][segment].append(radius) + band_values[band].append(radius) + + rows: list[list[float]] = [] + last_fallback = min_radius + for band in range(band_count): + if band_values[band]: + fallback = max(_quantile(band_values[band], 0.82), min_radius) + last_fallback = fallback + else: + fallback = last_fallback + row: list[float | None] = [] + for segment in range(radial_segments): + values = bins[band][segment] + row.append(max(_quantile(values, 0.88), min_radius) if values else None) + rows.append(_fill_missing_angular_profile(row, fallback)) + + if len(rows) >= 3: + vertical = [row[:] for row in rows] + for band in range(1, len(rows) - 1): + for segment in range(radial_segments): + vertical[band][segment] = ( + rows[band - 1][segment] * 0.18 + + rows[band][segment] * 0.64 + + rows[band + 1][segment] * 0.18 + ) + rows = vertical + + return [_smooth_angular_profile(row, min_radius) for row in rows] + + +def _fill_missing_angular_profile(values: list[float | None], fallback: float) -> list[float]: + if not values: + return [] + if all(value is None for value in values): + return [fallback for _value in values] + filled: list[float] = [] + count = len(values) + for idx, value in enumerate(values): + if value is not None: + filled.append(value) + continue + nearest: list[float] = [] + for step in range(1, count + 1): + left = values[(idx - step) % count] + right = values[(idx + step) % count] + if left is not None: + nearest.append(left) + if right is not None: + nearest.append(right) + if nearest: + break + filled.append(sum(nearest) / len(nearest) if nearest else fallback) + return filled + + +def _smooth_angular_profile(values: list[float], minimum: float) -> list[float]: + if len(values) < 3: + return [max(value, minimum) for value in values] + smoothed = [] + count = len(values) + for idx, value in enumerate(values): + smoothed.append(max(value * 0.8 + values[(idx - 1) % count] * 0.1 + values[(idx + 1) % count] * 0.1, minimum)) + return smoothed + + +def _quantile(values: list[float], quantile: float) -> float: + ordered = sorted(values) + if not ordered: + return 0.0 + index = min(max(int(round((len(ordered) - 1) * quantile)), 0), len(ordered) - 1) + return ordered[index] + + +def _fill_missing_profile(values: list[float | None], minimum: float) -> None: + last: float | None = None + for idx, value in enumerate(values): + if value is not None: + last = value + elif last is not None: + values[idx] = last + next_value: float | None = None + for idx in range(len(values) - 1, -1, -1): + if values[idx] is not None: + next_value = values[idx] + elif next_value is not None: + values[idx] = next_value + for idx, value in enumerate(values): + if value is None: + values[idx] = minimum + + +def _smooth_profile(values: list[float], minimum: float) -> list[float]: + if len(values) < 3: + return [max(value, minimum) for value in values] + smoothed = values[:] + for idx in range(1, len(values) - 1): + smoothed[idx] = (values[idx - 1] + values[idx] * 2.0 + values[idx + 1]) / 4.0 + return [max(value, minimum) for value in smoothed] + + +def _preview_band_colors( + path: Path, + vertices: list[tuple[str, tuple[float, float, float]]], + texcoords: list[str], + faces: list[_ObjFace], + band_count: int, + y_min: float, + y_span: float, +) -> list[tuple[float, float, float]]: + texture_image = _load_obj_diffuse_texture(path) if texcoords else None + bins: list[list[tuple[int, int, int]]] = [[] for _ in range(band_count)] + if texture_image is not None: + width, height = texture_image.size + for face in faces: + center_y = sum(vertices[vertex - 1][1][1] for vertex in face.vertices) / len(face.vertices) + band = min(max(int(((center_y - y_min) / y_span) * band_count), 0), band_count - 1) + for texcoord in face.texcoords: + if texcoord is None or texcoord <= 0 or texcoord > len(texcoords): + continue + parts = texcoords[texcoord - 1].split() + if len(parts) < 3: + continue + u = float(parts[1]) + v = float(parts[2]) + x = min(max(int(u * (width - 1)), 0), width - 1) + y = min(max(int((1.0 - v) * (height - 1)), 0), height - 1) + bins[band].append(texture_image.getpixel((x, y))) + + fallback = tuple(int(channel * 255) for channel in obj_average_color(path)) + colors: list[tuple[float, float, float]] = [] + for pixels in bins: + if pixels: + rgb = tuple(sum(pixel[idx] for pixel in pixels) / (255.0 * len(pixels)) for idx in range(3)) + else: + rgb = tuple(channel / 255.0 for channel in fallback) + colors.append(_visible_preview_color(rgb)) + return colors + + +def _visible_preview_color(color: tuple[float, float, float]) -> tuple[float, float, float]: + brightness = max(sum(color) / 3.0, 1e-6) + target = 0.32 + if brightness < target: + scale = min(target / brightness, 4.5) + color = tuple(min(channel * scale, 1.0) for channel in color) + return tuple(min(max(channel, 0.03), 1.0) for channel in color) + + +def _top_red_cap_material(colors: list[tuple[float, float, float]]) -> tuple[set[int], tuple[float, float, float] | None]: + if len(colors) < 4: + return set(), None + start = max(int(len(colors) * 0.72), 0) + candidates = [ + idx + for idx in range(start, len(colors)) + if _is_red_part_color(colors[idx]) + ] + if not candidates: + return set(), None + first, last = min(candidates), max(candidates) + if last - first > max(2, len(colors) // 5): + first = max(candidates) + while first > start and _is_red_part_color(colors[first - 1]): + first -= 1 + last = max(candidates) + bands = set(range(first, last + 1)) + selected = [colors[idx] for idx in sorted(bands)] + color = tuple(sum(pixel[channel] for pixel in selected) / len(selected) for channel in range(3)) + return bands, _visible_preview_color(color) + + +def _is_red_part_color(color: tuple[float, float, float]) -> bool: + r, g, b = color + return r >= 0.28 and r - max(g, b) >= 0.08 and max(color) - min(color) >= 0.12 + + +def _proxy_part_materials( + colors: list[tuple[float, float, float]], + *, + cap_bands: set[int], + cap_color: tuple[float, float, float] | None, + texture_name: str | None, +) -> tuple[list[str | None], dict[str, _ProxyPartMaterial]]: + band_keys: list[str | None] = [None for _color in colors] + if cap_color is not None: + for band in cap_bands: + if 0 <= band < len(band_keys): + band_keys[band] = "cap" + + for idx, color in enumerate(colors): + if band_keys[idx] is not None: + continue + key = _proxy_part_key(color) + if key is None: + continue + band_keys[idx] = key + + _promote_muted_label_neighbors(colors, band_keys) + + groups: dict[str, list[tuple[float, float, float]]] = {} + for idx, key in enumerate(band_keys): + if key is None: + continue + groups.setdefault(key, []).append(colors[idx]) + + return band_keys, { + key: _proxy_part_material(key, group, map_texture=(texture_name is not None and key != "cap")) + for key, group in groups.items() + } + + +def _proxy_part_key(color: tuple[float, float, float]) -> str | None: + if _is_red_part_color(color): + return "label_red" + if _is_white_label_color(color): + return "label_white" + if _is_dark_part_color(color): + return "dark_body" + if _is_clear_plastic_color(color): + return "clear_plastic" + return None + + +def _is_white_label_color(color: tuple[float, float, float]) -> bool: + brightness = sum(color) / 3.0 + saturation = max(color) - min(color) + return brightness >= 0.64 and saturation <= 0.24 + + +def _is_dark_part_color(color: tuple[float, float, float]) -> bool: + brightness = sum(color) / 3.0 + return brightness <= 0.26 + + +def _is_clear_plastic_color(color: tuple[float, float, float]) -> bool: + brightness = sum(color) / 3.0 + saturation = max(color) - min(color) + return 0.30 <= brightness <= 0.72 and saturation <= 0.16 + + +def _promote_muted_label_neighbors(colors: list[tuple[float, float, float]], band_keys: list[str | None]) -> None: + red_bands = [idx for idx, key in enumerate(band_keys) if key == "label_red"] + if not red_bands: + return + for idx, key in enumerate(band_keys): + if key not in {None, "clear_plastic"}: + continue + if not _is_muted_label_color(colors[idx]): + continue + if any(1 <= abs(idx - red_idx) <= 4 for red_idx in red_bands): + band_keys[idx] = "label_white" + + +def _is_muted_label_color(color: tuple[float, float, float]) -> bool: + brightness = sum(color) / 3.0 + saturation = max(color) - min(color) + return brightness >= 0.38 and saturation <= 0.18 + + +def _proxy_part_material( + key: str, + colors: list[tuple[float, float, float]], + *, + map_texture: bool, +) -> _ProxyPartMaterial: + color = tuple(sum(pixel[channel] for pixel in colors) / len(colors) for channel in range(3)) + color = _visible_preview_color(color) + if key == "clear_plastic": + color = tuple(min(max(channel * 1.25, 0.48), 0.92) for channel in color) + return _ProxyPartMaterial(key=key, color=color, map_texture=map_texture, specular=0.34, shininess=48.0, alpha=0.9) + if key == "label_white": + color = tuple(min(max(channel, 0.82), 1.0) for channel in color) + return _ProxyPartMaterial(key=key, color=color, map_texture=map_texture, specular=0.12, shininess=18.0) + if key == "label_red": + return _ProxyPartMaterial(key=key, color=color, map_texture=map_texture, specular=0.18, shininess=24.0) + if key == "dark_body": + return _ProxyPartMaterial(key=key, color=color, map_texture=map_texture, specular=0.28, shininess=36.0) + return _ProxyPartMaterial(key=key, color=color, map_texture=map_texture, specular=0.18, shininess=24.0) + + +def _write_proxy_texture( + path: Path, + *, + source_obj: Path, + vertices: list[tuple[str, tuple[float, float, float]]], + texcoords: list[str], + faces: list[_ObjFace], + width: int, + height: int, + center_x: float, + center_z: float, + y_min: float, + y_span: float, + fallback_band_colors: list[tuple[float, float, float]], +) -> bool: + try: + from PIL import Image + except Exception: + return False + + texture_image = _load_obj_diffuse_texture(source_obj) if texcoords else None + sums_r = [0.0] * (width * height) + sums_g = [0.0] * (width * height) + sums_b = [0.0] * (width * height) + counts = [0] * (width * height) + + if texture_image is not None: + for face in faces: + color = _face_texture_color(texture_image, texcoords, face) + if color is None: + continue + center = _face_center(vertices, face) + if center is None: + continue + theta = atan2(center[2] - center_z, center[0] - center_x) + if theta < 0.0: + theta += 2.0 * pi + u = theta / (2.0 * pi) + v = min(max((center[1] - y_min) / y_span, 0.0), 1.0) + x = min(max(int(u * (width - 1)), 0), width - 1) + y = min(max(int((1.0 - v) * (height - 1)), 0), height - 1) + for dy in (-1, 0, 1): + yy = y + dy + if yy < 0 or yy >= height: + continue + for dx in (-1, 0, 1): + xx = (x + dx) % width + index = yy * width + xx + sums_r[index] += color[0] + sums_g[index] += color[1] + sums_b[index] += color[2] + counts[index] += 1 + + data: list[tuple[int, int, int]] = [] + for y in range(height): + row: list[tuple[int, int, int] | None] = [] + for x in range(width): + index = y * width + x + if counts[index]: + row.append( + ( + int(round(sums_r[index] / counts[index])), + int(round(sums_g[index] / counts[index])), + int(round(sums_b[index] / counts[index])), + ) + ) + else: + row.append(None) + _fill_proxy_texture_row(row, _proxy_texture_row_fallback(y, height, fallback_band_colors)) + data.extend([pixel if pixel is not None else (184, 184, 184) for pixel in row]) + + data = _auto_expose_proxy_texture_data(data) + image = Image.new("RGB", (width, height)) + image.putdata(data) + image.save(path) + return True + + +def _auto_expose_proxy_texture_data(data: list[tuple[int, int, int]]) -> list[tuple[int, int, int]]: + if not data: + return data + lumas = sorted(_rgb_luminance(pixel) for pixel in data) + p50 = lumas[len(lumas) // 2] + p95 = lumas[min(max(int(len(lumas) * 0.95), 0), len(lumas) - 1)] + if p50 >= 55.0 or p95 >= 95.0: + return data + gain = min(max(72.0 / max(p50, 1.0), 1.0), 4.0) + gamma = 0.72 if p50 < 35.0 else 0.82 + lift = 8.0 if p50 < 35.0 else 4.0 + exposed: list[tuple[int, int, int]] = [] + for pixel in data: + exposed.append(tuple(_auto_expose_channel(channel, gain, gamma, lift) for channel in pixel)) + return exposed + + +def _rgb_luminance(pixel: tuple[int, int, int]) -> float: + r, g, b = pixel + return 0.2126 * r + 0.7152 * g + 0.0722 * b + + +def _auto_expose_channel(value: int, gain: float, gamma: float, lift: float) -> int: + normalized = min(max((value * gain) / 255.0, 0.0), 1.0) + return min(max(int(round((normalized ** gamma) * 255.0 + lift)), 0), 255) + + +def _face_center( + vertices: list[tuple[str, tuple[float, float, float]]], + face: _ObjFace, +) -> tuple[float, float, float] | None: + if not face.vertices: + return None + coords = [vertices[index - 1][1] for index in face.vertices if 0 < index <= len(vertices)] + if not coords: + return None + return tuple(sum(coord[idx] for coord in coords) / len(coords) for idx in range(3)) + + +def _face_texture_color(image, texcoords: list[str], face: _ObjFace) -> tuple[int, int, int] | None: + samples: list[tuple[int, int, int]] = [] + width, height = image.size + for texcoord in face.texcoords: + if texcoord is None or texcoord <= 0 or texcoord > len(texcoords): + continue + parts = texcoords[texcoord - 1].split() + if len(parts) < 3: + continue + u = float(parts[1]) + v = float(parts[2]) + x = min(max(int(u * (width - 1)), 0), width - 1) + y = min(max(int((1.0 - v) * (height - 1)), 0), height - 1) + samples.append(image.getpixel((x, y))) + if not samples: + return None + return tuple(int(round(sum(pixel[idx] for pixel in samples) / len(samples))) for idx in range(3)) + + +def _proxy_texture_row_fallback( + y: int, + height: int, + band_colors: list[tuple[float, float, float]], +) -> tuple[int, int, int]: + if not band_colors: + return (184, 184, 184) + v = 1.0 - (y / max(height - 1, 1)) + band = min(max(int(v * len(band_colors)), 0), len(band_colors) - 1) + return tuple(int(round(channel * 255.0)) for channel in band_colors[band]) + + +def _fill_proxy_texture_row(row: list[tuple[int, int, int] | None], fallback: tuple[int, int, int]) -> None: + last: tuple[int, int, int] | None = None + for idx, value in enumerate(row): + if value is not None: + last = value + elif last is not None: + row[idx] = last + next_value: tuple[int, int, int] | None = None + for idx in range(len(row) - 1, -1, -1): + if row[idx] is not None: + next_value = row[idx] + elif next_value is not None: + row[idx] = next_value + for idx, value in enumerate(row): + if value is None: + row[idx] = fallback + + +def _write_proxy_mtl( + path: Path, + stem: str, + colors: list[tuple[float, float, float]], + *, + texture_name: str | None = None, + part_materials: dict[str, _ProxyPartMaterial] | None = None, +) -> None: + part_materials = part_materials or {} + if texture_name is not None: + lines = [ + "# Generated MotrixSim textured preview proxy material\n", + f"newmtl {stem}_proxy_baked\n", + "Ka 1.000000 1.000000 1.000000\n", + "Kd 1.000000 1.000000 1.000000\n", + "Ks 0.120000 0.120000 0.120000\n", + "Ns 16.000000\n", + "d 1.000000\n", + "illum 2\n", + f"map_Kd {texture_name}\n", + ] + _append_proxy_part_materials(lines, stem, part_materials, texture_name=texture_name) + path.write_text("".join(lines), encoding="utf-8") + return + + lines = ["# Generated MotrixSim solid preview proxy materials\n"] + for idx, color in enumerate(colors): + r, g, b = color + lines.extend( + [ + f"newmtl {stem}_proxy_band_{idx:02d}\n", + f"Ka {r:.6f} {g:.6f} {b:.6f}\n", + f"Kd {r:.6f} {g:.6f} {b:.6f}\n", + "Ks 0.120000 0.120000 0.120000\n", + "Ns 16.000000\n", + "d 1.000000\n", + "illum 2\n\n", + ] + ) + _append_proxy_part_materials(lines, stem, part_materials, texture_name=None) + path.write_text("".join(lines), encoding="utf-8") + + +def _append_proxy_part_materials( + lines: list[str], + stem: str, + materials: dict[str, _ProxyPartMaterial], + *, + texture_name: str | None, +) -> None: + for key in sorted(materials): + material = materials[key] + r, g, b = material.color + lines.extend( + [ + "\n", + f"newmtl {stem}_proxy_{material.key}\n", + f"Ka {r:.6f} {g:.6f} {b:.6f}\n", + f"Kd {r:.6f} {g:.6f} {b:.6f}\n", + f"Ks {material.specular:.6f} {material.specular:.6f} {material.specular:.6f}\n", + f"Ns {material.shininess:.6f}\n", + f"d {material.alpha:.6f}\n", + "illum 2\n", + ] + ) + if texture_name is not None and material.map_texture: + lines.append(f"map_Kd {texture_name}\n") + + +def _write_pymeshlab_preview_lod(path: Path, *, max_faces: int) -> bool: + tmp_obj: Path | None = None + tmp_mtl: Path | None = None + try: + with tempfile.NamedTemporaryFile( + prefix=f".{path.stem}_lod_", + suffix=".obj", + dir=path.parent, + delete=False, + ) as tmp: + tmp_obj = Path(tmp.name) + code = r""" +import sys +import pymeshlab # type: ignore + +source, target, max_faces = sys.argv[1], sys.argv[2], int(sys.argv[3]) +meshset = pymeshlab.MeshSet() +meshset.load_new_mesh(source) +meshset.meshing_decimation_quadric_edge_collapse_with_texture( + targetfacenum=max_faces, + targetperc=0, + qualitythr=0.3, + extratcoordw=1.0, + preserveboundary=True, + boundaryweight=1.0, + optimalplacement=True, + preservenormal=True, + planarquadric=False, + selected=False, +) +if int(meshset.current_mesh().face_number()) > int(max_faces * 1.15): + raise SystemExit(2) +meshset.save_current_mesh(target) +""" + completed = subprocess.run( + [sys.executable, "-c", code, path.as_posix(), tmp_obj.as_posix(), str(int(max_faces))], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=90, + ) + if completed.returncode != 0: + if tmp_obj.exists(): + tmp_obj.unlink() + return False + tmp_mtl = tmp_obj.with_suffix(tmp_obj.suffix + ".mtl") + if tmp_mtl.is_file(): + final_mtl = path.with_name(f"{path.stem}_lod.mtl") + tmp_mtl.replace(final_mtl) + _rewrite_obj_mtllib(tmp_obj, final_mtl.name) + tmp_obj.replace(path) + return True + except Exception: + if tmp_obj is not None and tmp_obj.exists(): + tmp_obj.unlink() + if tmp_mtl is not None and tmp_mtl.exists(): + tmp_mtl.unlink() + return False + + +def _rewrite_obj_mtllib(path: Path, mtl_name: str) -> None: + lines = path.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True) + changed = False + output: list[str] = [] + for line in lines: + if line.startswith("mtllib ") and not changed: + output.append(f"mtllib {mtl_name}\n") + changed = True + else: + output.append(line) + if changed: + path.write_text("".join(output), encoding="utf-8") + + +def _count_obj_faces(path: Path) -> int: + count = 0 + with path.open("r", encoding="utf-8", errors="ignore") as f: + for line in f: + if line.startswith("f "): + count += 1 + return count + + +def ensure_obj_vertex_normals(path: Path) -> dict[str, int]: + """Add smooth vertex normals to an OBJ when face tokens do not reference them.""" + + raw_lines = path.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True) + parsed = _parse_obj(raw_lines) + vertices: list[tuple[str, tuple[float, float, float]]] = parsed["vertices"] + texcoords: list[str] = parsed["texcoords"] + faces: list[_ObjFace] = parsed["faces"] + if not vertices or not faces: + return {"vertices": len(vertices), "faces": len(faces), "normals_written": 0} + if all(all(normal is not None for normal in face.normals) for face in faces): + return {"vertices": len(vertices), "faces": len(faces), "normals_written": 0} + + accum = [(0.0, 0.0, 0.0) for _ in range(len(vertices) + 1)] + for face in faces: + if len(face.vertices) < 3: + continue + origin = vertices[face.vertices[0] - 1][1] + for idx in range(1, len(face.vertices) - 1): + a = vertices[face.vertices[idx] - 1][1] + b = vertices[face.vertices[idx + 1] - 1][1] + normal = _cross(_sub(a, origin), _sub(b, origin)) + for vertex_index in (face.vertices[0], face.vertices[idx], face.vertices[idx + 1]): + accum[vertex_index] = _add(accum[vertex_index], normal) + + normal_lines: list[str] = [] + for idx in range(1, len(vertices) + 1): + nx, ny, nz = _normalise_vector(accum[idx]) + normal_lines.append(f"vn {nx:.8f} {ny:.8f} {nz:.8f}\n") + + output: list[str] = [] + header: list[str] = parsed["header"] + if header: + output.extend(header) + else: + output.append(f"o {_slug(path.stem)}\n") + output.extend(line for line, _ in vertices) + output.extend(texcoords) + output.extend(normal_lines) + current_material: str | None = None + for face in faces: + if face.material != current_material: + if face.material: + output.append(f"usemtl {face.material}\n") + current_material = face.material + output.append("f " + " ".join(_face_token_with_vertex_normal(face, idx) for idx in range(len(face.vertices))) + "\n") + path.write_text("".join(output), encoding="utf-8") + return {"vertices": len(vertices), "faces": len(faces), "normals_written": len(vertices)} + + +def _sub(a: tuple[float, float, float], b: tuple[float, float, float]) -> tuple[float, float, float]: + return (a[0] - b[0], a[1] - b[1], a[2] - b[2]) + + +def _add(a: tuple[float, float, float], b: tuple[float, float, float]) -> tuple[float, float, float]: + return (a[0] + b[0], a[1] + b[1], a[2] + b[2]) + + +def _cross(a: tuple[float, float, float], b: tuple[float, float, float]) -> tuple[float, float, float]: + return ( + a[1] * b[2] - a[2] * b[1], + a[2] * b[0] - a[0] * b[2], + a[0] * b[1] - a[1] * b[0], + ) + + +def _normalise_vector(vector: tuple[float, float, float]) -> tuple[float, float, float]: + length = sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]) + if length <= 1e-12: + return (0.0, 0.0, 1.0) + return (vector[0] / length, vector[1] / length, vector[2] / length) + + +def _face_token_with_vertex_normal(face: _ObjFace, token_index: int) -> str: + vertex = face.vertices[token_index] + texcoord = face.texcoords[token_index] + normal = vertex + if texcoord is None: + return f"{vertex}//{normal}" + return f"{vertex}/{texcoord}/{normal}" + + +def _parse_obj(lines: list[str]) -> dict[str, object]: + header: list[str] = [] + vertices: list[tuple[str, tuple[float, float, float]]] = [] + texcoords: list[str] = [] + normals: list[str] = [] + faces: list[_ObjFace] = [] + current_material: str | None = None + for line in lines: + if line.startswith("v "): + parts = line.split() + if len(parts) >= 4: + vertices.append((line, (float(parts[1]), float(parts[2]), float(parts[3])))) + elif line.startswith("vt "): + texcoords.append(line) + elif line.startswith("vn "): + normals.append(line) + elif line.startswith("usemtl "): + current_material = line.split(maxsplit=1)[1].strip() + elif line.startswith("f "): + tokens = tuple(line.split()[1:]) + face = _parse_obj_face(tokens, len(vertices), len(texcoords), len(normals), current_material) + if face is not None: + faces.append(face) + elif line.startswith(("mtllib ", "o ", "g ", "s ")): + header.append(line) + face_roots = _obj_face_roots(faces, len(vertices)) + return { + "header": header, + "vertices": vertices, + "texcoords": texcoords, + "normals": normals, + "faces": faces, + "face_roots": face_roots, + } + + +def _parse_obj_face( + tokens: tuple[str, ...], + vertex_count: int, + texcoord_count: int, + normal_count: int, + material: str | None, +) -> _ObjFace | None: + vertices: list[int] = [] + texcoords: list[int | None] = [] + normals: list[int | None] = [] + for token in tokens: + parts = token.split("/") + vertex = _parse_obj_index(parts[0], vertex_count) + if vertex is None: + return None + texcoord = _parse_obj_index(parts[1], texcoord_count) if len(parts) >= 2 and parts[1] else None + normal = _parse_obj_index(parts[2], normal_count) if len(parts) >= 3 and parts[2] else None + vertices.append(vertex) + texcoords.append(texcoord) + normals.append(normal) + return _ObjFace(tokens=tokens, vertices=tuple(vertices), texcoords=tuple(texcoords), normals=tuple(normals), material=material) + + +def _select_lod_faces(faces: list[_ObjFace], vertex_count: int, *, max_faces: int) -> list[_ObjFace]: + components = _obj_face_components(faces, vertex_count) + component_faces = sorted(components.values(), key=len, reverse=True) + if len(component_faces) >= max_faces: + return [faces[indices[0]] for indices in component_faces[:max_faces]] + + allocations = [1] * len(component_faces) + remaining = max_faces - len(component_faces) + weights = [max(len(indices) - 1, 0) for indices in component_faces] + total_weight = sum(weights) + if remaining > 0 and total_weight > 0: + fractional: list[tuple[float, int]] = [] + for idx, weight in enumerate(weights): + exact = remaining * weight / total_weight + extra = int(exact) + allocations[idx] += extra + fractional.append((exact - extra, idx)) + assigned = sum(allocations) + for _, idx in sorted(fractional, reverse=True): + if assigned >= max_faces: + break + allocations[idx] += 1 + assigned += 1 + + selected_indices: list[int] = [] + for indices, count in zip(component_faces, allocations): + selected_indices.extend(_evenly_spaced_indices(indices, min(count, len(indices)))) + return [faces[idx] for idx in sorted(selected_indices[:max_faces])] + + +def _evenly_spaced_indices(values: list[int], count: int) -> list[int]: + if count <= 0: + return [] + if count >= len(values): + return values + if count == 1: + return [values[len(values) // 2]] + last = len(values) - 1 + selected = [] + for idx in range(count): + selected.append(values[round(idx * last / (count - 1))]) + return selected + + +def _parse_obj_index(raw: str, count: int) -> int | None: + try: + value = int(raw) + except ValueError: + return None + if value < 0: + value = count + value + 1 + if value <= 0 or value > count: + return None + return value + + +def _obj_face_roots(faces: list[_ObjFace], vertex_count: int) -> list[int]: + components = _obj_face_components(faces, vertex_count) + root_by_face: dict[int, int] = {} + for root, face_indices in components.items(): + for face_index in face_indices: + root_by_face[face_index] = root + return [root_by_face[idx] for idx in range(len(faces))] + + +def _obj_face_components(faces: list[_ObjFace], vertex_count: int) -> dict[int, list[int]]: + parent = list(range(vertex_count + 1)) + size = [1] * (vertex_count + 1) + + def find(value: int) -> int: + while parent[value] != value: + parent[value] = parent[parent[value]] + value = parent[value] + return value + + def union(a: int, b: int) -> None: + root_a = find(a) + root_b = find(b) + if root_a == root_b: + return + if size[root_a] < size[root_b]: + root_a, root_b = root_b, root_a + parent[root_b] = root_a + size[root_a] += size[root_b] + + for face in faces: + first = face.vertices[0] + for vertex in face.vertices[1:]: + union(first, vertex) + + components: dict[int, list[int]] = {} + for face_index, face in enumerate(faces): + components.setdefault(find(face.vertices[0]), []).append(face_index) + return components + + +def _vertex_bounds( + vertices: list[tuple[str, tuple[float, float, float]]], + vertex_indices: list[int], +) -> tuple[tuple[float, float, float], tuple[float, float, float]] | None: + if not vertex_indices: + return None + coords = [vertices[index - 1][1] for index in vertex_indices] + mins = tuple(min(coord[idx] for coord in coords) for idx in range(3)) + maxs = tuple(max(coord[idx] for coord in coords) for idx in range(3)) + return mins, maxs + + +def _write_compacted_obj( + path: Path, + header: list[str], + vertices: list[tuple[str, tuple[float, float, float]]], + texcoords: list[str], + normals: list[str], + faces: list[_ObjFace], +) -> None: + used_vertices = sorted({vertex for face in faces for vertex in face.vertices}) + used_texcoords = sorted({texcoord for face in faces for texcoord in face.texcoords if texcoord is not None}) + used_normals = sorted({normal for face in faces for normal in face.normals if normal is not None}) + vertex_map = {old: new for new, old in enumerate(used_vertices, start=1)} + texcoord_map = {old: new for new, old in enumerate(used_texcoords, start=1)} + normal_map = {old: new for new, old in enumerate(used_normals, start=1)} + + output: list[str] = [] + if header: + output.extend(header) + else: + output.append(f"o {_slug(path.stem)}\n") + for old_index in used_vertices: + output.append(vertices[old_index - 1][0]) + for old_index in used_texcoords: + output.append(texcoords[old_index - 1]) + for old_index in used_normals: + output.append(normals[old_index - 1]) + current_material: str | None = None + for face in faces: + if face.material != current_material: + if face.material: + output.append(f"usemtl {face.material}\n") + current_material = face.material + output.append("f " + " ".join(_rewrite_face_token(face, idx, vertex_map, texcoord_map, normal_map) for idx in range(len(face.vertices))) + "\n") + path.write_text("".join(output), encoding="utf-8") + + +def _rewrite_face_token( + face: _ObjFace, + token_index: int, + vertex_map: dict[int, int], + texcoord_map: dict[int, int], + normal_map: dict[int, int], +) -> str: + vertex = str(vertex_map[face.vertices[token_index]]) + texcoord = face.texcoords[token_index] + normal = face.normals[token_index] + if texcoord is None and normal is None: + return vertex + if normal is None: + return f"{vertex}/{texcoord_map[texcoord]}" if texcoord is not None else vertex + if texcoord is None: + return f"{vertex}//{normal_map[normal]}" + return f"{vertex}/{texcoord_map[texcoord]}/{normal_map[normal]}" + + +def materialize_obj_materials(path: Path, *, color_levels: int = 6) -> Path | None: + """Convert OBJ vertex colors into simple MTL face materials. + + SAM-3D meshes are exported with non-standard ``v x y z r g b`` vertex + colors. Some mesh viewers use those colors directly, but Motrix currently + reports zero loaded materials for such OBJ files. This helper preserves the + original OBJ geometry and adds a small MTL palette so Motrix has explicit + materials to load. + """ + + lines = path.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True) + if not lines or any(line.startswith("mtllib ") for line in lines): + return None + + vertex_colors: dict[int, tuple[float, float, float]] = {} + vertex_count = 0 + for line in lines: + if not line.startswith("v "): + continue + vertex_count += 1 + parts = line.split() + if len(parts) >= 7: + vertex_colors[vertex_count] = _normalise_rgb(parts[4:7]) + + face_materials: dict[int, str] = {} + materials: dict[str, tuple[float, float, float]] = {} + default_name = f"{_slug(path.stem)}_mat_default" + default_color = (0.72, 0.72, 0.72) + + for idx, line in enumerate(lines): + if not line.startswith("f "): + continue + color = _face_average_color(line, vertex_colors, vertex_count) or default_color + name, quantized = _quantized_material(path.stem, color, color_levels) + face_materials[idx] = name + materials.setdefault(name, quantized) + + if not face_materials: + return None + if not materials: + materials[default_name] = default_color + + mtl_path = path.with_suffix(".mtl") + _write_mtl(mtl_path, materials) + + output = [f"mtllib {mtl_path.name}\n"] + current_material = None + for idx, line in enumerate(lines): + material = face_materials.get(idx) + if material is not None and material != current_material: + output.append(f"usemtl {material}\n") + current_material = material + output.append(line) + path.write_text("".join(output), encoding="utf-8") + return mtl_path + + +def auto_expose_obj_diffuse_textures(path: Path) -> dict[str, int]: + """Brighten very dark baked diffuse textures referenced by an OBJ. + + SAM-3D baked textures for glossy black objects can be near-black under + MotrixSim's default preview lighting. The adjustment is gated by luminance + percentiles, so normal-bright textures are left unchanged. + """ + + changed = 0 + checked = 0 + for texture in _iter_obj_diffuse_textures(path): + checked += 1 + if _auto_expose_texture_file(texture): + changed += 1 + return {"textures_checked": checked, "textures_changed": changed} + + +def _iter_obj_diffuse_textures(path: Path) -> list[Path]: + textures: list[Path] = [] + seen: set[Path] = set() + obj_dir = path.parent + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + if not line.startswith("mtllib "): + continue + for raw_mtl in line.split()[1:]: + mtl_path = obj_dir / raw_mtl + if not mtl_path.is_file(): + continue + for mtl_line in mtl_path.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = mtl_line.split() + if len(parts) < 2 or parts[0] != "map_Kd": + continue + texture = (mtl_path.parent / parts[-1]).resolve() + if texture.is_file() and texture not in seen: + textures.append(texture) + seen.add(texture) + return textures + + +def _auto_expose_texture_file(path: Path) -> bool: + try: + from PIL import Image + + with Image.open(path).convert("RGB") as image: + pixel_data = getattr(image, "get_flattened_data", image.getdata) + data = list(pixel_data()) + exposed = _auto_expose_proxy_texture_data(data) + if exposed == data: + return False + output = Image.new("RGB", image.size) + output.putdata(exposed) + output.save(path) + return True + except Exception: + return False + + +def copy_obj_material_sidecars(source_obj: Path, packaged_obj: Path) -> None: + lines = packaged_obj.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True) + output: list[str] = [] + changed = False + for line in lines: + if not line.startswith("mtllib "): + output.append(line) + continue + names = line.split()[1:] + copied_names = [] + for raw_name in names: + src = (source_obj.parent / raw_name).resolve() + if not src.is_file(): + copied_names.append(raw_name) + continue + dst_name = namespaced_sidecar_name(packaged_obj.stem, raw_name) + dst = packaged_obj.parent / dst_name + dst.parent.mkdir(parents=True, exist_ok=True) + copy_mtl_with_texture_sidecars(src, dst, namespace=packaged_obj.stem) + copied_names.append(dst_name) + changed = True + output.append(f"mtllib {' '.join(copied_names)}\n") + if changed: + packaged_obj.write_text("".join(output), encoding="utf-8") + + +def copy_mtl_with_texture_sidecars(source_mtl: Path, packaged_mtl: Path, *, namespace: str) -> None: + texture_keys = { + "map_Ka", + "map_Kd", + "map_Ks", + "map_Ke", + "map_Ns", + "map_d", + "map_bump", + "bump", + "disp", + "decal", + "norm", + } + output: list[str] = [] + for line in source_mtl.read_text(encoding="utf-8", errors="ignore").splitlines(keepends=True): + parts = line.split() + if len(parts) < 2 or parts[0] not in texture_keys: + output.append(line) + continue + raw_name = parts[-1] + if raw_name.startswith("-"): + output.append(line) + continue + src = (source_mtl.parent / raw_name).resolve() + if not src.is_file(): + output.append(line) + continue + dst_name = namespaced_sidecar_name(namespace, raw_name) + dst = packaged_mtl.parent / dst_name + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + parts[-1] = dst_name + output.append(" ".join(parts) + "\n") + packaged_mtl.write_text("".join(output), encoding="utf-8") + + +def namespaced_sidecar_name(namespace: str, raw_name: str) -> str: + name = Path(raw_name).name + if name.startswith(f"{namespace}_"): + return name + return f"{namespace}_{name}" + + +def _face_average_color( + face_line: str, + vertex_colors: dict[int, tuple[float, float, float]], + vertex_count: int, +) -> tuple[float, float, float] | None: + colors = [] + for token in face_line.split()[1:]: + raw_index = token.split("/", 1)[0] + if not raw_index: + continue + try: + vertex_index = int(raw_index) + except ValueError: + continue + if vertex_index < 0: + vertex_index = vertex_count + vertex_index + 1 + color = vertex_colors.get(vertex_index) + if color is not None: + colors.append(color) + if not colors: + return None + return tuple(sum(color[channel] for color in colors) / len(colors) for channel in range(3)) + + +def _normalise_rgb(values: list[str]) -> tuple[float, float, float]: + rgb = [float(value) for value in values[:3]] + if max(rgb) > 1.0: + rgb = [value / 255.0 for value in rgb] + return tuple(min(max(value, 0.0), 1.0) for value in rgb) + + +def _quantized_material( + stem: str, + color: tuple[float, float, float], + levels: int, +) -> tuple[str, tuple[float, float, float]]: + levels = max(int(levels), 2) + quantized = tuple(round(channel * (levels - 1)) / (levels - 1) for channel in color) + suffix = "_".join(f"{int(round(channel * 255)):02x}" for channel in quantized) + return f"{_slug(stem)}_mat_{suffix}", quantized + + +def _write_mtl(path: Path, materials: dict[str, tuple[float, float, float]]) -> None: + lines = ["# Generated from OBJ vertex colors for MotrixSim preview\n"] + for name, color in sorted(materials.items()): + r, g, b = color + lines.extend( + [ + f"newmtl {name}\n", + f"Ka {r:.6f} {g:.6f} {b:.6f}\n", + f"Kd {r:.6f} {g:.6f} {b:.6f}\n", + "Ks 0.050000 0.050000 0.050000\n", + "Ns 12.000000\n", + "d 1.000000\n", + "illum 2\n", + "\n", + ] + ) + path.write_text("".join(lines), encoding="utf-8") + + +def _slug(value: str) -> str: + slug = re.sub(r"[^A-Za-z0-9_]+", "_", value).strip("_").lower() + return slug or "asset" + + +def _relative_posix(path: Path, base_dir: Path) -> str: + return Path(os.path.relpath(path.resolve(), base_dir.resolve())).as_posix() + + +def _vec(values: Iterable[float]) -> str: + return " ".join(f"{float(v):.8g}" for v in values) + + +def _rgba(color: Iterable[float], alpha: float = 1.0) -> str: + return _vec([*color, alpha]) + + +def _is_unit_scale(values: Iterable[float]) -> bool: + return all(abs(float(v) - 1.0) < 1e-8 for v in values) + + +def _indent_xml(elem: ET.Element, level: int = 0) -> None: + indent = "\n" + level * " " + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = indent + " " + for child in elem: + _indent_xml(child, level + 1) + if not child.tail or not child.tail.strip(): + child.tail = indent + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = indent diff --git a/vendor/gs_playground/real2sim/paper_strict.py b/vendor/gs_playground/real2sim/paper_strict.py new file mode 100644 index 0000000..9ad21ff --- /dev/null +++ b/vendor/gs_playground/real2sim/paper_strict.py @@ -0,0 +1,320 @@ +"""Paper-strict Real2Sim runtime validation.""" + +from __future__ import annotations + +import importlib.util as importlib_util +import os +import shlex +import shutil +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from .commands import command_contains_placeholder, command_preferring_env +from .errors import PipelineError + + +REQUIRED_STAGES = ( + "grounding_dino", + "sam2", + "instance_refinement", + "lama", + "anysplat", + "sam3d", + "pose_alignment", +) +MIN_SAM3D_VRAM_MIB = 32768 +REQUIRED_STAGE_PYTHON_MODULES = { + "grounding_dino": ("transformers",), + "sam2": ("sam2",), + "instance_refinement": ("numpy", "PIL"), + "lama": ("simple_lama_inpainting",), + "anysplat": ("torch", "pytorch3d", "gsplat", "plyfile", "src.model.model.anysplat", "src.model.ply_export"), + "sam3d": ("torch", "pytorch3d", "kaolin", "gsplat", "sam3d_objects", "inference"), + "pose_alignment": ("numpy", "PIL"), + "speedy_splat": ("gaussian_renderer",), +} + + +def validate_paper_strict( + config: Mapping[str, Any], + repo_root: Path, + *, + env: Mapping[str, str] | None = None, + gpu_memory_mib: int | None = None, +) -> dict[str, Any]: + report = check_paper_strict(config, repo_root, env=env, gpu_memory_mib=gpu_memory_mib) + if report["ok"]: + return report + summary = "; ".join(f"{failure['name']}: {failure['message']}" for failure in report["failures"]) + raise PipelineError(f"paper-strict Real2Sim prerequisites are not satisfied: {summary}") + + +def check_paper_strict( + config: Mapping[str, Any], + repo_root: Path, + *, + env: Mapping[str, str] | None = None, + gpu_memory_mib: int | None = None, +) -> dict[str, Any]: + env_map = dict(os.environ if env is None else env) + failures: list[dict[str, str]] = [] + checks: list[dict[str, Any]] = [] + + stages = config.get("stages", {}) if isinstance(config.get("stages", {}), Mapping) else {} + for stage in REQUIRED_STAGES: + command = configured_stage_command(stages, stage, env_map) + ok = bool(command) + checks.append({"name": f"stage_{stage}", "ok": ok}) + if not ok: + failures.append({"name": f"stage_{stage}", "message": f"missing command for {stage}"}) + elif command_contains_placeholder(command): + failures.append({"name": f"stage_{stage}", "message": f"command for {stage} still contains a placeholder path"}) + else: + executable_error = command_executable_error(command) + if executable_error: + failures.append({"name": f"stage_{stage}", "message": executable_error}) + + pruning = config.get("pruning", {}) if isinstance(config.get("pruning", {}), Mapping) else {} + pruning_mode = str(pruning.get("mode", "")).replace("-", "_").lower() + speedy_command = command_preferring_env(pruning.get("external_command"), env_map.get("REAL2SIM_SPEEDY_SPLAT_COMMAND", "")) + if pruning_mode != "speedy_splat": + failures.append({"name": "speedy_splat_mode", "message": "pruning.mode must be speedy_splat"}) + if not speedy_command: + failures.append({"name": "speedy_splat_command", "message": "missing Speedy-Splat external command"}) + elif command_contains_placeholder(speedy_command): + failures.append({"name": "speedy_splat_command", "message": "Speedy-Splat command still contains a placeholder path"}) + else: + executable_error = command_executable_error(speedy_command) + if executable_error: + failures.append({"name": "speedy_splat_command", "message": executable_error}) + + nested_grounding = configured_stage_command(stages, "grounding_dino", env_map) + nested_sam2 = configured_stage_command(stages, "sam2", env_map) + if not nested_grounding or not nested_sam2: + failures.append( + { + "name": "inpainting_redetect_loop", + "message": "strict LaMa recovery requires configured GroundingDINO and SAM2 commands", + } + ) + + anysplat_root = Path(env_map.get("REAL2SIM_ANYSPLAT_ROOT", "")) + require_path(failures, "anysplat_root", anysplat_root, "REAL2SIM_ANYSPLAT_ROOT") + if anysplat_root: + require_path(failures, "anysplat_api", anysplat_root / "src" / "model" / "model" / "anysplat.py", "AnySplat API") + require_path(failures, "anysplat_export", anysplat_root / "src" / "model" / "ply_export.py", "AnySplat export_ply") + + sam3d_root = Path(env_map.get("REAL2SIM_SAM3D_ROOT", "")) + require_path(failures, "sam3d_root", sam3d_root, "REAL2SIM_SAM3D_ROOT") + if sam3d_root: + require_path(failures, "sam3d_inference", sam3d_root / "notebook" / "inference.py", "SAM-3D notebook inference API") + sam3d_config = Path(env_map.get("REAL2SIM_SAM3D_CONFIG", "")) if env_map.get("REAL2SIM_SAM3D_CONFIG") else sam3d_root / "checkpoints" / "hf" / "pipeline.yaml" + require_path(failures, "sam3d_config", sam3d_config, "SAM-3D checkpoint pipeline.yaml") + + sam2_checkpoint = Path(env_map.get("REAL2SIM_SAM2_CHECKPOINT", "")) + require_path(failures, "sam2_checkpoint", sam2_checkpoint, "REAL2SIM_SAM2_CHECKPOINT") + speedy_root = Path(env_map.get("REAL2SIM_SPEEDY_SPLAT_ROOT", "")) + require_path(failures, "speedy_splat_root", speedy_root, "REAL2SIM_SPEEDY_SPLAT_ROOT") + + module_checks = [] + for stage, module_names in REQUIRED_STAGE_PYTHON_MODULES.items(): + command = speedy_command if stage == "speedy_splat" else configured_stage_command(stages, stage, env_map) + if not command: + continue + if command_contains_placeholder(command) or command_executable_error(command): + continue + python_executable = python_executable_for_command(command) + extra_paths = stage_module_extra_paths(stage, env_map) + for module_name in module_names: + ok = module_available(module_name, python_executable, repo_root, env_map, extra_paths=extra_paths) + module_checks.append( + { + "stage": stage, + "name": module_name, + "ok": ok, + "python": python_executable, + "extra_paths": list(extra_paths), + } + ) + if not ok: + failures.append( + { + "name": f"python_module_{module_name}", + "message": ( + f"missing required Python module for paper-strict Real2Sim stage " + f"{stage}: {module_name} in {python_executable}" + ), + } + ) + + detected_vram = detect_gpu_memory_mib() if gpu_memory_mib is None else gpu_memory_mib + if detected_vram is None: + failures.append({"name": "sam3d_vram", "message": "could not determine NVIDIA GPU memory"}) + elif detected_vram < MIN_SAM3D_VRAM_MIB: + failures.append( + { + "name": "sam3d_vram", + "message": f"SAM-3D official setup requires at least {MIN_SAM3D_VRAM_MIB} MiB VRAM, found {detected_vram} MiB", + } + ) + + return { + "ok": not failures, + "checks": checks, + "python_modules": module_checks, + "failures": failures, + "gpu_memory_mib": detected_vram, + "required_stages": list(REQUIRED_STAGES), + } + + +def stage_env(stage: str) -> str: + return { + "grounding_dino": "REAL2SIM_GROUNDING_DINO_COMMAND", + "sam2": "REAL2SIM_SAM2_COMMAND", + "instance_refinement": "REAL2SIM_INSTANCE_REFINEMENT_COMMAND", + "lama": "REAL2SIM_LAMA_COMMAND", + "anysplat": "REAL2SIM_ANYSPLAT_COMMAND", + "sam3d": "REAL2SIM_SAM3D_COMMAND", + "pose_alignment": "REAL2SIM_POSE_ALIGNMENT_COMMAND", + }[stage] + + +def configured_stage_command(stages: Mapping[str, Any], stage: str, env_map: Mapping[str, str]) -> Any: + stage_config = stages.get(stage, {}) if isinstance(stages, Mapping) else {} + command = stage_config.get("command") if isinstance(stage_config, Mapping) else None + return command_preferring_env(command, env_map.get(stage_env(stage), "")) + + +def require_path(failures: list[dict[str, str]], name: str, path: Path, description: str) -> None: + if not str(path) or str(path) == "." or not path.exists(): + failures.append({"name": name, "message": f"missing {description}: {path}"}) + + +def command_executable_error(command: Any) -> str | None: + executable = command_executable(command) + if not executable or ("{" in executable and "}" in executable): + return None + if os.sep in executable or (os.altsep and os.altsep in executable): + path = Path(executable) + if not path.exists(): + return f"command executable does not exist: {executable}" + if path.is_dir(): + return f"command executable is a directory: {executable}" + if not os.access(path, os.X_OK): + return f"command executable is not executable: {executable}" + return None + if shutil.which(executable) is None: + return f"command executable not found on PATH: {executable}" + return None + + +def command_executable(command: Any) -> str | None: + if command is None: + return None + if isinstance(command, str): + try: + parts = shlex.split(command) + except ValueError: + return "" + elif isinstance(command, (list, tuple)): + parts = [str(part) for part in command] + else: + parts = [str(command)] + return parts[0] if parts else None + + +def python_executable_for_command(command: Any) -> str: + executable = command_executable(command) + if not executable or ("{" in executable and "}" in executable): + return sys.executable + return executable + + +def stage_module_extra_paths(stage: str, env_map: Mapping[str, str]) -> tuple[str, ...]: + paths: list[str] = [] + if stage == "anysplat": + root = env_map.get("REAL2SIM_ANYSPLAT_ROOT", "") + if root: + paths.append(root) + elif stage == "sam3d": + root = env_map.get("REAL2SIM_SAM3D_ROOT", "") + if root: + paths.extend([(Path(root) / "notebook").as_posix(), root]) + elif stage == "speedy_splat": + root = env_map.get("REAL2SIM_SPEEDY_SPLAT_ROOT", "") + if root: + paths.append(root) + return tuple(paths) + + +def module_available( + module_name: str, + python_executable: str, + repo_root: Path, + env_map: Mapping[str, str], + *, + extra_paths: tuple[str, ...] = (), +) -> bool: + if python_executable == sys.executable and not extra_paths and not env_map.get("PYTHONPATH"): + try: + return importlib_util.find_spec(module_name) is not None + except (ImportError, AttributeError, ValueError): + return False + + module_env = dict(os.environ) + module_env.update(env_map) + module_env.setdefault("LIDRA_SKIP_INIT", "1") + pythonpath_parts = [path for path in extra_paths if path] + if module_env.get("PYTHONPATH"): + pythonpath_parts.append(module_env["PYTHONPATH"]) + if pythonpath_parts: + module_env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts) + code = "import importlib, sys; importlib.import_module(sys.argv[1])" + try: + completed = subprocess.run( + [python_executable, "-c", code, module_name], + cwd=repo_root, + env=module_env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (FileNotFoundError, NotADirectoryError, PermissionError, subprocess.TimeoutExpired): + return False + return completed.returncode == 0 + + +def detect_gpu_memory_mib() -> int | None: + try: + completed = subprocess.run( + ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError: + return None + if completed.returncode != 0: + return None + values = [] + for line in completed.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + values.append(int(float(line.split()[0]))) + except ValueError: + continue + return max(values) if values else None + + +def format_report(report: Mapping[str, Any]) -> list[str]: + lines = [f"paper_strict: {'ok' if report.get('ok') else 'failed'}"] + for failure in report.get("failures", []): + lines.append(f"{failure['name']}: {failure['message']}") + return lines diff --git a/vendor/gs_playground/real2sim/pipeline.py b/vendor/gs_playground/real2sim/pipeline.py new file mode 100644 index 0000000..a09bac4 --- /dev/null +++ b/vendor/gs_playground/real2sim/pipeline.py @@ -0,0 +1,2819 @@ +"""Automated Real2Sim pipeline orchestration. + +This follows the GS-Playground paper structure: + +1. GroundingDINO instance detection. +2. SAM/SAM2 instance segmentation. +3. Visual instance de-duplication/refinement. +4. LaMa background inpainting. +5. SAM-3D object reconstruction. +6. AnySplat scene reconstruction. +7. Speedy-Splat-compatible 3DGS pruning. +8. GS-Playground MJCF/config package creation. + +The heavy model stages are intentionally external command adapters. The public +preview repository does not ship those model checkpoints or code, so a run only +succeeds when real tools are configured and produce the declared artifacts. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +from .commands import StageCommand, command_preferring_env, resolve_command, run_command +from .builtin import run_builtin_real2sim +from .contracts import ( + Pose, + Real2SimAsset, + Real2SimManifest, + load_instances, + read_json, + require_file, + validate_detections, + write_json, +) +from .errors import MissingStageCommand, PipelineError, StageArtifactError +from .image_io import read_rgb, write_gray, write_rgb +from .packaging import PackageResult, PackageWriter +from .pruning import PruneOptions, prune_ply + + +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"} +VIDEO_SUFFIXES = {".mp4", ".mov", ".mkv", ".avi", ".webm"} + + +STAGES = { + "synthetic_multiview": StageCommand( + "synthetic_multiview", + "REAL2SIM_SYNTHETIC_MULTIVIEW_COMMAND", + "optional single-image to multi-view expansion; must write image frames under {synthetic_frames_dir}", + required=False, + ), + "grounding_dino": StageCommand( + "grounding_dino", + "REAL2SIM_GROUNDING_DINO_COMMAND", + "open-vocabulary detection; must write {detections_json}", + ), + "sam2": StageCommand( + "sam2", + "REAL2SIM_SAM2_COMMAND", + "instance segmentation/tracking; must write {instances_json} and masks under {masks_dir}", + ), + "instance_refinement": StageCommand( + "instance_refinement", + "REAL2SIM_INSTANCE_REFINEMENT_COMMAND", + "visual mask de-duplication and confidence ranking; may update {instances_json}", + ), + "lama": StageCommand( + "lama", + "REAL2SIM_LAMA_COMMAND", + "sequential background inpainting; must write {background_image}", + ), + "depth_estimation": StageCommand( + "depth_estimation", + "REAL2SIM_DEPTH_COMMAND", + "optional monocular depth estimation; should write {depth_map}", + required=False, + ), + "anysplat": StageCommand( + "anysplat", + "REAL2SIM_ANYSPLAT_COMMAND", + "scene-level 3DGS reconstruction; must write {scene_ply}", + ), + "sam3d": StageCommand( + "sam3d", + "REAL2SIM_SAM3D_COMMAND", + "object-level 3DGS/mesh/pose reconstruction; must write per-object gaussian.ply, mesh.obj, pose.json", + ), + "pose_alignment": StageCommand( + "pose_alignment", + "REAL2SIM_POSE_ALIGNMENT_COMMAND", + "optional pose/scale refinement; may update {aligned_assets_json}", + required=False, + ), +} + + +@dataclass(frozen=True) +class CaptureInfo: + source: Path + image: Path + frames_dir: Path | None + kind: str + + +@dataclass(frozen=True) +class PipelineRunResult: + work_dir: Path + package: PackageResult + manifest: Real2SimManifest + + +class Real2SimPipeline: + def __init__( + self, + *, + repo_root: Path, + config: Mapping[str, Any] | None = None, + robot: str = "go2", + dynamic_objects: bool = False, + dry_run: bool = False, + backend: str = "auto", + ) -> None: + self.repo_root = repo_root.resolve() + self.config = dict(config or {}) + self.robot = robot + self.dynamic_objects = dynamic_objects + self.dry_run = dry_run + self.backend = backend + + @classmethod + def from_config_file( + cls, + *, + repo_root: Path, + config_file: Path | None, + robot: str, + dynamic_objects: bool, + dry_run: bool = False, + backend: str = "auto", + ) -> "Real2SimPipeline": + config = read_json(config_file) if config_file else {} + if not isinstance(config, Mapping): + raise PipelineError(f"Real2Sim config must be a JSON object: {config_file}") + return cls( + repo_root=repo_root, + config=config, + robot=robot, + dynamic_objects=dynamic_objects, + dry_run=dry_run, + backend=backend, + ) + + def run( + self, + *, + input_path: Path, + output_dir: Path, + prompt: str, + work_dir: Path | None = None, + resume: bool = False, + prune_options: PruneOptions | None = None, + synthetic_multiview: bool = False, + synthetic_multiview_mode: str = "object", + enable_scene_reconstruction: bool = True, + enable_object_reconstruction: bool = True, + ) -> PipelineRunResult: + work = (work_dir or (output_dir / "_work")).resolve() + output = output_dir.resolve() + if not enable_scene_reconstruction and not work.exists(): + raise PipelineError( + "object-only mode requires an existing work-dir containing reconstructed scene artifacts; " + "provide a real output/work directory via --work-dir (or set --pipeline-mode scene-only first)." + ) + if work.exists() and not resume: + shutil.rmtree(work) + work.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=True) + + capture = self.prepare_capture(input_path.resolve(), work) + context = self._base_context(capture, work, output, prompt, synthetic_multiview_mode=synthetic_multiview_mode) + capture = self._run_synthetic_multiview_expansion(capture, context, resume=resume, enabled=synthetic_multiview) + + if self.backend == "hybrid" and not enable_scene_reconstruction: + raise PipelineError("hybrid backend cannot run object-only mode; use backend=external with cached scene artifacts") + if self.backend == "hybrid": + return self._run_hybrid_backend( + capture=capture, + context=context, + prompt=prompt, + work=work, + output=output, + resume=resume, + prune_options=prune_options, + enable_object_reconstruction=enable_object_reconstruction, + ) + + if self._should_use_builtin_backend(): + if not enable_scene_reconstruction: + raise PipelineError("built-in backend cannot run object-only mode; use an existing external scene package or backend=external") + manifest = run_builtin_real2sim( + capture, + context, + prompt, + self.config, + include_objects=enable_object_reconstruction, + ) + manifest = self._prune_assets(manifest, work, prune_options or self._prune_options()) + package = PackageWriter( + robot=self.robot, + dynamic_objects=self.dynamic_objects, + include_scene_mesh=manifest.scene.mesh_obj is not None, + ).write(manifest, output) + return PipelineRunResult(work_dir=work, package=package, manifest=manifest) + + detections_json = self._run_detection(context, resume=resume) + context["detections_json"] = detections_json + instances_json = self._run_segmentation(context, resume=resume) + context["instances_json"] = instances_json + instances_json = self._run_instance_refinement(context, resume=resume) + context["instances_json"] = instances_json + background_image = self._run_inpainting(context, resume=resume) + context["background_image"] = background_image + if enable_scene_reconstruction: + scene_ply, scene_mesh, camera_artifacts = self._run_scene_reconstruction(context, resume=resume) + context["scene_ply"] = scene_ply + if scene_mesh: + context["scene_mesh"] = scene_mesh + else: + scene_ply = Path(context["scene_ply"]) + scene_mesh = Path(context["scene_mesh"]) + scene_ply = require_file(scene_ply, "precomputed scene 3DGS PLY") + if scene_mesh.exists(): + scene_mesh = require_file(scene_mesh, "precomputed scene mesh OBJ") + else: + scene_mesh = None + camera_artifacts = { + "source_image": Path(context["image"]), + } + if scene_mesh is not None: + camera_artifacts["scene_mesh"] = scene_mesh + if Path(context["cameras_json"]).exists(): + camera_artifacts["cameras_json"] = Path(context["cameras_json"]) + if Path(context["depth_map"]).exists(): + camera_artifacts["depth_map"] = Path(context["depth_map"]) + + instances = load_instances(instances_json) if enable_object_reconstruction else [] + object_assets = self._run_object_reconstruction(context, instances, resume=resume) if enable_object_reconstruction else [] + scene_asset = Real2SimAsset(name="scene", label="background_scene", gaussian_ply=scene_ply, mesh_obj=scene_mesh) + manifest = Real2SimManifest( + source=capture.source, + prompt=prompt, + scene=scene_asset, + objects=tuple(object_assets), + camera_artifacts=camera_artifacts, + metadata={ + "pipeline": "GS-Playground Real2Sim MVP", + "capture_kind": capture.kind, + "stage_order": list(STAGES.keys()), + }, + ) + + if enable_object_reconstruction: + manifest = self._run_pose_alignment(context, manifest, resume=resume) + manifest = self._prune_assets(manifest, work, prune_options or self._prune_options()) + package = PackageWriter( + robot=self.robot, + dynamic_objects=self.dynamic_objects, + include_scene_mesh=manifest.scene.mesh_obj is not None, + ).write(manifest, output) + return PipelineRunResult(work_dir=work, package=package, manifest=manifest) + + def package_existing( + self, + *, + source: Path, + scene_ply: Path, + output_dir: Path, + prompt: str, + scene_mesh: Path | None = None, + objects_json: Path | None = None, + cameras_json: Path | None = None, + depth_map: Path | None = None, + background_image: Path | None = None, + preview_image: Path | None = None, + mesh_cleanup_report: Path | None = None, + texture_bake_report: Path | None = None, + scene_structure_report: Path | None = None, + ) -> PackageResult: + scene = Real2SimAsset( + name="scene", + label="background_scene", + gaussian_ply=require_file(scene_ply.resolve(), "scene 3DGS PLY"), + mesh_obj=require_file(scene_mesh.resolve(), "scene mesh OBJ") if scene_mesh else None, + ) + objects: list[Real2SimAsset] = [] + if objects_json is not None: + raw = read_json(objects_json) + entries = raw.get("objects", raw) if isinstance(raw, Mapping) else raw + if not isinstance(entries, list): + raise StageArtifactError("objects JSON must be a list or object with objects list") + for entry in entries: + objects.append(self._asset_from_json_entry(entry, objects_json.parent)) + + camera_artifacts = {} + if cameras_json is not None: + camera_artifacts["cameras_json"] = require_file(cameras_json.resolve(), "camera JSON") + if depth_map is not None: + camera_artifacts["depth_map"] = require_file(depth_map.resolve(), "depth map") + if background_image is not None: + camera_artifacts["background_image"] = require_file(background_image.resolve(), "background image") + if preview_image is not None: + camera_artifacts["source_gs_preview"] = require_file(preview_image.resolve(), "source-view 3DGS preview image") + + manifest = Real2SimManifest( + source=source.resolve(), + prompt=prompt, + scene=scene, + objects=tuple(objects), + camera_artifacts=camera_artifacts, + metadata={"pipeline": "manual-existing-assets"}, + ) + result = PackageWriter( + robot=self.robot, + dynamic_objects=self.dynamic_objects, + include_scene_mesh=manifest.scene.mesh_obj is not None, + ).write(manifest, output_dir) + copy_optional_package_report( + mesh_cleanup_report, + result.root / "artifacts" / "mesh_cleanup" / "cleanup_report.json", + "mesh cleanup report", + ) + copy_optional_package_report( + texture_bake_report, + result.root / "artifacts" / "colmap_texture_bake" / "texture_bake_report.json", + "texture bake report", + ) + copy_optional_package_report( + scene_structure_report, + result.root / "artifacts" / "scene_structure" / "scene_structure_report.json", + "scene structure report", + ) + self._attach_scene_structure_report_to_import(result.root) + return result + + def _attach_scene_structure_report_to_import(self, root: Path) -> None: + report = root / "artifacts" / "scene_structure" / "scene_structure_report.json" + import_path = root / "motrix_import.json" + if not report.is_file() or not import_path.is_file(): + return + scene_xml = root / "mjcf" / "scene.xml" + report_data = read_json(report) + structure_layers_path = root / "artifacts" / "scene_structure" / "static_structure.json" + structure_layers = scene_structure_layers(report_data, proxy_count=0) + source_proxy_count = self._attach_scene_structure_collision_proxies(scene_xml, report_data) + completion_proxy_count = self._attach_scene_completion_collision_proxies(scene_xml, structure_layers) + proxy_count = source_proxy_count + completion_proxy_count + structure_layers.setdefault("quality", {})["source_collision_proxy_count"] = source_proxy_count + structure_layers["quality"]["completion_collision_proxy_count"] = completion_proxy_count + structure_layers["quality"]["total_collision_proxy_count"] = proxy_count + write_json(structure_layers_path, structure_layers) + commercial_readiness_path = root / "artifacts" / "quality" / "commercial_readiness.json" + room_completeness_path = root / "artifacts" / "quality" / "room_completeness.json" + metric_scale_path = root / "artifacts" / "quality" / "metric_scale.json" + navigation_acceptance_path = root / "artifacts" / "quality" / "navigation_acceptance.json" + material_readiness_path = root / "artifacts" / "quality" / "material_readiness.json" + task_affordance_path = root / "artifacts" / "quality" / "task_affordance.json" + commercial_acceptance_path = root / "artifacts" / "quality" / "commercial_acceptance.json" + movable_assets_path = root / "artifacts" / "assets" / "movable_assets.json" + movable_assets = movable_asset_layers(root, structure_layers=structure_layers) + movable_assets_path.parent.mkdir(parents=True, exist_ok=True) + write_json(movable_assets_path, movable_assets) + data = read_json(import_path) + movable_tasks_path = root / "artifacts" / "assets" / "movable_extraction_tasks.json" + movable_tasks = movable_extraction_tasks(root, data, movable_assets) + write_json(movable_tasks_path, movable_tasks) + metric_scale = metric_scale_report(root, structure_layers, report_data) + navigation_acceptance = navigation_acceptance_report(root, structure_layers) + material_readiness = material_readiness_report(root) + task_affordance = task_affordance_report(root, movable_assets) + commercial_readiness = commercial_readiness_report( + root, + structure_layers, + report_data, + proxy_count=proxy_count, + metric_scale=metric_scale, + navigation_acceptance=navigation_acceptance, + material_readiness=material_readiness, + task_affordance=task_affordance, + ) + room_completeness = room_completeness_report(root, structure_layers, report_data) + commercial_readiness_path.parent.mkdir(parents=True, exist_ok=True) + write_json(commercial_readiness_path, commercial_readiness) + write_json(room_completeness_path, room_completeness) + write_json(metric_scale_path, metric_scale) + write_json(navigation_acceptance_path, navigation_acceptance) + write_json(material_readiness_path, material_readiness) + write_json(task_affordance_path, task_affordance) + data["scene_structure_report"] = report.relative_to(root).as_posix() + data["scene_structure_layers"] = structure_layers_path.relative_to(root).as_posix() + data["movable_asset_layers"] = movable_assets_path.relative_to(root).as_posix() + data["movable_extraction_tasks"] = movable_tasks_path.relative_to(root).as_posix() + data["commercial_readiness_report"] = commercial_readiness_path.relative_to(root).as_posix() + data["room_completeness_report"] = room_completeness_path.relative_to(root).as_posix() + data["metric_scale_report"] = metric_scale_path.relative_to(root).as_posix() + data["navigation_acceptance_report"] = navigation_acceptance_path.relative_to(root).as_posix() + data["material_readiness_report"] = material_readiness_path.relative_to(root).as_posix() + data["task_affordance_report"] = task_affordance_path.relative_to(root).as_posix() + data["commercial_acceptance_report"] = commercial_acceptance_path.relative_to(root).as_posix() + data.setdefault("verification", {})["scene_structure"] = report_data + data["verification"]["scene_structure_collision_proxies"] = proxy_count + data["verification"]["scene_structure_layers"] = { + "static_collision_proxy_count": len(structure_layers["static_collision_proxies"]), + "completion_proxy_count": len(structure_layers.get("completion_proxies", [])), + "navigation_surface_count": len(structure_layers["navigation_surfaces"]), + "support_surface_count": len(structure_layers["support_surfaces"]), + "wall_count": len(structure_layers["walls"]), + } + data["verification"]["movable_asset_layers"] = { + "movable_asset_count": len(movable_assets["movable_assets"]), + "dynamic_asset_count": sum(1 for asset in movable_assets["movable_assets"] if asset.get("dynamic")), + "candidate_slot_count": len(movable_assets["candidate_slots"]), + "extraction_task_count": len(movable_tasks["tasks"]), + "production_asset_count": movable_assets["quality"].get("production_asset_count"), + "provisional_asset_count": movable_assets["quality"].get("provisional_asset_count"), + "incomplete_asset_count": movable_assets["quality"].get("incomplete_asset_count"), + "commercial_asset_grade": movable_assets["quality"].get("commercial_asset_grade"), + } + data["verification"]["commercial_readiness"] = commercial_readiness["summary"] + data["verification"]["room_completeness"] = room_completeness["summary"] + data["verification"]["metric_scale"] = metric_scale["summary"] + data["verification"]["navigation_acceptance"] = navigation_acceptance["summary"] + data["verification"]["material_readiness"] = material_readiness["summary"] + data["verification"]["task_affordance"] = task_affordance["summary"] + commercial_acceptance = commercial_acceptance_manifest( + root, + commercial_readiness=commercial_readiness, + room_completeness=room_completeness, + metric_scale=metric_scale, + navigation_acceptance=navigation_acceptance, + material_readiness=material_readiness, + task_affordance=task_affordance, + movable_assets=movable_assets, + structure_layers=structure_layers, + ) + data["verification"]["commercial_acceptance"] = commercial_acceptance["summary"] + write_json(import_path, data) + commercial_acceptance = commercial_acceptance_manifest( + root, + commercial_readiness=commercial_readiness, + room_completeness=room_completeness, + metric_scale=metric_scale, + navigation_acceptance=navigation_acceptance, + material_readiness=material_readiness, + task_affordance=task_affordance, + movable_assets=movable_assets, + structure_layers=structure_layers, + ) + write_json(commercial_acceptance_path, commercial_acceptance) + + def _attach_scene_structure_collision_proxies(self, scene_xml: Path, report: Mapping[str, Any]) -> int: + if not scene_xml.is_file(): + return 0 + semantic_planes = report.get("semantic_planes") + if not isinstance(semantic_planes, list): + return 0 + tree = ET.parse(scene_xml) + world = tree.getroot().find("worldbody") + if world is None: + return 0 + scene_body = world.find("./body[@name='scene']") + body = scene_body if scene_body is not None else world + added = 0 + for index, plane in enumerate(semantic_planes[:12]): + if not isinstance(plane, Mapping): + continue + proxy = plane.get("collision_proxy_hint") + if not isinstance(proxy, Mapping) or proxy.get("type") != "box": + continue + center = numeric_triplet(proxy.get("center")) + size = numeric_triplet(proxy.get("size")) + if center is None or size is None: + continue + label = str(plane.get("semantic_label") or proxy.get("label") or "structure") + if label not in {"floor", "ceiling", "support_surface", "wall_visible"} and not label.startswith("wall_"): + continue + ET.SubElement( + body, + "geom", + { + "name": f"scene_structure_collision_{slug_token(label)}_{index:03d}", + "type": "box", + "pos": vec(center), + "size": vec(size), + "rgba": "0.1 0.7 1 0.08", + "group": "3", + "contype": "1", + "conaffinity": "1", + }, + ) + added += 1 + if added: + indent_xml(tree.getroot()) + tree.write(scene_xml, encoding="utf-8", xml_declaration=True) + return added + + def _attach_scene_completion_collision_proxies(self, scene_xml: Path, structure_layers: Mapping[str, Any]) -> int: + if not scene_xml.is_file(): + return 0 + proxies = structure_layers.get("completion_proxies") + if not isinstance(proxies, list) or not proxies: + return 0 + tree = ET.parse(scene_xml) + world = tree.getroot().find("worldbody") + if world is None: + return 0 + scene_body = world.find("./body[@name='scene']") + body = scene_body if scene_body is not None else world + added = 0 + for proxy in proxies[:8]: + if not isinstance(proxy, Mapping) or proxy.get("type") != "box": + continue + center = numeric_triplet(proxy.get("center")) + size = numeric_triplet(proxy.get("size")) + if center is None or size is None: + continue + name = str(proxy.get("mjcf_geom_name") or f"scene_completion_collision_{added:03d}") + ET.SubElement( + body, + "geom", + { + "name": name, + "type": "box", + "pos": vec(center), + "size": vec(size), + "rgba": "0.2 0.9 0.35 0.10", + "group": "3", + "contype": "1", + "conaffinity": "1", + }, + ) + added += 1 + if added: + indent_xml(tree.getroot()) + tree.write(scene_xml, encoding="utf-8", xml_declaration=True) + return added + + def check(self) -> dict[str, Any]: + status: dict[str, Any] = {"backend": self.backend, "stages": {}, "python_packages": {}} + for name, stage in STAGES.items(): + try: + command = resolve_command(stage, self.config) + status["stages"][name] = {"configured": command is not None, "env": stage.env_var} + except MissingStageCommand as exc: + status["stages"][name] = {"configured": False, "env": stage.env_var, "error": str(exc)} + + for module in ("numpy", "torch", "gaussian_renderer", "gsplat", "plyfile"): + try: + __import__(module) + status["python_packages"][module] = True + except Exception: + status["python_packages"][module] = False + status["builtin_ready"] = status["python_packages"]["numpy"] + return status + + def prepare_capture(self, input_path: Path, work: Path) -> CaptureInfo: + if not input_path.exists(): + raise FileNotFoundError(input_path) + input_dir = work / "00_input" + input_dir.mkdir(parents=True, exist_ok=True) + + if input_path.is_dir(): + source_images = sorted(path for path in input_path.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES) + if not source_images: + raise StageArtifactError(f"image directory contains no supported images: {input_path}") + frames_dir = input_dir / "frames" + frames_dir.mkdir(parents=True, exist_ok=True) + frames: list[Path] = [] + for index, source_image in enumerate(source_images, start=1): + frame = frames_dir / f"frame_{index:06d}{source_image.suffix.lower()}" + shutil.copy2(source_image, frame) + frames.append(frame) + return CaptureInfo(source=input_path, image=frames[0], frames_dir=frames_dir, kind="multi_image") + + suffix = input_path.suffix.lower() + if suffix in IMAGE_SUFFIXES: + image = input_dir / f"input{suffix}" + shutil.copy2(input_path, image) + return CaptureInfo(source=input_path, image=image, frames_dir=None, kind="single_image") + + if suffix in VIDEO_SUFFIXES: + frames_dir = input_dir / "frames" + frames_dir.mkdir(parents=True, exist_ok=True) + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + raise PipelineError("video input requires ffmpeg to extract frames") + cmd = [ffmpeg, "-y", "-i", input_path.as_posix(), "-vf", "fps=2", (frames_dir / "frame_%06d.png").as_posix()] + completed = subprocess.run(cmd, check=False) + if completed.returncode != 0: + raise PipelineError(f"ffmpeg frame extraction failed with exit code {completed.returncode}") + frames = sorted(frames_dir.glob("frame_*.png")) + if not frames: + raise StageArtifactError(f"ffmpeg produced no frames in {frames_dir}") + return CaptureInfo(source=input_path, image=frames[0], frames_dir=frames_dir, kind="video") + + raise PipelineError(f"unsupported input type: {input_path}") + + def _base_context( + self, + capture: CaptureInfo, + work: Path, + output: Path, + prompt: str, + *, + synthetic_multiview_mode: str = "object", + ) -> dict[str, Any]: + mode = self._synthetic_multiview_mode(synthetic_multiview_mode) + return { + "source": capture.source, + "input": capture.source, + "python": sys.executable, + "image": capture.image, + "frames_dir": capture.frames_dir or "", + "synthetic_multiview_dir": work / "00_input" / "synthetic_multiview", + "synthetic_frames_dir": work / "00_input" / "synthetic_multiview" / "frames", + "synthetic_contact_sheet": work / "00_input" / "synthetic_multiview" / "contact_sheet.png", + "synthetic_multiview_mode": mode, + "synthetic_multiview_prompt": self._synthetic_multiview_prompt(prompt, mode=mode), + "work_dir": work, + "output_dir": output, + "prompt": prompt, + "detections_json": work / "01_detection" / "detections.json", + "instances_json": work / "02_segmentation" / "instances.json", + "masks_dir": work / "02_segmentation" / "masks", + "background_image": work / "03_inpainting" / "background.png", + "scene_reconstruction_dir": work / "04_reconstruction" / "scene", + "scene_ply": work / "04_reconstruction" / "scene" / "point_cloud.ply", + "scene_mesh": work / "04_reconstruction" / "scene" / "mesh.obj", + "depth_map": work / "04_reconstruction" / "scene" / "depth.pgm", + "cameras_json": work / "04_reconstruction" / "scene" / "cameras.json", + "objects_dir": work / "04_reconstruction" / "objects", + "aligned_assets_json": work / "05_alignment" / "assets.json", + } + + def _should_use_builtin_backend(self) -> bool: + if self.backend == "builtin": + return True + if self.backend == "hybrid": + return False + if self.backend == "external": + return False + if self.backend != "auto": + raise PipelineError(f"unknown Real2Sim backend: {self.backend}") + return not self._external_stages_configured() + + def _external_stages_configured(self) -> bool: + required = ("grounding_dino", "sam2", "instance_refinement", "lama", "anysplat", "sam3d") + for name in required: + try: + resolve_command(STAGES[name], self.config) + except MissingStageCommand: + return False + return True + + def _run_hybrid_backend( + self, + *, + capture: CaptureInfo, + context: dict[str, Any], + prompt: str, + work: Path, + output: Path, + resume: bool, + prune_options: PruneOptions | None, + enable_object_reconstruction: bool = True, + ) -> PipelineRunResult: + try: + self._run_detection(context, resume=resume) + self._run_segmentation(context, resume=resume) + self._run_instance_refinement(context, resume=resume) + self._run_inpainting(context, resume=resume) + self._run_depth_estimation(context, resume=resume) + except MissingStageCommand: + pass + + hybrid_config = dict(self.config) + builtin_config = dict(hybrid_config.get("builtin", {})) if isinstance(hybrid_config.get("builtin", {}), Mapping) else {} + builtin_config["prefer_stage_artifacts"] = True + hybrid_config["builtin"] = builtin_config + manifest = run_builtin_real2sim( + capture, + context, + prompt, + hybrid_config, + include_objects=enable_object_reconstruction, + ) + manifest = self._prune_assets(manifest, work, prune_options or self._prune_options()) + package = PackageWriter( + robot=self.robot, + dynamic_objects=self.dynamic_objects, + include_scene_mesh=manifest.scene.mesh_obj is not None, + ).write(manifest, output) + return PipelineRunResult(work_dir=work, package=package, manifest=manifest) + + def _run_detection(self, context: dict[str, Any], *, resume: bool) -> Path: + output = Path(context["detections_json"]) + if resume and output.exists(): + validate_detections(output) + return output + output.parent.mkdir(parents=True, exist_ok=True) + command = resolve_command(STAGES["grounding_dino"], self.config) + run_command(STAGES["grounding_dino"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + validate_detections(output) + return output + + def _run_segmentation(self, context: dict[str, Any], *, resume: bool) -> Path: + output = Path(context["instances_json"]) + if resume and output.exists(): + load_instances(output) + return output + output.parent.mkdir(parents=True, exist_ok=True) + Path(context["masks_dir"]).mkdir(parents=True, exist_ok=True) + command = resolve_command(STAGES["sam2"], self.config) + run_command(STAGES["sam2"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + load_instances(output) + return output + + def _run_instance_refinement(self, context: dict[str, Any], *, resume: bool) -> Path: + output = Path(context["instances_json"]) + command = resolve_command(STAGES["instance_refinement"], self.config) + if resume and output.exists() and instances_are_refined(output): + load_instances(output) + return output + run_command(STAGES["instance_refinement"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + load_instances(output) + return output + + def _run_inpainting(self, context: dict[str, Any], *, resume: bool) -> Path: + output = Path(context["background_image"]) + if resume and output.exists(): + return require_file(output, "inpainted background image") + output.parent.mkdir(parents=True, exist_ok=True) + command = resolve_command(STAGES["lama"], self.config) + run_command(STAGES["lama"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + require_file(output, "inpainted background image") + return output + + def _run_depth_estimation(self, context: dict[str, Any], *, resume: bool) -> Path | None: + command = resolve_command(STAGES["depth_estimation"], self.config) + if command is None: + return None + output = Path(context["depth_map"]) + if resume and output.exists(): + return require_file(output, "monocular depth map") + output.parent.mkdir(parents=True, exist_ok=True) + run_command(STAGES["depth_estimation"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + require_file(output, "monocular depth map") + return output + + def _run_scene_reconstruction(self, context: dict[str, Any], *, resume: bool) -> tuple[Path, Path | None, dict[str, Path]]: + scene_dir = Path(context["scene_reconstruction_dir"]) + scene_dir.mkdir(parents=True, exist_ok=True) + ply = Path(context["scene_ply"]) + if not (resume and ply.exists()): + command = resolve_command(STAGES["anysplat"], self.config) + run_command(STAGES["anysplat"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + require_file(ply, "scene 3DGS PLY") + + mesh = Path(context["scene_mesh"]) + camera_artifacts = {} + source_image = Path(context["image"]) + if source_image.exists(): + camera_artifacts["source_image"] = source_image + cameras_json = Path(context["cameras_json"]) + depth_map = Path(context["depth_map"]) + if mesh.exists(): + camera_artifacts["scene_mesh"] = mesh + if cameras_json.exists(): + camera_artifacts["cameras_json"] = cameras_json + if depth_map.exists(): + camera_artifacts["depth_map"] = depth_map + return ply, mesh if mesh.exists() else None, camera_artifacts + + def _run_object_reconstruction( + self, + context: dict[str, Any], + instances: list[Any], + *, + resume: bool, + ) -> list[Real2SimAsset]: + if not instances: + return [] + + command = resolve_command(STAGES["sam3d"], self.config) + objects: list[Real2SimAsset] = [] + objects_dir = Path(context["objects_dir"]) + objects_dir.mkdir(parents=True, exist_ok=True) + for instance in instances: + object_dir = objects_dir / _safe_name(instance.instance_id) + object_dir.mkdir(parents=True, exist_ok=True) + instance_json = object_dir / "instance.json" + write_json( + instance_json, + { + "id": instance.instance_id, + "label": instance.label, + "score": instance.score, + "bbox_xyxy": list(instance.bbox_xyxy) if instance.bbox_xyxy else None, + "mask_path": instance.mask_path.as_posix() if instance.mask_path else None, + }, + ) + object_context = { + **context, + "object_dir": object_dir, + "instance_json": instance_json, + "instance_id": instance.instance_id, + "instance_label": instance.label, + "mask": instance.mask_path or "", + "object_ply": object_dir / "gaussian.ply", + "object_mesh": object_dir / "mesh.obj", + "object_pose": object_dir / "pose.json", + } + if not (resume and Path(object_context["object_ply"]).exists()): + run_command(STAGES["sam3d"], command, object_context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + ply = require_file(Path(object_context["object_ply"]), f"object 3DGS PLY for {instance.instance_id}") + mesh = require_file(Path(object_context["object_mesh"]), f"object mesh OBJ for {instance.instance_id}") + pose = Pose.from_json(Path(object_context["object_pose"])) + objects.append( + Real2SimAsset( + name=_safe_name(instance.instance_id), + label=instance.label, + gaussian_ply=ply, + mesh_obj=mesh, + pose=pose, + dynamic=True, + source_instance_id=instance.instance_id, + ) + ) + return objects + + def _run_pose_alignment(self, context: dict[str, Any], manifest: Real2SimManifest, *, resume: bool) -> Real2SimManifest: + try: + command = resolve_command(STAGES["pose_alignment"], self.config) + except MissingStageCommand: + command = None + if command is None: + return manifest + aligned_json = Path(context["aligned_assets_json"]) + if not (resume and aligned_json.exists()): + aligned_json.parent.mkdir(parents=True, exist_ok=True) + write_json(aligned_json, manifest.to_json(context["work_dir"])) + run_command(STAGES["pose_alignment"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if self.dry_run or not aligned_json.exists(): + return manifest + return _manifest_from_alignment_json(aligned_json, manifest) + + def _run_synthetic_multiview_expansion( + self, + capture: CaptureInfo, + context: dict[str, Any], + *, + resume: bool, + enabled: bool, + ) -> CaptureInfo: + if not enabled: + return capture + if capture.kind != "single_image": + return capture + command = resolve_command(STAGES["synthetic_multiview"], self.config) + if command is None: + raise MissingStageCommand( + "synthetic_multiview was requested but no command was configured. " + "Set stages.synthetic_multiview.command or export REAL2SIM_SYNTHETIC_MULTIVIEW_COMMAND." + ) + + frames_dir = Path(context["synthetic_frames_dir"]) + if not _supported_images(frames_dir): + frames_dir.mkdir(parents=True, exist_ok=True) + run_command(STAGES["synthetic_multiview"], command, context, cwd=self.repo_root, dry_run=self.dry_run) + if not self.dry_run: + frames = _supported_images(frames_dir) + if not frames: + raise StageArtifactError(f"synthetic multiview produced no supported images in {frames_dir}") + context["frames_dir"] = frames_dir + return CaptureInfo(source=capture.source, image=capture.image, frames_dir=frames_dir, kind="synthetic_multiview") + + def _synthetic_multiview_mode(self, requested: str) -> str: + cfg = self.config.get("synthetic_multiview", {}) + if isinstance(cfg, Mapping) and isinstance(cfg.get("mode"), str) and cfg["mode"].strip(): + requested = str(cfg["mode"]) + mode = requested.strip().lower() + if mode not in {"object", "scene"}: + raise PipelineError(f"synthetic_multiview mode must be 'object' or 'scene', got: {requested}") + return mode + + def _synthetic_multiview_prompt(self, prompt: str, *, mode: str = "object") -> str: + cfg = self.config.get("synthetic_multiview", {}) + if isinstance(cfg, Mapping) and isinstance(cfg.get("prompt"), str) and cfg["prompt"].strip(): + return str(cfg["prompt"]) + if mode == "scene": + return ( + "Create a photorealistic 4 columns x 2 rows multiview capture contact sheet for scene-level real2sim reconstruction. " + f"Scene context: {prompt}. Treat the input as the canonical source view and preserve the exact room or outdoor layout, " + "floor plane, walls, doors, windows, large furniture, static fixtures, lighting direction, material colors, and visible object identities. " + "Infer only physically plausible nearby camera views from the same static environment; do not invent a different room, replace furniture, " + "add people, add text, add UI, or move structural elements. Produce exactly 8 equal portrait panels with thin white gutters and no labels, " + "numbers, captions, icons, or watermarks. Views in reading order: source/front view, front-left 45 degrees, left 90 degrees, rear-left " + "135 degrees, rear/back, rear-right 225 degrees, right 270 degrees, and top-oblique 30 to 45 degrees. Keep camera height, focal length, " + "scale, horizon, navigable ground continuity, occlusion relationships, and shadows consistent across all panels. When unseen regions are " + "not directly observable, complete them conservatively as simple continuations of the visible surfaces so AnySplat receives coherent " + "scene geometry evidence rather than object-only turntable views." + ) + return ( + "Create a photorealistic 4 columns x 2 rows multiview capture contact sheet for real2sim reconstruction. " + f"Subject: {prompt}. Preserve the exact object identity, geometry, material, color, labels, markings, and size relationships. " + "Produce exactly 8 equal portrait panels with thin white gutters and no text, titles, numbers, captions, UI icons, or watermarks. " + "Views in reading order: front main, front-left 45 degrees, left 90 degrees, rear-left 135 degrees, rear/back, rear-right 225 degrees, " + "right 270 degrees, and top-oblique 30 to 45 degrees. Keep the same stationary object or objects on the same tabletop, consistent focal length, " + "stable soft lighting, sharp focus, full silhouettes with 10 to 15 percent margin, no hands, no people blocking the object, no play overlays, " + "and avoid blown-out reflections." + ) + + def _prune_assets(self, manifest: Real2SimManifest, work: Path, options: PruneOptions) -> Real2SimManifest: + pruned_dir = work / "06_pruned" + prune_context = self._prune_command_context(manifest) + scene_ply = prune_ply( + manifest.scene.gaussian_ply, + pruned_dir / "scene.ply", + options, + self.repo_root, + extra_context=prune_context, + ) + scene = Real2SimAsset( + name=manifest.scene.name, + label=manifest.scene.label, + gaussian_ply=scene_ply, + mesh_obj=manifest.scene.mesh_obj, + pose=manifest.scene.pose, + dynamic=manifest.scene.dynamic, + source_instance_id=manifest.scene.source_instance_id, + ) + objects = [] + for asset in manifest.objects: + obj_ply = prune_ply( + asset.gaussian_ply, + pruned_dir / "objects" / f"{asset.name}.ply", + options, + self.repo_root, + extra_context={**prune_context, "asset_name": asset.name, "asset_label": asset.label}, + ) + objects.append( + Real2SimAsset( + name=asset.name, + label=asset.label, + gaussian_ply=obj_ply, + mesh_obj=asset.mesh_obj, + pose=asset.pose, + dynamic=asset.dynamic, + source_instance_id=asset.source_instance_id, + ) + ) + return Real2SimManifest( + source=manifest.source, + prompt=manifest.prompt, + scene=scene, + objects=tuple(objects), + camera_artifacts=manifest.camera_artifacts, + metadata={**manifest.metadata, "pruning": options.__dict__}, + ) + + def _prune_command_context(self, manifest: Real2SimManifest) -> dict[str, Path | str]: + cameras = manifest.camera_artifacts.get("cameras_json") + source = manifest.source + source_image = manifest.camera_artifacts.get("source_image", source) + return { + "source": source, + "source_image": source_image, + "image": source_image, + "cameras_json": cameras or "", + "cameras": cameras or "", + } + + def _prune_options(self) -> PruneOptions: + cfg = self.config.get("pruning", {}) + if not isinstance(cfg, Mapping): + cfg = {} + return PruneOptions( + enabled=bool(cfg.get("enabled", True)), + keep_ratio=float(cfg.get("keep_ratio", 0.3)), + keep_count=int(cfg["keep_count"]) if cfg.get("keep_count") is not None else None, + mode=str(cfg.get("mode", "opacity_area")), + external_command=command_preferring_env(cfg.get("external_command"), os.environ.get("REAL2SIM_SPEEDY_SPLAT_COMMAND")), + ) + + def _asset_from_json_entry(self, entry: Mapping[str, Any], base_dir: Path) -> Real2SimAsset: + if not isinstance(entry, Mapping): + raise StageArtifactError(f"object asset entry must be a JSON object: {entry}") + name = str(entry.get("name", entry.get("id", entry.get("label", "object")))) + label = str(entry.get("label", name)) + ply = _resolve_path(base_dir, entry.get("gaussian_ply", entry.get("ply"))) + mesh_raw = entry.get("mesh_obj", entry.get("mesh")) + pose_raw = entry.get("pose") + if isinstance(pose_raw, Mapping): + pose_file = base_dir / f".{_safe_name(name)}_pose.json" + write_json(pose_file, pose_raw) + pose = Pose.from_json(pose_file) + pose_file.unlink(missing_ok=True) + elif pose_raw: + pose = Pose.from_json(_resolve_path(base_dir, pose_raw)) + else: + pose = Pose() + return Real2SimAsset( + name=_safe_name(name), + label=label, + gaussian_ply=require_file(ply, f"object gaussian PLY for {name}"), + mesh_obj=require_file(_resolve_path(base_dir, mesh_raw), f"object mesh OBJ for {name}") if mesh_raw else None, + pose=pose, + dynamic=bool(entry.get("dynamic", True)), + source_instance_id=str(entry.get("source_instance_id", name)), + ) + + +def load_config(path: Path | None) -> dict[str, Any]: + if path is None: + return {} + data = read_json(path) + if not isinstance(data, dict): + raise PipelineError(f"config must be a JSON object: {path}") + return data + + +def copy_optional_package_report(source: Path | None, destination: Path, label: str) -> None: + if source is None: + return + src = require_file(source.resolve(), label) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, destination) + + +def scene_structure_layers(report: Mapping[str, Any], *, proxy_count: int = 0) -> dict[str, Any]: + semantic_planes = report.get("semantic_planes") + planes = semantic_planes if isinstance(semantic_planes, list) else [] + static_proxies: list[dict[str, Any]] = [] + walls: list[dict[str, Any]] = [] + support_surfaces: list[dict[str, Any]] = [] + navigation_surfaces: list[dict[str, Any]] = [] + has_floor = False + for index, plane in enumerate(planes[:32]): + if not isinstance(plane, Mapping): + continue + label = str(plane.get("semantic_label") or "unknown") + has_floor = has_floor or label == "floor" + proxy = plane.get("collision_proxy_hint") + normalized_proxy = normalized_collision_proxy(proxy, label=label, index=index) + if normalized_proxy is not None and is_static_structure_label(label): + static_proxies.append(normalized_proxy) + surface = { + "id": f"{slug_token(label)}_{index:03d}", + "semantic_label": label, + "axis": plane.get("axis"), + "coordinate": plane.get("coordinate"), + "area_ratio": plane.get("area_ratio"), + "extent": plane.get("extent"), + "collision_proxy_id": normalized_proxy["id"] if normalized_proxy else None, + } + if label.startswith("wall_") or label == "wall_visible": + walls.append(surface) + if label == "support_surface": + support_surfaces.append(surface) + navigation_surfaces.append({**surface, "navigation_role": "placeable_support"}) + elif label == "floor": + navigation_surfaces.append({**surface, "navigation_role": "walkable_floor"}) + completion_proxies = room_completion_proxies(report, has_floor=has_floor, walls=walls, support_surfaces=support_surfaces) + for completion_proxy in completion_proxies: + static_proxies.append(completion_proxy) + if completion_proxy.get("semantic_label") == "floor_completion": + navigation_surfaces.append( + { + "id": completion_proxy["id"], + "semantic_label": "floor_completion", + "axis": completion_proxy.get("axis"), + "coordinate": completion_proxy.get("coordinate"), + "area_ratio": None, + "extent": completion_proxy.get("extent"), + "collision_proxy_id": completion_proxy["id"], + "navigation_role": "walkable_floor_inferred", + "inferred_completion": True, + } + ) + readiness = report.get("commercial_readiness") if isinstance(report.get("commercial_readiness"), Mapping) else {} + return { + "format": "gs-playground-real2sim-static-structure-v1", + "source_report_format": report.get("format"), + "static_collision_proxies": static_proxies, + "completion_proxies": completion_proxies, + "walls": walls, + "support_surfaces": support_surfaces, + "navigation_surfaces": navigation_surfaces, + "quality": { + "structure_score": report.get("structure_score"), + "semantic_structure_score": readiness.get("semantic_structure_score"), + "source_collision_proxy_count": proxy_count, + "completion_proxy_count": len(completion_proxies), + "has_floor": readiness.get("has_floor"), + "has_floor_completion": any(proxy.get("semantic_label") == "floor_completion" for proxy in completion_proxies), + "has_wall": readiness.get("has_wall"), + "has_support_surface": readiness.get("has_support_surface"), + }, + "commercial_gap_notes": [ + "Static structure layers are now consumable by simulation code; inferred completion proxies are marked and should not be confused with directly observed mesh.", + "Navigation surfaces are proxy hints; production navmesh generation still needs topology cleanup and traversability validation.", + ], + } + + +def room_completion_proxies( + report: Mapping[str, Any], + *, + has_floor: bool, + walls: list[dict[str, Any]], + support_surfaces: list[dict[str, Any]], +) -> list[dict[str, Any]]: + if has_floor: + return [] + bounds = report.get("bounds") + if not isinstance(bounds, Mapping): + return [] + low = numeric_triplet(bounds.get("p05") or bounds.get("min")) + high = numeric_triplet(bounds.get("p95") or bounds.get("max")) + if low is None or high is None: + return [] + up_axis_name = str(report.get("up_axis") or "z") + up_axis = "xyz".find(up_axis_name) + if up_axis < 0: + up_axis = 2 + spans = [max(high[idx] - low[idx], 0.02) for idx in range(3)] + if not walls and not support_surfaces: + return [] + center = [(low[idx] + high[idx]) * 0.5 for idx in range(3)] + center[up_axis] = low[up_axis] + size = [max(spans[idx] * 0.5, 0.05) for idx in range(3)] + size[up_axis] = max(spans[up_axis] * 0.01, 0.02) + extent = {name: [round(low[idx], 6), round(high[idx], 6)] for idx, name in enumerate("xyz")} + return [ + { + "id": "floor_completion_000", + "type": "box", + "semantic_label": "floor_completion", + "center": [round(value, 6) for value in center], + "size": [round(value, 6) for value in size], + "axis": up_axis_name if up_axis_name in {"x", "y", "z"} else "z", + "coordinate": round(float(center[up_axis]), 6), + "extent": extent, + "mjcf_geom_name": "scene_completion_collision_floor_000", + "inferred_completion": True, + "inference_source": { + "reason": "observed walls/support surfaces but no explicit floor plane", + "bounds_source": "scene_structure_report.bounds.p05/p95", + }, + "physics_material": default_static_physics_material("floor"), + } + ] + + +def normalized_collision_proxy(proxy: Any, *, label: str, index: int) -> dict[str, Any] | None: + if not isinstance(proxy, Mapping) or proxy.get("type") != "box": + return None + center = numeric_triplet(proxy.get("center")) + size = numeric_triplet(proxy.get("size")) + if center is None or size is None: + return None + return { + "id": f"{slug_token(label)}_{index:03d}", + "type": "box", + "semantic_label": label, + "center": [round(value, 6) for value in center], + "size": [round(value, 6) for value in size], + "mjcf_geom_name": f"scene_structure_collision_{slug_token(label)}_{index:03d}", + "physics_material": default_static_physics_material(label), + } + + +def is_static_structure_label(label: str) -> bool: + return label in {"floor", "ceiling", "support_surface", "wall_visible"} or label.startswith("wall_") + + +def default_static_physics_material(label: str) -> dict[str, Any]: + if label == "support_surface": + return {"movable": False, "friction": [0.85, 0.02, 0.001], "restitution": 0.02, "contact_role": "support"} + if label == "floor": + return {"movable": False, "friction": [0.9, 0.02, 0.001], "restitution": 0.01, "contact_role": "walkable"} + if label == "ceiling": + return {"movable": False, "friction": [0.7, 0.02, 0.001], "restitution": 0.01, "contact_role": "overhead_boundary"} + return {"movable": False, "friction": [0.75, 0.02, 0.001], "restitution": 0.01, "contact_role": "static_boundary"} + + +def commercial_readiness_report( + root: Path, + structure_layers: Mapping[str, Any], + scene_structure_report: Mapping[str, Any], + *, + proxy_count: int, + metric_scale: Mapping[str, Any] | None = None, + navigation_acceptance: Mapping[str, Any] | None = None, + material_readiness: Mapping[str, Any] | None = None, + task_affordance: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + manifest = read_json(root / "package_manifest.json") if (root / "package_manifest.json").is_file() else {} + texture_report = read_optional_json(root / "artifacts" / "colmap_texture_bake" / "texture_bake_report.json") + mesh_cleanup = read_optional_json(root / "artifacts" / "mesh_cleanup" / "cleanup_report.json") + movable_report = read_optional_json(root / "artifacts" / "assets" / "movable_assets.json") + objects = manifest.get("objects") if isinstance(manifest, Mapping) else [] + object_count = len(objects) if isinstance(objects, list) else 0 + scene = manifest.get("scene") if isinstance(manifest, Mapping) else {} + scene_mesh_present = isinstance(scene, Mapping) and bool(scene.get("mesh_obj")) + static_proxies = structure_layers.get("static_collision_proxies") if isinstance(structure_layers, Mapping) else [] + navigation_surfaces = structure_layers.get("navigation_surfaces") if isinstance(structure_layers, Mapping) else [] + walls = structure_layers.get("walls") if isinstance(structure_layers, Mapping) else [] + support_surfaces = structure_layers.get("support_surfaces") if isinstance(structure_layers, Mapping) else [] + texture_metrics = texture_quality_metrics(texture_report) + candidate_slots = movable_report.get("candidate_slots") if isinstance(movable_report, Mapping) else [] + candidate_slot_count = len(candidate_slots) if isinstance(candidate_slots, list) else 0 + movable_quality = movable_report.get("quality") if isinstance(movable_report, Mapping) and isinstance(movable_report.get("quality"), Mapping) else {} + movable_assets = movable_report.get("movable_assets") if isinstance(movable_report, Mapping) and isinstance(movable_report.get("movable_assets"), list) else [] + movable_physics = movable_physics_material_quality(movable_assets) + commercial_movable_assets = bool(movable_quality.get("commercial_asset_grade")) + production_asset_count = int(movable_quality.get("production_asset_count") or 0) + provisional_asset_count = int(movable_quality.get("provisional_asset_count") or 0) + incomplete_asset_count = int(movable_quality.get("incomplete_asset_count") or 0) + static_material_hint_count = sum( + 1 for proxy in static_proxies if isinstance(proxy, Mapping) and valid_static_physics_material(proxy.get("physics_material")) + ) + static_materials_ready = bool(static_proxies) and static_material_hint_count == len(static_proxies) + room_report = room_completeness_report(root, structure_layers, scene_structure_report) + room_summary = room_report.get("summary") if isinstance(room_report.get("summary"), Mapping) else {} + metric_scale_summary = metric_scale.get("summary") if isinstance(metric_scale, Mapping) and isinstance(metric_scale.get("summary"), Mapping) else {} + navigation_summary = ( + navigation_acceptance.get("summary") + if isinstance(navigation_acceptance, Mapping) and isinstance(navigation_acceptance.get("summary"), Mapping) + else {} + ) + material_summary = ( + material_readiness.get("summary") + if isinstance(material_readiness, Mapping) and isinstance(material_readiness.get("summary"), Mapping) + else {} + ) + task_affordance_summary = ( + task_affordance.get("summary") + if isinstance(task_affordance, Mapping) and isinstance(task_affordance.get("summary"), Mapping) + else {} + ) + gates = [ + readiness_gate( + "dense_scene_geometry", + bool(scene_mesh_present and int(scene_structure_report.get("face_count") or 0) >= 100_000), + 0.15, + "scene mesh with at least 100k analyzed faces", + {"face_count": scene_structure_report.get("face_count"), "scene_mesh_present": scene_mesh_present}, + ), + readiness_gate( + "texture_bake", + bool(texture_metrics["passed"]), + 0.12, + "projected texture covers most reconstructed mesh faces, with atlas pixel coverage reported separately", + texture_metrics, + ), + readiness_gate( + "material_readiness", + bool(material_summary.get("passed")), + 0.05, + "package contains texture-backed MJCF/OBJ material assets with all referenced texture files present", + { + "score": material_summary.get("score"), + "texture_backed_material_count": material_summary.get("texture_backed_material_count"), + "missing_texture_file_count": material_summary.get("missing_texture_file_count"), + "scene_mesh_has_texture": material_summary.get("scene_mesh_has_texture"), + "blockers": material_summary.get("blockers"), + }, + ), + readiness_gate( + "static_structure_layers", + bool(static_proxies and (walls or support_surfaces)), + 0.16, + "walls/support surfaces are exported as static structure layers", + {"static_collision_proxy_count": len(static_proxies), "wall_count": len(walls), "support_surface_count": len(support_surfaces)}, + ), + readiness_gate( + "physics_collision", + proxy_count > 0, + 0.14, + "semantic collision proxies are injected into MJCF", + {"scene_structure_collision_proxies": proxy_count}, + ), + readiness_gate( + "navigation_surface_hints", + bool(navigation_surfaces), + 0.07, + "walkable or placeable navigation surfaces are available", + {"navigation_surface_count": len(navigation_surfaces)}, + ), + readiness_gate( + "movable_asset_instances", + commercial_movable_assets, + 0.18, + "movable objects are separated from the static scene as production-grade reconstructed assets", + { + "object_count": object_count, + "candidate_slot_count": candidate_slot_count, + "production_asset_count": production_asset_count, + "provisional_asset_count": provisional_asset_count, + "incomplete_asset_count": incomplete_asset_count, + "commercial_asset_grade": commercial_movable_assets, + }, + ), + readiness_gate( + "physical_material_hints", + static_materials_ready and bool(movable_physics["passed"]), + 0.08, + "static structure and movable assets include mass, friction, restitution, and contact-role physical material hints where applicable", + { + "static_material_hint_count": static_material_hint_count, + "static_proxy_count": len(static_proxies), + "movable_material_hint_count": movable_physics["ready_count"], + "movable_asset_count": movable_physics["asset_count"], + "movable_material_incomplete_count": movable_physics["incomplete_count"], + "movable_material_blockers": movable_physics["blockers"], + }, + ), + readiness_gate( + "task_affordance_targets", + bool(task_affordance_summary.get("passed")), + 0.04, + "production movable assets expose finite task poses and actionable contact roles for robot manipulation probes", + { + "score": task_affordance_summary.get("score"), + "target_count": task_affordance_summary.get("target_count"), + "ready_target_count": task_affordance_summary.get("ready_target_count"), + "blockers": task_affordance_summary.get("blockers"), + }, + ), + readiness_gate( + "room_completeness", + bool(room_summary.get("passed")), + 0.07, + "capture or explicit completion contains enough structure for floor plus wall-level room context", + { + "score": room_summary.get("score"), + "has_observed_floor": room_summary.get("has_observed_floor"), + "has_floor_completion": room_summary.get("has_floor_completion"), + "wall_count": len(walls), + "missing_required_elements": room_summary.get("missing_required_elements"), + }, + ), + readiness_gate( + "metric_scale_plausibility", + bool(metric_scale_summary.get("passed")), + 0.05, + "scene bounds and structure proxies are finite and plausible for metric robot simulation", + { + "score": metric_scale_summary.get("score"), + "bounds_extent": metric_scale_summary.get("bounds_extent"), + "horizontal_extent_range": metric_scale_summary.get("horizontal_extent_range"), + "vertical_extent": metric_scale_summary.get("vertical_extent"), + "blockers": metric_scale_summary.get("blockers"), + }, + ), + readiness_gate( + "navigation_acceptance", + bool(navigation_summary.get("passed")), + 0.03, + "robot navigation has a walkable surface with enough metric footprint for route planning", + { + "score": navigation_summary.get("score"), + "walkable_surface_count": navigation_summary.get("walkable_surface_count"), + "largest_walkable_area": navigation_summary.get("largest_walkable_area"), + "blockers": navigation_summary.get("blockers"), + }, + ), + ] + total_weight = sum(float(gate["weight"]) for gate in gates) + raw_score = sum(float(gate["weight"]) for gate in gates if gate["passed"]) + score = round(raw_score / max(total_weight, 1e-9), 6) + blockers = [gate["id"] for gate in gates if not gate["passed"]] + return { + "format": "gs-playground-real2sim-commercial-readiness-v1", + "summary": { + "score": score, + "target": 0.85, + "commercial_grade": score >= 0.85 and not blockers, + "blocker_count": len(blockers), + "blockers": blockers, + }, + "gates": gates, + "evidence": { + "object_count": object_count, + "movable_candidate_slot_count": candidate_slot_count, + "movable_production_asset_count": production_asset_count, + "movable_provisional_asset_count": provisional_asset_count, + "movable_incomplete_asset_count": incomplete_asset_count, + "movable_commercial_asset_grade": commercial_movable_assets, + "movable_physics_material_ready_count": movable_physics["ready_count"], + "movable_physics_material_incomplete_count": movable_physics["incomplete_count"], + "task_affordance_score": task_affordance_summary.get("score"), + "scene_structure_score": scene_structure_report.get("structure_score"), + "semantic_structure_score": scene_structure_report.get("commercial_readiness", {}).get("semantic_structure_score") + if isinstance(scene_structure_report.get("commercial_readiness"), Mapping) + else None, + "mesh_cleanup_kept_area_ratio": mesh_cleanup.get("kept_area_ratio") if isinstance(mesh_cleanup, Mapping) else None, + "texture_coverage_ratio": texture_metrics.get("texture_coverage_ratio"), + "textured_face_ratio": texture_metrics.get("textured_face_ratio"), + "projected_face_ratio": texture_metrics.get("projected_face_ratio"), + "atlas_pixel_coverage_ratio": texture_metrics.get("atlas_pixel_coverage_ratio"), + "room_completeness_score": room_summary.get("score"), + "metric_scale_score": metric_scale_summary.get("score"), + "navigation_acceptance_score": navigation_summary.get("score"), + "material_readiness_score": material_summary.get("score"), + }, + "next_actions": commercial_next_actions(blockers), + } + + +def commercial_acceptance_manifest( + root: Path, + *, + commercial_readiness: Mapping[str, Any], + room_completeness: Mapping[str, Any], + metric_scale: Mapping[str, Any], + navigation_acceptance: Mapping[str, Any], + material_readiness: Mapping[str, Any], + task_affordance: Mapping[str, Any], + movable_assets: Mapping[str, Any], + structure_layers: Mapping[str, Any], +) -> dict[str, Any]: + reports = { + "commercial_readiness": "artifacts/quality/commercial_readiness.json", + "room_completeness": "artifacts/quality/room_completeness.json", + "metric_scale": "artifacts/quality/metric_scale.json", + "navigation_acceptance": "artifacts/quality/navigation_acceptance.json", + "material_readiness": "artifacts/quality/material_readiness.json", + "task_affordance": "artifacts/quality/task_affordance.json", + "movable_assets": "artifacts/assets/movable_assets.json", + "static_structure_layers": "artifacts/scene_structure/static_structure.json", + } + entrypoints = { + "motrix_import": "motrix_import.json", + "package_manifest": "package_manifest.json", + "scene_xml": "mjcf/scene.xml", + "viewer_config": "configs/go2_real2sim.json", + "scene_gaussians": "3dgs/scene.ply", + } + entrypoint_audits = {key: package_file_audit(root, path) for key, path in entrypoints.items()} + report_audits = {key: package_file_audit(root, path) for key, path in reports.items()} + manifest = read_json(root / "package_manifest.json") if (root / "package_manifest.json").is_file() else {} + metadata = manifest.get("metadata") if isinstance(manifest, Mapping) and isinstance(manifest.get("metadata"), Mapping) else {} + checks = [ + acceptance_check( + "package_entrypoints", + {}, + explicit_passed=all(item["present"] and item["size"] > 0 for item in entrypoint_audits.values()), + evidence=entrypoint_audits, + ), + acceptance_check( + "quality_report_files", + {}, + explicit_passed=all(item["present"] and item["size"] > 0 for item in report_audits.values()), + evidence=report_audits, + ), + acceptance_check("commercial_readiness", commercial_readiness, summary_key="commercial_grade"), + acceptance_check("room_completeness", room_completeness), + acceptance_check("metric_scale", metric_scale), + acceptance_check("navigation_acceptance", navigation_acceptance), + acceptance_check("material_readiness", material_readiness), + acceptance_check("task_affordance", task_affordance), + acceptance_check( + "movable_assets", + movable_assets, + summary_key="commercial_asset_grade", + quality_summary=True, + ), + acceptance_check( + "static_structure_layers", + structure_layers, + explicit_passed=bool( + structure_layers.get("static_collision_proxies") + and structure_layers.get("navigation_surfaces") + and structure_layers.get("quality", {}).get("total_collision_proxy_count") + ) + if isinstance(structure_layers, Mapping) + else False, + ), + ] + blockers = [check["id"] for check in checks if not check["passed"]] + score = sum(1 for check in checks if check["passed"]) / float(max(len(checks), 1)) + return { + "format": "gs-playground-real2sim-commercial-acceptance-v1", + "package_root": root.as_posix(), + "summary": { + "score": round(float(score), 6), + "target": 1.0, + "accepted": not blockers, + "check_count": len(checks), + "passed_check_count": len(checks) - len(blockers), + "blocker_count": len(blockers), + "blockers": blockers, + }, + "package": { + "format": metadata.get("package_format"), + "robot": metadata.get("robot"), + "motrix_scene_visual_mode": metadata.get("motrix_scene_visual_mode"), + "motrix_visual_maxgeom": metadata.get("motrix_visual_maxgeom"), + }, + "entrypoints": entrypoints, + "reports": reports, + "artifact_integrity": { + "entrypoints": entrypoint_audits, + "reports": report_audits, + }, + "checks": checks, + "next_actions": commercial_next_actions( + commercial_readiness.get("summary", {}).get("blockers", []) + if isinstance(commercial_readiness.get("summary"), Mapping) + else [] + ), + "notes": [ + "This manifest is the package-level commercial delivery ledger.", + "It summarizes generated quality artifacts and points consumers to the detailed reports.", + ], + } + + +def package_file_audit(root: Path, package_path: str) -> dict[str, Any]: + path = root / package_path + present = path.is_file() + size = path.stat().st_size if present else 0 + return { + "path": package_path, + "present": present, + "size": size, + "sha256": sha256_file(path) if present and size > 0 else None, + } + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def acceptance_check( + check_id: str, + report: Mapping[str, Any], + *, + summary_key: str = "passed", + quality_summary: bool = False, + explicit_passed: bool | None = None, + evidence: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + if explicit_passed is not None: + return {"id": check_id, "passed": bool(explicit_passed), "summary": {}, "evidence": dict(evidence or {})} + summary = report.get("quality") if quality_summary else report.get("summary") + summary = summary if isinstance(summary, Mapping) else {} + return { + "id": check_id, + "passed": bool(summary.get(summary_key)), + "summary": dict(summary), + "evidence": dict(evidence or {}), + } + + +def material_readiness_report(root: Path) -> dict[str, Any]: + manifest = read_json(root / "package_manifest.json") if (root / "package_manifest.json").is_file() else {} + scene = manifest.get("scene") if isinstance(manifest, Mapping) else {} + objects = manifest.get("objects") if isinstance(manifest.get("objects"), list) else [] + scene_mesh = resolve_package_relative_path(root, scene.get("mesh_obj")) if isinstance(scene, Mapping) else None + object_meshes = [ + path + for entry in objects + if isinstance(entry, Mapping) + for path in [resolve_package_relative_path(root, entry.get("mesh_obj"))] + if path is not None + ] + mjcf_materials = mjcf_material_readiness(root / "mjcf" / "scene.xml", root) + mesh_materials = obj_material_readiness([path for path in [scene_mesh, *object_meshes] if path is not None], root) + all_texture_files = sorted({*mjcf_materials["texture_files"], *mesh_materials["texture_files"]}) + missing_texture_files = sorted({*mjcf_materials["missing_texture_files"], *mesh_materials["missing_texture_files"]}) + scene_mesh_has_texture = bool(scene_mesh is not None and mesh_materials["mesh_texture_counts"].get(scene_mesh.as_posix(), 0) > 0) + texture_backed_material_count = int(mjcf_materials["texture_backed_material_count"]) + int(mesh_materials["texture_backed_material_count"]) + blockers: list[str] = [] + if scene_mesh is not None and not scene_mesh_has_texture: + blockers.append("scene_mesh_missing_texture_material") + if texture_backed_material_count <= 0: + blockers.append("missing_texture_backed_material") + if missing_texture_files: + blockers.append("missing_texture_files") + if scene_mesh is None and not object_meshes: + blockers.append("missing_mesh_assets") + score = 1.0 + if "missing_mesh_assets" in blockers: + score = 0.0 + elif "missing_texture_backed_material" in blockers: + score = 0.25 + elif "scene_mesh_missing_texture_material" in blockers: + score = 0.55 + elif "missing_texture_files" in blockers: + score = 0.7 + return { + "format": "gs-playground-real2sim-material-readiness-v1", + "summary": { + "score": round(score, 6), + "passed": not blockers, + "scene_mesh_present": scene_mesh is not None, + "scene_mesh_has_texture": scene_mesh_has_texture, + "object_mesh_count": len(object_meshes), + "texture_backed_material_count": texture_backed_material_count, + "texture_file_count": len(all_texture_files), + "missing_texture_file_count": len(missing_texture_files), + "blockers": blockers, + }, + "inputs": { + "package_manifest": "package_manifest.json", + "scene_xml": "mjcf/scene.xml", + "scene_mesh": scene.get("mesh_obj") if isinstance(scene, Mapping) else None, + "object_meshes": [path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix() for path in object_meshes], + }, + "evidence": { + "mjcf": mjcf_materials["evidence"], + "meshes": mesh_materials["evidence"], + "texture_files": [path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix() for path in all_texture_files], + "missing_texture_files": [path.relative_to(root).as_posix() if path.is_relative_to(root) else path.as_posix() for path in missing_texture_files], + }, + "diagnostics": material_readiness_diagnostics(blockers), + } + + +def resolve_package_relative_path(root: Path, value: Any) -> Path | None: + if not value: + return None + path = Path(str(value)) + if path.is_absolute(): + return path if path.is_file() else None + resolved = (root / path).resolve() + return resolved if resolved.is_file() else None + + +def mjcf_material_readiness(scene_xml: Path, root: Path) -> dict[str, Any]: + if not scene_xml.is_file(): + return {"texture_files": [], "missing_texture_files": [], "texture_backed_material_count": 0, "evidence": {"present": False}} + try: + tree = ET.parse(scene_xml) + except ET.ParseError: + return {"texture_files": [], "missing_texture_files": [], "texture_backed_material_count": 0, "evidence": {"present": True, "parse_error": True}} + texture_by_name: dict[str, Path] = {} + for texture in tree.findall(".//texture"): + name = texture.get("name") + file_value = texture.get("file") + if not name or not file_value: + continue + path = (scene_xml.parent / "../meshes" / file_value).resolve() + if (root / file_value).is_file(): + path = (root / file_value).resolve() + texture_by_name[name] = path + backed = 0 + for material in tree.findall(".//material"): + if material.get("texture") in texture_by_name: + backed += 1 + texture_files = [path for path in texture_by_name.values() if path.is_file()] + missing = [path for path in texture_by_name.values() if not path.is_file()] + return { + "texture_files": texture_files, + "missing_texture_files": missing, + "texture_backed_material_count": backed, + "evidence": { + "present": True, + "texture_count": len(texture_by_name), + "material_with_texture_count": backed, + "missing_texture_file_count": len(missing), + }, + } + + +def obj_material_readiness(meshes: list[Path], root: Path) -> dict[str, Any]: + texture_files: list[Path] = [] + missing: list[Path] = [] + backed = 0 + mesh_texture_counts: dict[str, int] = {} + mesh_evidence: list[dict[str, Any]] = [] + for mesh in meshes: + mesh_textures, mesh_missing, material_count = obj_texture_materials(mesh) + texture_files.extend(mesh_textures) + missing.extend(mesh_missing) + backed += material_count + mesh_texture_counts[mesh.as_posix()] = material_count + mesh_evidence.append( + { + "mesh": mesh.relative_to(root).as_posix() if mesh.is_relative_to(root) else mesh.as_posix(), + "texture_backed_material_count": material_count, + "texture_file_count": len(mesh_textures), + "missing_texture_file_count": len(mesh_missing), + } + ) + return { + "texture_files": texture_files, + "missing_texture_files": missing, + "texture_backed_material_count": backed, + "mesh_texture_counts": mesh_texture_counts, + "evidence": mesh_evidence, + } + + +def obj_texture_materials(mesh: Path) -> tuple[list[Path], list[Path], int]: + texture_files: list[Path] = [] + missing: list[Path] = [] + material_count = 0 + if not mesh.is_file(): + return texture_files, missing, material_count + mtl_names: list[str] = [] + for line in mesh.read_text(encoding="utf-8", errors="ignore").splitlines(): + if line.startswith("mtllib "): + mtl_names.extend(line.split()[1:]) + for name in mtl_names: + mtl = (mesh.parent / name).resolve() + if not mtl.is_file(): + missing.append(mtl) + continue + for texture_name in mtl_diffuse_texture_names(mtl): + texture_path = (mtl.parent / texture_name).resolve() + material_count += 1 + if texture_path.is_file(): + texture_files.append(texture_path) + else: + missing.append(texture_path) + return texture_files, missing, material_count + + +def mtl_diffuse_texture_names(mtl: Path) -> list[str]: + names: list[str] = [] + for line in mtl.read_text(encoding="utf-8", errors="ignore").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + parts = stripped.split() + if parts and parts[0] == "map_Kd" and len(parts) >= 2: + names.append(parts[-1]) + return names + + +def metric_scale_report(root: Path, structure_layers: Mapping[str, Any], scene_structure_report: Mapping[str, Any]) -> dict[str, Any]: + bounds = scene_structure_report.get("bounds") if isinstance(scene_structure_report.get("bounds"), Mapping) else {} + low = numeric_triplet(bounds.get("p05") or bounds.get("min")) if isinstance(bounds, Mapping) else None + high = numeric_triplet(bounds.get("p95") or bounds.get("max")) if isinstance(bounds, Mapping) else None + extents = tuple(max(float(high[idx] - low[idx]), 0.0) for idx in range(3)) if low is not None and high is not None else None + up_axis_name = str(scene_structure_report.get("up_axis") or "z") + up_axis = "xyz".find(up_axis_name) + if up_axis < 0: + up_axis = 2 + horizontal = [extents[idx] for idx in range(3) if idx != up_axis] if extents is not None else [] + horizontal_max = max(horizontal) if horizontal else 0.0 + horizontal_min = min(horizontal) if horizontal else 0.0 + vertical = extents[up_axis] if extents is not None else 0.0 + proxies = structure_layers.get("static_collision_proxies") if isinstance(structure_layers, Mapping) else [] + proxies = proxies if isinstance(proxies, list) else [] + proxy_sizes: list[tuple[float, float, float]] = [] + for proxy in proxies: + if not isinstance(proxy, Mapping): + continue + size = numeric_triplet(proxy.get("size")) + if size is not None: + proxy_sizes.append(size) + proxy_max_extent = max((max(size) for size in proxy_sizes), default=0.0) + proxy_min_extent = min((min(size) for size in proxy_sizes), default=0.0) + blockers: list[str] = [] + if extents is None: + blockers.append("missing_metric_bounds") + elif not all(0.0 < value < 50.0 for value in extents): + blockers.append("invalid_or_extreme_bounds") + if extents is not None and not (0.5 <= horizontal_max <= 20.0 and 0.2 <= horizontal_min <= 20.0): + blockers.append("implausible_horizontal_extent") + if extents is not None and not (0.4 <= vertical <= 6.0): + blockers.append("implausible_vertical_extent") + if not proxy_sizes: + blockers.append("missing_structure_proxy_sizes") + elif not (0.005 <= proxy_min_extent and proxy_max_extent <= 20.0): + blockers.append("implausible_structure_proxy_size") + score = 1.0 + if "missing_metric_bounds" in blockers or "invalid_or_extreme_bounds" in blockers: + score -= 0.45 + if "implausible_horizontal_extent" in blockers: + score -= 0.22 + if "implausible_vertical_extent" in blockers: + score -= 0.18 + if "missing_structure_proxy_sizes" in blockers or "implausible_structure_proxy_size" in blockers: + score -= 0.15 + score = round(max(score, 0.0), 6) + return { + "format": "gs-playground-real2sim-metric-scale-v1", + "package_root": root.as_posix(), + "summary": { + "score": score, + "target": 0.8, + "passed": score >= 0.8 and not blockers, + "blocker_count": len(blockers), + "blockers": blockers, + "bounds_extent": [round(value, 6) for value in extents] if extents is not None else None, + "horizontal_extent_range": [round(horizontal_min, 6), round(horizontal_max, 6)] if horizontal else None, + "vertical_extent": round(float(vertical), 6) if extents is not None else None, + "structure_proxy_count": len(proxy_sizes), + "structure_proxy_extent_range": [round(float(proxy_min_extent), 6), round(float(proxy_max_extent), 6)] if proxy_sizes else None, + }, + "inputs": { + "scene_structure_report": "artifacts/scene_structure/scene_structure_report.json", + "static_structure_layers": "artifacts/scene_structure/static_structure.json", + "bounds": scene_structure_report.get("bounds"), + "up_axis": up_axis_name if up_axis_name in {"x", "y", "z"} else "z", + }, + "diagnostics": metric_scale_diagnostics(blockers), + } + + +def navigation_acceptance_report(root: Path, structure_layers: Mapping[str, Any]) -> dict[str, Any]: + surfaces = structure_layers.get("navigation_surfaces") if isinstance(structure_layers, Mapping) else [] + surfaces = surfaces if isinstance(surfaces, list) else [] + walkable: list[dict[str, Any]] = [] + for surface in surfaces: + if not isinstance(surface, Mapping): + continue + role = str(surface.get("navigation_role") or "") + if not role.startswith("walkable_floor"): + continue + extent = surface.get("extent") + size = navigation_surface_size(extent) + area = size[0] * size[1] if size is not None else 0.0 + walkable.append( + { + "id": surface.get("id"), + "semantic_label": surface.get("semantic_label"), + "navigation_role": role, + "inferred_completion": bool(surface.get("inferred_completion")), + "size": [round(float(value), 6) for value in size] if size is not None else None, + "area": round(float(area), 6), + } + ) + largest_area = max((float(item["area"]) for item in walkable), default=0.0) + blockers: list[str] = [] + if not walkable: + blockers.append("missing_walkable_floor_surface") + if walkable and largest_area < 0.5: + blockers.append("walkable_surface_too_small") + if walkable and all(item.get("inferred_completion") for item in walkable): + blockers.append("walkable_surface_inferred_only") + score = 1.0 + if "missing_walkable_floor_surface" in blockers: + score -= 0.55 + if "walkable_surface_too_small" in blockers: + score -= 0.25 + if "walkable_surface_inferred_only" in blockers: + score -= 0.2 + score = round(max(score, 0.0), 6) + return { + "format": "gs-playground-real2sim-navigation-acceptance-v1", + "package_root": root.as_posix(), + "summary": { + "score": score, + "target": 0.8, + "passed": score >= 0.8 and not blockers, + "blocker_count": len(blockers), + "blockers": blockers, + "navigation_surface_count": len(surfaces), + "walkable_surface_count": len(walkable), + "largest_walkable_area": round(float(largest_area), 6), + }, + "inputs": { + "static_structure_layers": "artifacts/scene_structure/static_structure.json", + }, + "walkable_surfaces": walkable, + "diagnostics": navigation_acceptance_diagnostics(blockers), + } + + +def navigation_surface_size(extent: Any) -> tuple[float, float] | None: + if not isinstance(extent, Mapping): + return None + spans: list[float] = [] + for axis in "xyz": + values = extent.get(axis) + if isinstance(values, list) and len(values) == 2: + try: + spans.append(max(float(values[1]) - float(values[0]), 0.0)) + except (TypeError, ValueError): + return None + if len(spans) != 3: + return None + horizontal = sorted(spans, reverse=True)[:2] + return float(horizontal[0]), float(horizontal[1]) + + +def texture_quality_metrics(texture_report: Any) -> dict[str, Any]: + if not isinstance(texture_report, Mapping): + return { + "passed": False, + "texture_coverage_ratio": None, + "atlas_pixel_coverage_ratio": None, + "textured_face_ratio": None, + "projected_face_ratio": None, + "reason": "texture bake report is missing", + } + face_count = int(texture_report.get("face_count") or texture_report.get("selected_face_count") or 0) + selected_count = int(texture_report.get("selected_face_count") or face_count or 0) + drawn_faces = int(texture_report.get("drawn_face_count") or 0) + projected_faces = int(texture_report.get("projected_face_count") or 0) + denominator = max(selected_count or face_count, 1) + textured_face_ratio = texture_report.get("textured_face_ratio") + if not isinstance(textured_face_ratio, (int, float)): + textured_face_ratio = drawn_faces / float(denominator) + projected_face_ratio = texture_report.get("projected_face_ratio") + if not isinstance(projected_face_ratio, (int, float)): + projected_face_ratio = projected_faces / float(denominator) + atlas_pixel_coverage = texture_report.get("atlas_pixel_coverage_ratio") + if not isinstance(atlas_pixel_coverage, (int, float)): + atlas_pixel_coverage = texture_report.get("texture_coverage_ratio") + texture_coverage = texture_report.get("texture_coverage_ratio") + passed = bool(float(textured_face_ratio) >= 0.95 and float(projected_face_ratio) >= 0.90) + if not passed and isinstance(atlas_pixel_coverage, (int, float)): + passed = float(atlas_pixel_coverage) >= 0.45 and float(textured_face_ratio) >= 0.80 + return { + "passed": passed, + "texture_coverage_ratio": texture_coverage, + "atlas_pixel_coverage_ratio": round(float(atlas_pixel_coverage), 6) if isinstance(atlas_pixel_coverage, (int, float)) else None, + "textured_face_ratio": round(float(textured_face_ratio), 6), + "projected_face_ratio": round(float(projected_face_ratio), 6), + "drawn_face_count": drawn_faces, + "projected_face_count": projected_faces, + "selected_face_count": selected_count, + "face_count": face_count, + "acceptance_basis": "mesh_face_coverage", + "note": "Atlas pixel coverage is retained as a UV packing diagnostic; the commercial gate uses mesh face coverage to avoid penalizing valid UV island padding.", + } + + +def movable_physics_material_quality(assets: Any) -> dict[str, Any]: + entries = assets if isinstance(assets, list) else [] + ready_count = 0 + blockers: list[str] = [] + details: list[dict[str, Any]] = [] + for index, asset in enumerate(entries): + if not isinstance(asset, Mapping): + blockers.append("invalid_movable_asset_entry") + details.append({"index": index, "ready": False, "blockers": ["invalid_movable_asset_entry"]}) + continue + material = asset.get("physics_material") + item_blockers = movable_physics_material_blockers(material) + if item_blockers: + blockers.extend(item_blockers) + else: + ready_count += 1 + details.append( + { + "id": asset.get("id") or asset.get("name") or f"asset_{index:03d}", + "ready": not item_blockers, + "blockers": item_blockers, + } + ) + unique_blockers = sorted(set(blockers)) + return { + "asset_count": len(entries), + "ready_count": ready_count, + "incomplete_count": len(entries) - ready_count, + "passed": not entries or ready_count == len(entries), + "blockers": unique_blockers, + "details": details, + } + + +def task_affordance_report(root: Path, movable_assets: Mapping[str, Any]) -> dict[str, Any]: + assets = movable_assets.get("movable_assets") if isinstance(movable_assets.get("movable_assets"), list) else [] + targets: list[dict[str, Any]] = [] + aggregate_blockers: list[str] = [] + for index, asset in enumerate(assets): + if not isinstance(asset, Mapping): + blockers = ["invalid_movable_asset_entry"] + aggregate_blockers.extend(blockers) + targets.append({"id": f"asset_{index:03d}", "ready": False, "blockers": blockers}) + continue + pose = asset.get("pose") if isinstance(asset.get("pose"), Mapping) else {} + material = asset.get("physics_material") if isinstance(asset.get("physics_material"), Mapping) else {} + reconstruction = asset.get("reconstruction_quality") if isinstance(asset.get("reconstruction_quality"), Mapping) else {} + blockers = task_affordance_blockers(asset, pose, material, reconstruction) + aggregate_blockers.extend(blockers) + targets.append( + { + "id": asset.get("id") or asset.get("name") or f"asset_{index:03d}", + "label": asset.get("label"), + "dynamic": bool(asset.get("dynamic")), + "contact_role": material.get("contact_role"), + "pose": { + "position": pose.get("position"), + "quat_wxyz": pose.get("quat_wxyz"), + "scale": pose.get("scale"), + }, + "ready": not blockers, + "blockers": blockers, + } + ) + ready_count = sum(1 for target in targets if target.get("ready")) + blockers = sorted(set(aggregate_blockers)) + score = ready_count / float(max(len(targets), 1)) if targets else 0.0 + return { + "format": "gs-playground-real2sim-task-affordance-v1", + "package_root": root.as_posix(), + "summary": { + "score": round(float(score), 6), + "target": 1.0, + "passed": bool(targets and ready_count == len(targets) and not blockers), + "target_count": len(targets), + "ready_target_count": ready_count, + "blocker_count": len(blockers), + "blockers": blockers, + }, + "inputs": { + "movable_asset_layers": "artifacts/assets/movable_assets.json", + }, + "targets": targets, + "diagnostics": task_affordance_diagnostics(blockers), + } + + +def task_affordance_blockers( + asset: Mapping[str, Any], + pose: Mapping[str, Any], + material: Mapping[str, Any], + reconstruction: Mapping[str, Any], +) -> list[str]: + blockers: list[str] = [] + if not bool(asset.get("dynamic")): + blockers.append("target_not_dynamic") + if not reconstruction.get("commercial_asset_grade"): + blockers.append("target_not_production_asset") + if material.get("contact_role") not in {"graspable", "pushable", "movable_asset"}: + blockers.append("missing_actionable_contact_role") + if not finite_pose_triplet(pose.get("position"), max_abs=10.0): + blockers.append("invalid_target_position") + if not finite_pose_triplet(pose.get("scale"), max_abs=20.0, positive=True): + blockers.append("invalid_target_scale") + quat = pose.get("quat_wxyz") + if not isinstance(quat, (list, tuple)) or len(quat) != 4: + blockers.append("invalid_target_orientation") + else: + try: + if not all(np.isfinite(float(value)) for value in quat): + blockers.append("invalid_target_orientation") + except (TypeError, ValueError): + blockers.append("invalid_target_orientation") + return blockers + + +def finite_pose_triplet(value: Any, *, max_abs: float, positive: bool = False) -> bool: + if not isinstance(value, (list, tuple)) or len(value) != 3: + return False + try: + values = [float(item) for item in value] + except (TypeError, ValueError): + return False + if not all(np.isfinite(item) and abs(item) <= max_abs for item in values): + return False + if positive and not all(item > 0.0 for item in values): + return False + return True + + +def movable_physics_material_blockers(material: Any) -> list[str]: + if not isinstance(material, Mapping): + return ["missing_movable_physics_material"] + blockers: list[str] = [] + if material.get("movable") is not True: + blockers.append("movable_material_not_marked_movable") + mass = material.get("mass") + if not isinstance(mass, (int, float)) or not (0.02 <= float(mass) <= 5.0): + blockers.append("invalid_or_implausible_movable_mass") + friction = material.get("friction") + if not valid_friction_triplet(friction): + blockers.append("invalid_movable_friction") + role = str(material.get("contact_role") or "") + if role not in {"graspable", "pushable", "movable_asset"}: + blockers.append("missing_movable_contact_role") + restitution = material.get("restitution") + if not isinstance(restitution, (int, float)) or not (0.0 <= float(restitution) <= 1.0): + blockers.append("invalid_movable_restitution") + return blockers + + +def valid_static_physics_material(material: Any) -> bool: + if not isinstance(material, Mapping): + return False + if material.get("movable") is not False: + return False + if not valid_friction_triplet(material.get("friction")): + return False + return bool(str(material.get("contact_role") or "")) + + +def valid_friction_triplet(value: Any) -> bool: + if not isinstance(value, (list, tuple)) or len(value) != 3: + return False + try: + slide, torsion, rolling = (float(value[0]), float(value[1]), float(value[2])) + except (TypeError, ValueError): + return False + return slide >= 0.2 and torsion >= 0.0 and rolling >= 0.0 + + +def room_completeness_report(root: Path, structure_layers: Mapping[str, Any], scene_structure_report: Mapping[str, Any]) -> dict[str, Any]: + walls = structure_layers.get("walls") if isinstance(structure_layers, Mapping) else [] + support_surfaces = structure_layers.get("support_surfaces") if isinstance(structure_layers, Mapping) else [] + navigation_surfaces = structure_layers.get("navigation_surfaces") if isinstance(structure_layers, Mapping) else [] + completion_proxies = structure_layers.get("completion_proxies") if isinstance(structure_layers, Mapping) else [] + walls = walls if isinstance(walls, list) else [] + support_surfaces = support_surfaces if isinstance(support_surfaces, list) else [] + navigation_surfaces = navigation_surfaces if isinstance(navigation_surfaces, list) else [] + completion_proxies = completion_proxies if isinstance(completion_proxies, list) else [] + readiness = scene_structure_report.get("commercial_readiness") + readiness = readiness if isinstance(readiness, Mapping) else {} + has_observed_floor = bool(readiness.get("has_floor")) or any( + isinstance(surface, Mapping) and surface.get("navigation_role") == "walkable_floor" for surface in navigation_surfaces + ) + has_floor_completion = any( + isinstance(proxy, Mapping) and proxy.get("semantic_label") == "floor_completion" for proxy in completion_proxies + ) + wall_count = len(walls) + support_count = len(support_surfaces) + navigation_count = len(navigation_surfaces) + score = ( + (0.45 if has_observed_floor else 0.34 if has_floor_completion else 0.0) + + min(wall_count, 2) * 0.18 + + (0.12 if support_count else 0.0) + + (0.07 if navigation_count else 0.0) + ) + score = round(min(score, 1.0), 6) + missing: list[str] = [] + if not (has_observed_floor or has_floor_completion): + missing.append("floor_or_floor_completion") + if wall_count < 2: + missing.append("at_least_two_wall_planes") + if navigation_count < 1: + missing.append("walkable_or_placeable_navigation_surface") + passed = score >= 0.7 and not missing + return { + "format": "gs-playground-real2sim-room-completeness-v1", + "package_root": root.as_posix(), + "summary": { + "score": score, + "target": 0.7, + "passed": passed, + "has_observed_floor": has_observed_floor, + "has_floor_completion": has_floor_completion, + "wall_count": wall_count, + "support_surface_count": support_count, + "navigation_surface_count": navigation_count, + "completion_proxy_count": len(completion_proxies), + "missing_required_elements": missing, + }, + "inputs": { + "scene_structure_report": "artifacts/scene_structure/scene_structure_report.json", + "static_structure_layers": "artifacts/scene_structure/static_structure.json", + "bounds": scene_structure_report.get("bounds"), + "semantic_planes_count": len(scene_structure_report.get("semantic_planes", [])) + if isinstance(scene_structure_report.get("semantic_planes"), list) + else 0, + }, + "outputs": { + "completion_proxies": completion_proxies, + "navigation_surfaces": navigation_surfaces, + }, + "diagnostics": room_completeness_diagnostics( + has_observed_floor=has_observed_floor, + has_floor_completion=has_floor_completion, + wall_count=wall_count, + support_count=support_count, + navigation_count=navigation_count, + ), + } + + +def movable_asset_layers(root: Path, *, structure_layers: Mapping[str, Any] | None = None) -> dict[str, Any]: + manifest = read_json(root / "package_manifest.json") if (root / "package_manifest.json").is_file() else {} + objects = manifest.get("objects") if isinstance(manifest, Mapping) else [] + assets: list[dict[str, Any]] = [] + if isinstance(objects, list): + for index, asset in enumerate(objects): + if not isinstance(asset, Mapping): + continue + name = str(asset.get("name") or f"object_{index:03d}") + pose = asset.get("pose") if isinstance(asset.get("pose"), Mapping) else {} + assets.append( + { + "id": slug_token(name), + "name": name, + "label": asset.get("label"), + "dynamic": bool(asset.get("dynamic", True)), + "source_instance_id": asset.get("source_instance_id"), + "gaussian_ply": asset.get("gaussian_ply"), + "mesh_obj": asset.get("mesh_obj"), + "pose": { + "position": pose.get("position", [0.0, 0.0, 0.0]), + "quat_wxyz": pose.get("quat_wxyz", [1.0, 0.0, 0.0, 0.0]), + "scale": pose.get("scale", [1.0, 1.0, 1.0]), + }, + "physics_material": default_movable_physics_material(str(asset.get("label") or name)), + "reconstruction_quality": movable_asset_reconstruction_quality(asset), + } + ) + candidate_slots = movable_candidate_slots(structure_layers or {}, enabled=not assets) + production_assets = [asset for asset in assets if asset["reconstruction_quality"]["commercial_asset_grade"]] + provisional_assets = [asset for asset in assets if asset["reconstruction_quality"]["reconstruction_source"] == "provisional_extraction_task"] + incomplete_assets = [asset for asset in assets if asset["reconstruction_quality"]["asset_completeness_blockers"]] + physics_material_quality = movable_physics_material_quality(assets) + return { + "format": "gs-playground-real2sim-movable-assets-v1", + "movable_assets": assets, + "candidate_slots": candidate_slots, + "quality": { + "movable_asset_count": len(assets), + "dynamic_asset_count": sum(1 for asset in assets if asset.get("dynamic")), + "has_separate_movable_assets": bool(assets), + "candidate_slot_count": len(candidate_slots), + "production_asset_count": len(production_assets), + "provisional_asset_count": len(provisional_assets), + "incomplete_asset_count": len(incomplete_assets), + "physics_material_ready_count": physics_material_quality["ready_count"], + "physics_material_incomplete_count": physics_material_quality["incomplete_count"], + "commercial_asset_grade": bool(assets) and len(production_assets) == len(assets), + }, + "commercial_gap_notes": [ + "Commercial-grade Real2Sim requires movable assets to be segmented and reconstructed separately from the static scene.", + "Candidate slots identify support/navigation surfaces where object segmentation and reconstruction should run next.", + "An empty movable asset list means the package is scene-only even if movable objects are visually embedded in the mesh.", + ] + if not assets + else [ + "Movable assets exist, but at least one asset is provisional and should be replaced by SAM/SAM3D or multi-view object reconstruction." + ] + if provisional_assets + else [ + "Movable assets exist, but at least one asset is missing gaussian, mesh, or pose artifacts required for commercial import." + ] + if incomplete_assets + else [], + } + + +def movable_asset_reconstruction_quality(asset: Mapping[str, Any]) -> dict[str, Any]: + source_id = str(asset.get("source_instance_id") or "") + name = str(asset.get("name") or "") + explicit_provisional = bool(asset.get("provisional")) + inferred_provisional = explicit_provisional or source_id.startswith("candidate_slot_") or "_bottle" in name and source_id.startswith("candidate_") + source = "provisional_extraction_task" if inferred_provisional else "external_or_sam3d_reconstruction" + has_gaussian = bool(asset.get("gaussian_ply")) + has_mesh = bool(asset.get("mesh_obj")) + has_pose = isinstance(asset.get("pose"), Mapping) + completeness_blockers: list[str] = [] + if not has_gaussian: + completeness_blockers.append("missing_gaussian_asset") + if not has_mesh: + completeness_blockers.append("missing_mesh_asset") + if not has_pose: + completeness_blockers.append("missing_pose") + return { + "reconstruction_source": source, + "commercial_asset_grade": not inferred_provisional and not completeness_blockers, + "has_gaussian_asset": has_gaussian, + "has_mesh_asset": has_mesh, + "has_pose": has_pose, + "asset_completeness_blockers": completeness_blockers, + "source_instance_id": source_id or None, + "required_upgrade": "replace_with_sam3d_or_multiview_object_reconstruction" + if inferred_provisional + else "complete_gaussian_mesh_pose_assets" + if completeness_blockers + else None, + } + + +def movable_candidate_slots(structure_layers: Mapping[str, Any], *, enabled: bool) -> list[dict[str, Any]]: + if not enabled: + return [] + navigation = structure_layers.get("navigation_surfaces") if isinstance(structure_layers, Mapping) else [] + if not isinstance(navigation, list): + return [] + slots: list[dict[str, Any]] = [] + for index, surface in enumerate(navigation[:8]): + if not isinstance(surface, Mapping): + continue + role = str(surface.get("navigation_role") or "") + if role not in {"placeable_support", "walkable_floor"}: + continue + extent = surface.get("extent") if isinstance(surface.get("extent"), Mapping) else {} + slots.append( + { + "id": f"candidate_slot_{index:03d}", + "source_surface_id": surface.get("id"), + "source_surface_label": surface.get("semantic_label"), + "navigation_role": role, + "extent": extent, + "candidate_type": "movable_objects_on_support" if role == "placeable_support" else "movable_objects_on_floor", + "recommended_prompts": ["bottle", "cup", "book", "box", "monitor", "tool"] + if role == "placeable_support" + else ["chair", "box", "bag", "movable obstacle"], + "required_next_stage": "segment_and_reconstruct_objects", + } + ) + return slots + + +def movable_extraction_tasks(root: Path, motrix_import: Mapping[str, Any], movable_assets: Mapping[str, Any]) -> dict[str, Any]: + slots = movable_assets.get("candidate_slots") if isinstance(movable_assets, Mapping) else [] + if not isinstance(slots, list) or not slots: + return { + "format": "gs-playground-real2sim-movable-extraction-tasks-v1", + "tasks": [], + "quality": {"task_count": 0, "candidate_slot_count": 0}, + } + preview_value = motrix_import.get("preview") + preview_path = root / str(preview_value) if preview_value else None + task_dir = root / "artifacts" / "assets" / "movable_candidates" + task_dir.mkdir(parents=True, exist_ok=True) + try: + rgb = read_rgb_with_pillow_fallback(preview_path) if preview_path is not None else None + except Exception as exc: + return { + "format": "gs-playground-real2sim-movable-extraction-tasks-v1", + "tasks": [], + "quality": {"task_count": 0, "candidate_slot_count": len(slots)}, + "warnings": [f"failed to read preview image for movable extraction tasks: {exc}"], + } + if rgb is None: + return { + "format": "gs-playground-real2sim-movable-extraction-tasks-v1", + "tasks": [], + "quality": {"task_count": 0, "candidate_slot_count": len(slots)}, + "warnings": ["motrix import does not include a preview image"], + } + detections = image_movable_candidate_boxes(rgb, max_candidates=max(1, min(6, len(slots) * 3))) + tasks: list[dict[str, Any]] = [] + for slot_index, slot in enumerate(slots): + if not isinstance(slot, Mapping): + continue + prompts = slot.get("recommended_prompts") if isinstance(slot.get("recommended_prompts"), list) else ["movable object"] + for det_index, detection in enumerate(detections[:3]): + bbox = detection["bbox_xyxy"] + x0, y0, x1, y1 = bbox + crop = rgb[y0:y1, x0:x1, :] + mask = np.zeros(rgb.shape[:2], dtype=np.float32) + mask[y0:y1, x0:x1] = 1.0 + task_id = f"{slot.get('id', f'slot_{slot_index:03d}')}_{det_index:03d}" + crop_path = task_dir / f"{task_id}_crop.ppm" + mask_path = task_dir / f"{task_id}_mask.pgm" + write_rgb(crop_path, crop) + write_gray(mask_path, mask) + tasks.append( + { + "id": task_id, + "source_slot_id": slot.get("id"), + "source_surface_id": slot.get("source_surface_id"), + "prompt_candidates": prompts, + "bbox_xyxy": bbox, + "score": detection["score"], + "crop": crop_path.relative_to(root).as_posix(), + "mask": mask_path.relative_to(root).as_posix(), + "required_next_stage": "sam_segment_then_sam3d_reconstruct", + "expected_output": "objects_json entry with gaussian_ply, mesh_obj, pose, dynamic=true", + } + ) + return { + "format": "gs-playground-real2sim-movable-extraction-tasks-v1", + "preview": str(preview_value) if preview_value else None, + "tasks": tasks, + "quality": { + "task_count": len(tasks), + "candidate_slot_count": len(slots), + "image_candidate_count": len(detections), + }, + "commercial_gap_notes": [ + "These are extraction tasks, not completed movable assets.", + "Commercial readiness still requires the tasks to be segmented, reconstructed, and packaged as object assets.", + ], + } + + +def image_movable_candidate_boxes(rgb: np.ndarray, *, max_candidates: int) -> list[dict[str, Any]]: + height, width = rgb.shape[:2] + if height < 8 or width < 8: + return [] + gray = rgb[:, :, :3] @ np.array([0.299, 0.587, 0.114], dtype=np.float32) + saturation = rgb[:, :, :3].max(axis=2) - rgb[:, :, :3].min(axis=2) + gx = np.zeros_like(gray) + gy = np.zeros_like(gray) + gx[:, 1:-1] = np.abs(gray[:, 2:] - gray[:, :-2]) + gy[1:-1, :] = np.abs(gray[2:, :] - gray[:-2, :]) + saliency = saturation * 0.55 + (gx + gy) * 0.45 + y_min = int(height * 0.42) + saliency[:y_min, :] = 0.0 + threshold = float(np.percentile(saliency[y_min:, :], 92.0)) if y_min < height else float(np.percentile(saliency, 92.0)) + mask = saliency > max(threshold, 0.025) + components = connected_components(mask) + candidates: list[dict[str, Any]] = [] + for component in components: + ys, xs = component + if len(xs) < 12: + continue + x0, x1 = int(xs.min()), int(xs.max()) + 1 + y0, y1 = int(ys.min()), int(ys.max()) + 1 + bw = x1 - x0 + bh = y1 - y0 + area = len(xs) + if bw < width * 0.025 or bh < height * 0.025: + continue + if bw > width * 0.7 or bh > height * 0.7: + continue + margin_x = max(3, int(bw * 0.25)) + margin_y = max(3, int(bh * 0.25)) + ex0 = max(0, x0 - margin_x) + ey0 = max(0, y0 - margin_y) + ex1 = min(width, x1 + margin_x) + ey1 = min(height, y1 + margin_y) + score = float(saliency[ys, xs].mean() * np.sqrt(area)) + candidates.append({"bbox_xyxy": [ex0, ey0, ex1, ey1], "score": round(score, 6)}) + candidates.sort(key=lambda item: item["score"], reverse=True) + return non_overlapping_boxes(candidates, max_candidates=max_candidates) + + +def read_rgb_with_pillow_fallback(path: Path) -> np.ndarray: + try: + return read_rgb(path) + except Exception: + try: + from PIL import Image + except Exception: + raise + image = Image.open(path).convert("RGB") + arr = np.asarray(image, dtype=np.float32) / 255.0 + return np.clip(arr, 0.0, 1.0) + + +def connected_components(mask: np.ndarray) -> list[tuple[np.ndarray, np.ndarray]]: + visited = np.zeros(mask.shape, dtype=bool) + components: list[tuple[np.ndarray, np.ndarray]] = [] + height, width = mask.shape + for y in range(height): + xs = np.where(mask[y] & ~visited[y])[0] + for x_start in xs.tolist(): + if visited[y, x_start] or not mask[y, x_start]: + continue + stack = [(y, x_start)] + visited[y, x_start] = True + ys: list[int] = [] + xs_component: list[int] = [] + while stack: + cy, cx = stack.pop() + ys.append(cy) + xs_component.append(cx) + for ny, nx in ((cy - 1, cx), (cy + 1, cx), (cy, cx - 1), (cy, cx + 1)): + if 0 <= ny < height and 0 <= nx < width and mask[ny, nx] and not visited[ny, nx]: + visited[ny, nx] = True + stack.append((ny, nx)) + components.append((np.asarray(ys, dtype=np.int32), np.asarray(xs_component, dtype=np.int32))) + return components + + +def non_overlapping_boxes(candidates: list[dict[str, Any]], *, max_candidates: int) -> list[dict[str, Any]]: + selected: list[dict[str, Any]] = [] + for candidate in candidates: + if all(box_iou(candidate["bbox_xyxy"], other["bbox_xyxy"]) < 0.35 for other in selected): + selected.append(candidate) + if len(selected) >= max_candidates: + break + return selected + + +def box_iou(a: list[int], b: list[int]) -> float: + ax0, ay0, ax1, ay1 = a + bx0, by0, bx1, by1 = b + ix0, iy0 = max(ax0, bx0), max(ay0, by0) + ix1, iy1 = min(ax1, bx1), min(ay1, by1) + inter = max(0, ix1 - ix0) * max(0, iy1 - iy0) + area_a = max(0, ax1 - ax0) * max(0, ay1 - ay0) + area_b = max(0, bx1 - bx0) * max(0, by1 - by0) + union = area_a + area_b - inter + return inter / union if union > 0 else 0.0 + + +def default_movable_physics_material(label: str) -> dict[str, Any]: + semantic = label.lower() + if "bottle" in semantic or "cup" in semantic: + return {"movable": True, "mass": 0.08, "friction": [0.65, 0.02, 0.001], "restitution": 0.04, "contact_role": "graspable"} + if "book" in semantic or "box" in semantic: + return {"movable": True, "mass": 0.35, "friction": [0.8, 0.02, 0.001], "restitution": 0.02, "contact_role": "pushable"} + return {"movable": True, "mass": 0.2, "friction": [0.7, 0.02, 0.001], "restitution": 0.03, "contact_role": "movable_asset"} + + +def readiness_gate(gate_id: str, passed: bool, weight: float, requirement: str, evidence: Mapping[str, Any]) -> dict[str, Any]: + return { + "id": gate_id, + "passed": bool(passed), + "weight": float(weight), + "requirement": requirement, + "evidence": dict(evidence), + } + + +def commercial_next_actions(blockers: list[str]) -> list[str]: + actions = { + "texture_bake": "Improve multi-view texture selection, seam handling, and visibility filtering before final visual export.", + "movable_asset_instances": "Run or integrate instance segmentation/reconstruction so bottles, monitors, books, and tools become separate movable assets.", + "room_completeness": "Capture missing floor/back-side views or promote verified completion proxies into scene geometry, then regenerate navigation surfaces.", + "navigation_surface_hints": "Generate a real navmesh from floor/support surfaces and validate traversability.", + "physical_material_hints": "Infer or assign task-specific friction, mass, and contact roles for every structure and movable asset.", + "static_structure_layers": "Recover semantic walls/support surfaces before exporting to the simulator.", + "physics_collision": "Generate collision proxies for semantic structure before Motrix import.", + "dense_scene_geometry": "Re-run dense reconstruction or mesh cleanup until a sufficiently detailed scene mesh is available.", + "metric_scale_plausibility": "Calibrate metric scale from COLMAP poses, known robot/object dimensions, or manual room measurements before commercial simulation export.", + "navigation_acceptance": "Recover an observed floor or verified floor mesh, generate a walkable nav surface, and validate robot footprint clearance.", + "material_readiness": "Export texture-backed OBJ/MTL or MJCF materials and include every referenced texture file in the MotrixSim package.", + "task_affordance_targets": "Mark production movable assets as dynamic task targets with finite poses and actionable graspable or pushable contact roles.", + } + return [actions[item] for item in blockers if item in actions] + + +def metric_scale_diagnostics(blockers: list[str]) -> list[dict[str, str]]: + messages = { + "missing_metric_bounds": { + "problem": "scene_structure_report did not expose metric p05/p95 bounds", + "next_step": "write scene bounds during structure analysis or attach COLMAP/mesh-derived bounds before packaging", + }, + "invalid_or_extreme_bounds": { + "problem": "scene bounds contain zero, non-finite, or extreme extents", + "next_step": "verify coordinate units and remove outlier geometry before exporting MJCF", + }, + "implausible_horizontal_extent": { + "problem": "horizontal room extent is outside the expected meter-scale envelope", + "next_step": "calibrate reconstruction scale from camera poses, robot footprint, table width, or manual measurements", + }, + "implausible_vertical_extent": { + "problem": "vertical room extent is outside the expected meter-scale envelope", + "next_step": "fix up-axis selection, floor/ceiling detection, or monocular scale before physics validation", + }, + "missing_structure_proxy_sizes": { + "problem": "static structure proxies have no usable sizes", + "next_step": "regenerate semantic collision proxies from observed floor, walls, and support surfaces", + }, + "implausible_structure_proxy_size": { + "problem": "static collision proxy sizes are too small or too large for robot simulation", + "next_step": "recompute proxy boxes after metric scale calibration and outlier cleanup", + }, + } + return [messages[item] for item in blockers if item in messages] + + +def navigation_acceptance_diagnostics(blockers: list[str]) -> list[dict[str, str]]: + messages = { + "missing_walkable_floor_surface": { + "problem": "navigation layers contain no walkable floor surface", + "next_step": "capture or reconstruct the floor plane and mark it as walkable_floor before robot navigation export", + }, + "walkable_surface_too_small": { + "problem": "largest walkable surface is too small for robot route planning", + "next_step": "expand floor reconstruction, remove cropped bounds, or calibrate metric scale before navmesh generation", + }, + "walkable_surface_inferred_only": { + "problem": "all walkable surfaces are inferred completion proxies rather than directly observed geometry", + "next_step": "verify the completed floor against additional views or promote a real reconstructed floor mesh before commercial acceptance", + }, + } + return [messages[item] for item in blockers if item in messages] + + +def material_readiness_diagnostics(blockers: list[str]) -> list[dict[str, str]]: + messages = { + "scene_mesh_missing_texture_material": { + "problem": "the scene mesh is packaged but has no OBJ/MTL diffuse texture material", + "next_step": "run texture baking or preserve COLMAP/OpenMVS texture sidecars before package export", + }, + "missing_texture_backed_material": { + "problem": "no MJCF or OBJ material references a real texture image", + "next_step": "export UVs, MTL files, and map_Kd images, or add MJCF texture/material assets for simulator rendering", + }, + "missing_texture_files": { + "problem": "at least one material references a texture file that is not present in the package", + "next_step": "copy texture sidecars into meshes/ and rewrite material paths to package-relative names", + }, + "missing_mesh_assets": { + "problem": "the package contains no mesh assets whose materials can be validated", + "next_step": "export a scene mesh or production object meshes in addition to 3DGS assets", + }, + } + return [messages[item] for item in blockers if item in messages] + + +def task_affordance_diagnostics(blockers: list[str]) -> list[dict[str, str]]: + messages = { + "target_not_dynamic": { + "problem": "a task target is not marked dynamic", + "next_step": "enable dynamic body export for movable objects that should be manipulated or pushed", + }, + "target_not_production_asset": { + "problem": "a task target is not a production-grade reconstructed movable asset", + "next_step": "replace candidate/provisional targets with complete SAM3D or multi-view object reconstructions", + }, + "missing_actionable_contact_role": { + "problem": "a task target has no graspable/pushable/movable contact role", + "next_step": "assign contact roles from object semantics and intended robot task", + }, + "invalid_target_position": { + "problem": "a task target pose has missing, non-finite, or out-of-workspace position", + "next_step": "run pose alignment or scale calibration before task export", + }, + "invalid_target_scale": { + "problem": "a task target scale is missing, non-positive, non-finite, or implausibly large", + "next_step": "normalize object reconstruction scale and update pose metadata", + }, + "invalid_target_orientation": { + "problem": "a task target orientation quaternion is missing or non-finite", + "next_step": "write a valid quat_wxyz pose during object alignment", + }, + } + return [messages[item] for item in blockers if item in messages] + + +def room_completeness_diagnostics( + *, + has_observed_floor: bool, + has_floor_completion: bool, + wall_count: int, + support_count: int, + navigation_count: int, +) -> list[dict[str, Any]]: + diagnostics: list[dict[str, Any]] = [] + if has_observed_floor: + diagnostics.append( + { + "id": "floor", + "status": "observed", + "meaning": "a floor plane was detected directly from the reconstructed mesh", + "input": "scene_structure_report.semantic_planes", + "output": "walkable_floor navigation surface", + } + ) + elif has_floor_completion: + diagnostics.append( + { + "id": "floor", + "status": "inferred_completion", + "meaning": "no direct floor plane was detected, so a bounded floor proxy was inferred for simulation stability", + "input": "scene_structure_report.bounds plus observed walls/support surfaces", + "output": "floor_completion collision and walkable_floor_inferred navigation surface", + } + ) + else: + diagnostics.append( + { + "id": "floor", + "status": "missing", + "meaning": "the package has no directly observed or inferred floor, so room-level simulation is incomplete", + "input": "missing low horizontal plane or usable room bounds", + "output": "none", + } + ) + diagnostics.append( + { + "id": "walls", + "status": "sufficient" if wall_count >= 2 else "partial", + "meaning": "two or more wall planes are enough to anchor a coarse room frame; four walls are preferred for commercial scans", + "input": "scene_structure_report.semantic_planes with wall labels", + "output": f"{wall_count} wall structure surfaces", + } + ) + diagnostics.append( + { + "id": "support_surfaces", + "status": "present" if support_count else "missing", + "meaning": "support surfaces such as tables or shelves create candidate slots for movable object extraction", + "input": "support_surface semantic planes", + "output": f"{support_count} support surfaces", + } + ) + diagnostics.append( + { + "id": "navigation", + "status": "present" if navigation_count else "missing", + "meaning": "navigation surfaces are what MotrixSim and downstream planners can consume as walkable/placeable hints", + "input": "floor/support surfaces and completion proxies", + "output": f"{navigation_count} navigation surfaces", + } + ) + return diagnostics + + +def read_optional_json(path: Path) -> Any: + return read_json(path) if path.is_file() else None + + +def numeric_triplet(value: Any) -> tuple[float, float, float] | None: + if not isinstance(value, (list, tuple)) or len(value) != 3: + return None + try: + return (float(value[0]), float(value[1]), float(value[2])) + except (TypeError, ValueError): + return None + + +def vec(values: tuple[float, float, float]) -> str: + return " ".join(f"{value:.6g}" for value in values) + + +def slug_token(value: str) -> str: + return "".join(ch if ch.isalnum() or ch == "_" else "_" for ch in value).strip("_").lower() or "structure" + + +def indent_xml(element: ET.Element, level: int = 0) -> None: + indent = "\n" + level * " " + if len(element): + if not element.text or not element.text.strip(): + element.text = indent + " " + for child in element: + indent_xml(child, level + 1) + if not child.tail or not child.tail.strip(): + child.tail = indent + if level and (not element.tail or not element.tail.strip()): + element.tail = indent + + +def _manifest_from_alignment_json(path: Path, fallback: Real2SimManifest) -> Real2SimManifest: + data = read_json(path) + if not isinstance(data, Mapping): + raise StageArtifactError(f"aligned assets file must be a JSON object: {path}") + # Alignment tools can either update object poses in-place using the same + # manifest schema, or leave the file untouched. The conservative path keeps + # existing artifacts and only applies pose overrides by name. + overrides = {} + for entry in data.get("objects", []): + if isinstance(entry, Mapping) and "name" in entry and "pose" in entry: + overrides[str(entry["name"])] = entry["pose"] + if not overrides: + return fallback + + objects = [] + for asset in fallback.objects: + pose_raw = overrides.get(asset.name) + if pose_raw is None: + objects.append(asset) + continue + pose_file = path.parent / f".{asset.name}_pose_override.json" + write_json(pose_file, pose_raw) + pose = Pose.from_json(pose_file) + pose_file.unlink(missing_ok=True) + objects.append( + Real2SimAsset( + name=asset.name, + label=asset.label, + gaussian_ply=asset.gaussian_ply, + mesh_obj=asset.mesh_obj, + pose=pose, + dynamic=asset.dynamic, + source_instance_id=asset.source_instance_id, + ) + ) + return Real2SimManifest( + source=fallback.source, + prompt=fallback.prompt, + scene=fallback.scene, + objects=tuple(objects), + camera_artifacts=fallback.camera_artifacts, + metadata={**fallback.metadata, "pose_alignment": path.as_posix()}, + ) + + +def _resolve_path(base_dir: Path, value: Any) -> Path: + if value is None: + raise StageArtifactError("expected path value, got null") + path = Path(str(value)) + return path if path.is_absolute() else (base_dir / path).resolve() + + +def _supported_images(directory: Path) -> list[Path]: + if not directory.is_dir(): + return [] + return sorted(path for path in directory.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES) + + +def instances_are_refined(path: Path) -> bool: + if not path.is_file(): + return False + try: + raw = read_json(path) + except Exception: + return False + metadata = raw.get("metadata", {}) if isinstance(raw, Mapping) else {} + return isinstance(metadata, Mapping) and isinstance(metadata.get("instance_refinement"), Mapping) + + +def _safe_name(value: str) -> str: + return "".join(ch if ch.isalnum() or ch == "_" else "_" for ch in value).strip("_").lower() or "object" diff --git a/vendor/gs_playground/real2sim/pruning.py b/vendor/gs_playground/real2sim/pruning.py new file mode 100644 index 0000000..4d7968e --- /dev/null +++ b/vendor/gs_playground/real2sim/pruning.py @@ -0,0 +1,94 @@ +"""3DGS pruning helpers.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from collections.abc import Mapping +from typing import Any, Sequence, Union + +from .commands import format_command +from .errors import PipelineError + + +CommandValue = Union[str, Sequence[str]] + + +@dataclass(frozen=True) +class PruneOptions: + enabled: bool = True + keep_ratio: float = 0.3 + keep_count: int | None = None + mode: str = "opacity_area" + external_command: CommandValue | None = None + + +def prune_ply( + src: Path, + dst: Path, + options: PruneOptions, + repo_root: Path, + *, + extra_context: Mapping[str, Any] | None = None, +) -> Path: + dst.parent.mkdir(parents=True, exist_ok=True) + if not options.enabled or options.keep_ratio >= 1.0: + shutil.copy2(src, dst) + return dst + + if options.mode.replace("-", "_").lower() in {"speedy_splat", "speedysplat"}: + return prune_with_speedy_splat(src, dst, options, repo_root, extra_context=extra_context) + + script = repo_root / "benchmark" / "scripts" / "prune_gaussians.py" + if not script.is_file(): + raise PipelineError(f"pruning script not found: {script}") + + cmd = [sys.executable, script.as_posix(), "--input", src.as_posix(), "--output", dst.as_posix(), "--mode", options.mode] + if options.keep_count is not None: + cmd.extend(["--keep-count", str(options.keep_count)]) + else: + cmd.extend(["--keep-ratio", str(options.keep_ratio)]) + + completed = subprocess.run(cmd, cwd=repo_root, check=False) + if completed.returncode != 0: + raise PipelineError(f"3DGS pruning failed with exit code {completed.returncode}: {cmd}") + return dst + + +def prune_with_speedy_splat( + src: Path, + dst: Path, + options: PruneOptions, + repo_root: Path, + *, + extra_context: Mapping[str, Any] | None = None, +) -> Path: + if options.external_command is None: + raise PipelineError( + "Speedy-Splat pruning requires pruning.external_command or an equivalent " + "configured command. The built-in opacity_area pruner is not paper-equivalent." + ) + extras = dict(extra_context or {}) + context = { + "input": src, + "output": dst, + "keep_ratio": options.keep_ratio, + "keep_count": options.keep_count or "", + "mode": options.mode, + "python": sys.executable, + "source": extras.get("source", ""), + "source_image": extras.get("source_image", extras.get("image", "")), + "image": extras.get("image", extras.get("source_image", "")), + "cameras_json": extras.get("cameras_json", extras.get("cameras", "")), + "cameras": extras.get("cameras", extras.get("cameras_json", "")), + } + argv = format_command(options.external_command, context) + completed = subprocess.run(argv, cwd=repo_root, check=False) + if completed.returncode != 0: + raise PipelineError(f"Speedy-Splat pruning failed with exit code {completed.returncode}: {argv}") + if not dst.exists(): + raise PipelineError(f"Speedy-Splat command did not write output PLY: {dst}") + return dst diff --git a/vendor/gs_playground/scripts/real2sim_capture_acceptance.py b/vendor/gs_playground/scripts/real2sim_capture_acceptance.py new file mode 100644 index 0000000..682fca0 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_capture_acceptance.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Create a capture acceptance report for high-fidelity Real2Sim reconstruction.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + + +def main() -> int: + parser = argparse.ArgumentParser(description="Evaluate whether a capture is ready for high-fidelity Real2Sim reconstruction") + parser.add_argument("--sfm-report", type=Path, required=True, help="COLMAP capture report JSON") + parser.add_argument("--output-json", type=Path, required=True) + parser.add_argument("--output-md", type=Path, required=True) + parser.add_argument("--min-frames", type=int, default=30) + parser.add_argument("--min-registered-ratio", type=float, default=0.70) + parser.add_argument("--min-registered-images", type=int, default=20) + parser.add_argument("--min-sparse-points", type=int, default=5000) + parser.add_argument("--min-verified-pairs", type=int, default=40) + parser.add_argument( + "--min-camera-path-length", + type=float, + default=1.0, + help="Minimum COLMAP camera path length for space-level capture acceptance", + ) + parser.add_argument( + "--min-camera-planar-coverage-ratio", + type=float, + default=0.12, + help="Minimum second-largest camera extent / largest extent ratio; rejects nearly 1D camera tracks", + ) + parser.add_argument( + "--max-camera-step-ratio", + type=float, + default=0.55, + help="Maximum largest camera step / full path length ratio; rejects discontinuous captures", + ) + args = parser.parse_args() + + sfm = json.loads(args.sfm_report.read_text(encoding="utf-8")) + report = build_acceptance_report(sfm, args) + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_md.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + args.output_md.write_text(render_markdown(report), encoding="utf-8") + return 0 if report["accepted"] else 2 + + +def build_acceptance_report(sfm: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: + stats = sfm.get("database_stats") if isinstance(sfm.get("database_stats"), dict) else {} + camera_metrics = camera_space_metrics(sfm) + metrics = { + "frame_count": int(sfm.get("frame_count", 0) or 0), + "registered_image_count": int(sfm.get("registered_image_count", 0) or 0), + "registered_ratio": float(sfm.get("registered_ratio", 0.0) or 0.0), + "sparse_point_count": int(sfm.get("sparse_point_count", 0) or 0), + "keypoint_total": int(stats.get("keypoint_total", 0) or 0), + "keypoint_mean": float(stats.get("keypoint_mean", 0.0) or 0.0), + "matched_pair_count": int(stats.get("matched_pair_count", 0) or 0), + "verified_pair_count": int(stats.get("verified_pair_count", 0) or 0), + "verified_pair_mean": float(stats.get("verified_pair_mean", 0.0) or 0.0), + **camera_metrics, + } + gates = { + "enough_frames": gate(metrics["frame_count"] >= args.min_frames, metrics["frame_count"], args.min_frames), + "enough_registered_images": gate( + metrics["registered_image_count"] >= args.min_registered_images, + metrics["registered_image_count"], + args.min_registered_images, + ), + "registered_ratio": gate( + metrics["registered_ratio"] >= args.min_registered_ratio, + round(metrics["registered_ratio"], 4), + args.min_registered_ratio, + ), + "sparse_points": gate(metrics["sparse_point_count"] >= args.min_sparse_points, metrics["sparse_point_count"], args.min_sparse_points), + "verified_pairs": gate(metrics["verified_pair_count"] >= args.min_verified_pairs, metrics["verified_pair_count"], args.min_verified_pairs), + "camera_path_length": gate( + metrics["camera_path_length"] >= args.min_camera_path_length, + round(metrics["camera_path_length"], 4), + args.min_camera_path_length, + ), + "camera_planar_coverage": gate( + metrics["camera_planar_coverage_ratio"] >= args.min_camera_planar_coverage_ratio, + round(metrics["camera_planar_coverage_ratio"], 4), + args.min_camera_planar_coverage_ratio, + ), + "camera_step_consistency": gate( + metrics["camera_max_step_ratio"] is not None and metrics["camera_max_step_ratio"] <= args.max_camera_step_ratio, + round(metrics["camera_max_step_ratio"], 4) if metrics["camera_max_step_ratio"] is not None else None, + args.max_camera_step_ratio, + ), + } + accepted = all(item["passed"] for item in gates.values()) + blockers = [name for name, item in gates.items() if not item["passed"]] + return { + "format": "gs-playground-real2sim-capture-acceptance-v1", + "accepted": accepted, + "target_quality": "polycam_realitycapture_like_space", + "decision": "ready_for_pose_initialized_reconstruction" if accepted else "recapture_required_before_high_fidelity_reconstruction", + "metrics": metrics, + "gates": gates, + "blockers": blockers, + "sfm_warnings": sfm.get("warnings", []), + "sfm_recommendations": sfm.get("recommendations", []), + "capture_requirements": capture_requirements(blockers, metrics), + } + + +def camera_space_metrics(sfm: dict[str, Any]) -> dict[str, Any]: + path = sfm.get("camera_path") if isinstance(sfm.get("camera_path"), dict) else {} + poses = camera_positions(sfm.get("camera_poses")) + if poses: + extents = axis_extents(poses) + path_length = path_length_from_positions(poses) + step_distances = [ + distance(poses[index - 1], poses[index]) + for index in range(1, len(poses)) + ] + closure_distance = distance(poses[0], poses[-1]) if len(poses) >= 2 else 0.0 + else: + extents = numeric_list(path.get("bounds_extent"), expected=3) or [0.0, 0.0, 0.0] + path_length = float(path.get("path_length", 0.0) or 0.0) + step_distances = [] + closure_distance = None + + sorted_extents = sorted((max(float(value), 0.0) for value in extents), reverse=True) + dominant_extent = sorted_extents[0] if sorted_extents else 0.0 + secondary_extent = sorted_extents[1] if len(sorted_extents) >= 2 else 0.0 + planar_ratio = secondary_extent / dominant_extent if dominant_extent > 1e-9 else 0.0 + max_step = max(step_distances) if step_distances else None + median_step = percentile(step_distances, 0.5) if step_distances else None + max_step_ratio = max_step / path_length if max_step is not None and path_length > 1e-9 else None + closure_ratio = closure_distance / path_length if closure_distance is not None and path_length > 1e-9 else None + return { + "camera_pose_count": int(path.get("pose_count", len(poses)) or len(poses)), + "camera_path_length": round(float(path_length), 6), + "camera_bounds_extent": round_list(extents), + "camera_dominant_extent": round(float(dominant_extent), 6), + "camera_secondary_extent": round(float(secondary_extent), 6), + "camera_planar_coverage_ratio": round(float(planar_ratio), 6), + "camera_max_step_distance": round(float(max_step), 6) if max_step is not None else None, + "camera_median_step_distance": round(float(median_step), 6) if median_step is not None else None, + "camera_max_step_ratio": round(float(max_step_ratio), 6) if max_step_ratio is not None else None, + "camera_loop_closure_distance": round(float(closure_distance), 6) if closure_distance is not None else None, + "camera_loop_closure_ratio": round(float(closure_ratio), 6) if closure_ratio is not None else None, + } + + +def camera_positions(raw_poses: Any) -> list[list[float]]: + positions: list[list[float]] = [] + if not isinstance(raw_poses, list): + return positions + for pose in raw_poses: + if not isinstance(pose, dict): + continue + position = numeric_list(pose.get("position"), expected=3) + if position is not None: + positions.append(position) + return positions + + +def numeric_list(value: Any, *, expected: int) -> list[float] | None: + if not isinstance(value, list) or len(value) != expected: + return None + try: + values = [float(item) for item in value] + except (TypeError, ValueError): + return None + if not all(math.isfinite(item) for item in values): + return None + return values + + +def axis_extents(positions: list[list[float]]) -> list[float]: + return [ + max(position[axis] for position in positions) - min(position[axis] for position in positions) + for axis in range(3) + ] + + +def path_length_from_positions(positions: list[list[float]]) -> float: + return sum(distance(positions[index - 1], positions[index]) for index in range(1, len(positions))) + + +def distance(a: list[float], b: list[float]) -> float: + return math.sqrt(sum((a[index] - b[index]) ** 2 for index in range(3))) + + +def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(float(value) for value in values) + index = min(max(int(round((len(ordered) - 1) * fraction)), 0), len(ordered) - 1) + return ordered[index] + + +def round_list(values: list[float]) -> list[float]: + return [round(float(value), 6) for value in values] + + +def gate(passed: bool, value: Any, threshold: int | float) -> dict[str, Any]: + return {"passed": bool(passed), "value": value, "threshold": threshold} + + +def capture_requirements(blockers: list[str], metrics: dict[str, Any]) -> list[str]: + requirements = [ + "Capture one room-sized volume with continuous overlap; do not jump between unrelated viewpoints.", + "Move slowly with strong sideways parallax around desks, cabinets, floor, and wall features.", + "Keep 30-80 sharp frames after blur filtering; avoid fast pans, glass-only regions, and blank walls.", + ] + if "registered_ratio" in blockers or "enough_registered_images" in blockers: + requirements.append( + f"Current registration is {metrics['registered_image_count']} images / {metrics['frame_count']} frames; recapture until COLMAP registers at least 70 percent." + ) + if "sparse_points" in blockers: + requirements.append( + f"Current sparse points are {metrics['sparse_point_count']}; add textured overlap and lateral baseline until sparse points exceed 5000." + ) + if "verified_pairs" in blockers: + requirements.append( + f"Current verified pairs are {metrics['verified_pair_count']}; increase adjacent overlap and reduce motion blur." + ) + if "camera_path_length" in blockers: + requirements.append( + f"Current registered camera path length is {metrics['camera_path_length']}; walk a longer continuous path with sideways parallax around the room volume." + ) + if "camera_planar_coverage" in blockers: + requirements.append( + f"Current camera planar coverage ratio is {metrics['camera_planar_coverage_ratio']}; avoid a single straight-line sweep and capture from at least two lateral directions." + ) + if "camera_step_consistency" in blockers: + requirements.append( + f"Current camera max-step ratio is {metrics['camera_max_step_ratio']}; remove discontinuous jumps or recapture as one continuous overlapping trajectory." + ) + return requirements + + +def render_markdown(report: dict[str, Any]) -> str: + metrics = report["metrics"] + lines = [ + "# Real2Sim Capture Acceptance Report", + "", + f"Decision: `{report['decision']}`", + f"Accepted: `{str(report['accepted']).lower()}`", + "", + "## Metrics", + "", + "| Metric | Value |", + "| --- | ---: |", + ] + for key, value in metrics.items(): + lines.append(f"| `{key}` | {value} |") + lines.extend(["", "## Gates", "", "| Gate | Value | Threshold | Passed |", "| --- | ---: | ---: | --- |"]) + for name, item in report["gates"].items(): + lines.append(f"| `{name}` | {item['value']} | {item['threshold']} | `{str(item['passed']).lower()}` |") + if report["blockers"]: + lines.extend(["", "## Blockers", ""]) + for blocker in report["blockers"]: + lines.append(f"- `{blocker}`") + if report["capture_requirements"]: + lines.extend(["", "## Capture Requirements", ""]) + for requirement in report["capture_requirements"]: + lines.append(f"- {requirement}") + if report["sfm_warnings"]: + lines.extend(["", "## SfM Warnings", ""]) + for warning in report["sfm_warnings"]: + lines.append(f"- {warning}") + if report["sfm_recommendations"]: + lines.extend(["", "## SfM Recommendations", ""]) + for recommendation in report["sfm_recommendations"]: + lines.append(f"- {recommendation}") + lines.append("") + return "\n".join(lines) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_colmap_capture_report.py b/vendor/gs_playground/scripts/real2sim_colmap_capture_report.py new file mode 100644 index 0000000..d1ff712 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_colmap_capture_report.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +"""Run a COLMAP SfM preflight report for Real2Sim video keyframes.""" + +from __future__ import annotations + +import argparse +import json +import math +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path +from typing import Any + +from PIL import Image, ImageDraw + + +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"} + + +class ColmapCommandError(RuntimeError): + def __init__(self, command: list[str], returncode: int) -> None: + super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}") + self.command = command + self.returncode = returncode + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate a COLMAP capture-quality report from selected frames") + parser.add_argument("--frames-dir", type=Path, required=True, help="Directory containing selected image frames") + parser.add_argument("--work-dir", type=Path, required=True, help="COLMAP intermediate directory") + parser.add_argument("--output", type=Path, required=True, help="Output JSON report") + parser.add_argument("--colmap-bin", default="colmap") + parser.add_argument("--matcher", choices=["exhaustive", "sequential"], default="sequential") + parser.add_argument("--sequential-overlap", type=int, default=10) + parser.add_argument("--camera-model", default="SIMPLE_PINHOLE") + parser.add_argument("--max-image-size", type=int, default=0) + parser.add_argument("--sift-max-num-features", type=int, default=8192) + parser.add_argument("--sift-peak-threshold", type=float, default=None) + parser.add_argument("--feature-threads", type=int, default=1) + parser.add_argument("--matcher-threads", type=int, default=1) + parser.add_argument("--guided-matching", action="store_true") + parser.add_argument("--mapper-random-seed", type=int, default=0) + parser.add_argument("--min-registered-ratio", type=float, default=0.70) + parser.add_argument("--min-sparse-points", type=int, default=500) + parser.add_argument("--require-pass", action="store_true") + args = parser.parse_args() + + report = run_colmap_report(args) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + if args.require_pass and not report["passed"]: + return 1 + return 0 + + +def run_colmap_report(args: argparse.Namespace) -> dict[str, Any]: + frames = collect_frames(args.frames_dir) + if not frames: + raise SystemExit(f"frames directory contains no supported images: {args.frames_dir}") + colmap = resolve_colmap(args.colmap_bin) + + work = args.work_dir.resolve() + if work.exists(): + shutil.rmtree(work) + work.mkdir(parents=True, exist_ok=True) + database = work / "database.db" + sparse = work / "sparse" + sparse.mkdir(parents=True, exist_ok=True) + text_model = work / "sparse_text" + + commands: list[dict[str, Any]] = [] + warnings: list[str] = [] + try: + feature_command = [ + colmap, + "feature_extractor", + "--database_path", + database.as_posix(), + "--image_path", + args.frames_dir.resolve().as_posix(), + "--ImageReader.single_camera", + "1", + "--ImageReader.camera_model", + str(args.camera_model).upper(), + ] + feature_gpu_option = colmap_option(colmap, "feature_extractor", ["--FeatureExtraction.use_gpu", "--SiftExtraction.use_gpu"]) + if feature_gpu_option is not None: + feature_command.extend([feature_gpu_option, "0"]) + feature_threads_option = colmap_option( + colmap, + "feature_extractor", + ["--FeatureExtraction.num_threads", "--SiftExtraction.num_threads"], + ) + if feature_threads_option is not None and int(args.feature_threads) > 0: + feature_command.extend([feature_threads_option, str(int(args.feature_threads))]) + if int(args.max_image_size) > 0: + append_option_if_supported( + colmap, + "feature_extractor", + feature_command, + ["--SiftExtraction.max_image_size"], + str(int(args.max_image_size)), + ) + append_option_if_supported( + colmap, + "feature_extractor", + feature_command, + ["--SiftExtraction.max_num_features"], + str(max(int(args.sift_max_num_features), 1)), + ) + if args.sift_peak_threshold is not None: + append_option_if_supported( + colmap, + "feature_extractor", + feature_command, + ["--SiftExtraction.peak_threshold"], + str(float(args.sift_peak_threshold)), + ) + run_logged(feature_command, commands) + matcher_command = build_matcher_command( + colmap, + args.matcher, + database, + sequential_overlap=args.sequential_overlap, + matcher_threads=int(args.matcher_threads), + guided_matching=bool(args.guided_matching), + ) + run_logged(matcher_command, commands) + mapper_command = [ + colmap, + "mapper", + "--database_path", + database.as_posix(), + "--image_path", + args.frames_dir.resolve().as_posix(), + "--output_path", + sparse.as_posix(), + ] + append_option_if_supported( + colmap, + "mapper", + mapper_command, + ["--Mapper.random_seed"], + str(int(args.mapper_random_seed)), + ) + run_logged(mapper_command, commands) + except ColmapCommandError as exc: + warnings.append(str(exc)) + + model_dir, registered_images, sparse_points, model_candidates = select_best_model_dir( + colmap, + sparse, + text_model, + commands, + ) + if model_dir is None: + warnings.append("COLMAP mapper produced no sparse model") + + registered_ratio = len(registered_images) / max(float(len(frames)), 1.0) + passed = registered_ratio >= args.min_registered_ratio and sparse_points >= args.min_sparse_points + if registered_ratio < args.min_registered_ratio: + warnings.append( + f"registered image ratio {registered_ratio:.3f} < {float(args.min_registered_ratio):.3f}" + ) + if sparse_points < args.min_sparse_points: + warnings.append(f"sparse point count {sparse_points} < {int(args.min_sparse_points)}") + database_stats = read_database_stats(database) + geometry = read_best_model_geometry(text_model, work, warnings) + + return { + "format": "gs-playground-real2sim-colmap-capture-report-v1", + "frames_dir": args.frames_dir.resolve().as_posix(), + "frame_count": len(frames), + "frame_names": [path.name for path in frames], + "colmap_bin": colmap, + "matcher": args.matcher, + "camera_model": str(args.camera_model).upper(), + "max_image_size": int(args.max_image_size), + "sift_max_num_features": int(args.sift_max_num_features), + "sift_peak_threshold": float(args.sift_peak_threshold) if args.sift_peak_threshold is not None else None, + "feature_threads": int(args.feature_threads), + "matcher_threads": int(args.matcher_threads), + "model_dir": model_dir.as_posix() if model_dir is not None else None, + "model_candidates": model_candidates, + "registered_image_count": len(registered_images), + "registered_ratio": round(float(registered_ratio), 4), + "registered_images": registered_images, + "sparse_point_count": int(sparse_points), + "best_model_text_dir": text_model.as_posix() if model_dir is not None else None, + "camera_models": geometry["camera_models"], + "camera_poses": geometry["camera_poses"], + "camera_path": geometry["camera_path"], + "sparse_bounds": geometry["sparse_bounds"], + "coverage_preview": geometry["coverage_preview"], + "database_stats": database_stats, + "gates": { + "min_registered_ratio": float(args.min_registered_ratio), + "min_sparse_points": int(args.min_sparse_points), + }, + "passed": bool(passed), + "warnings": warnings, + "recommendations": capture_recommendations( + passed=bool(passed), + registered_ratio=registered_ratio, + sparse_points=sparse_points, + database_stats=database_stats, + warnings=warnings, + ), + "commands": commands, + } + + +def collect_frames(frames_dir: Path) -> list[Path]: + if not frames_dir.is_dir(): + raise SystemExit(f"frames directory not found: {frames_dir}") + return sorted(path for path in frames_dir.iterdir() if is_frame_image(path)) + + +def is_frame_image(path: Path) -> bool: + if not path.is_file() or path.suffix.lower() not in IMAGE_SUFFIXES: + return False + name = path.name.lower() + return "contact_sheet" not in name and "preview" not in name + + +def resolve_colmap(colmap_bin: str) -> str: + path = shutil.which(colmap_bin) + if path is not None: + return path + candidate = Path(colmap_bin) + if candidate.is_file(): + return candidate.resolve().as_posix() + raise SystemExit(f"COLMAP binary not found: {colmap_bin}") + + +def run_logged(command: list[str], commands: list[dict[str, Any]]) -> None: + completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + commands.append( + { + "argv": list(command), + "returncode": completed.returncode, + "output_tail": "\n".join(completed.stdout.splitlines()[-80:]), + } + ) + if completed.returncode != 0: + sys.stderr.write(completed.stdout) + raise ColmapCommandError(command, completed.returncode) + + +def build_matcher_command( + colmap: str, + matcher: str, + database: Path, + *, + sequential_overlap: int, + matcher_threads: int, + guided_matching: bool, +) -> list[str]: + command_name = "sequential_matcher" if matcher == "sequential" else "exhaustive_matcher" + command = [ + colmap, + command_name, + "--database_path", + database.as_posix(), + ] + matcher_gpu_option = colmap_option(colmap, command_name, ["--FeatureMatching.use_gpu", "--SiftMatching.use_gpu"]) + if matcher_gpu_option is not None: + command.extend([matcher_gpu_option, "0"]) + matcher_threads_option = colmap_option( + colmap, + command_name, + ["--FeatureMatching.num_threads", "--SiftMatching.num_threads"], + ) + if matcher_threads_option is not None and matcher_threads > 0: + command.extend([matcher_threads_option, str(matcher_threads)]) + if guided_matching: + append_option_if_supported( + colmap, + command_name, + command, + ["--FeatureMatching.guided_matching", "--SiftMatching.guided_matching"], + "1", + ) + append_cpu_bruteforce_option(colmap, command_name, command) + if command_name == "sequential_matcher": + overlap_option = colmap_option(colmap, command_name, ["--SequentialMatching.overlap"]) + if overlap_option is not None: + command.extend([overlap_option, str(max(int(sequential_overlap), 1))]) + return command + + +def colmap_option(colmap: str, command: str, names: list[str]) -> str | None: + completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + help_text = completed.stdout + for name in names: + if name in help_text: + return name + return None + + +def append_cpu_bruteforce_option(colmap: str, command: str, argv: list[str]) -> None: + option = colmap_option(colmap, command, ["--SiftMatching.cpu_brute_force_matcher"]) + if option is not None: + argv.extend([option, "1"]) + + +def append_option_if_supported( + colmap: str, + command: str, + argv: list[str], + option_names: list[str], + value: str, +) -> None: + option = colmap_option(colmap, command, option_names) + if option is not None: + argv.extend([option, value]) + + +def select_best_model_dir( + colmap: str, + sparse_dir: Path, + text_model: Path, + commands: list[dict[str, Any]], +) -> tuple[Path | None, list[str], int, list[dict[str, Any]]]: + candidates = sorted(path for path in sparse_dir.iterdir() if path.is_dir()) if sparse_dir.is_dir() else [] + if not candidates: + return None, [], 0, [] + + candidate_text_root = text_model.parent / f"{text_model.name}_candidates" + if candidate_text_root.exists(): + shutil.rmtree(candidate_text_root) + candidate_text_root.mkdir(parents=True, exist_ok=True) + + scored: list[tuple[int, int, Path, Path, list[str]]] = [] + summaries: list[dict[str, Any]] = [] + for candidate in candidates: + candidate_text = candidate_text_root / candidate.name + candidate_text.mkdir(parents=True, exist_ok=True) + run_logged( + [ + colmap, + "model_converter", + "--input_path", + candidate.as_posix(), + "--output_path", + candidate_text.as_posix(), + "--output_type", + "TXT", + ], + commands, + ) + registered = parse_registered_images(candidate_text / "images.txt") + points = count_sparse_points(candidate_text / "points3D.txt") + summaries.append( + { + "model_dir": candidate.as_posix(), + "registered_image_count": len(registered), + "sparse_point_count": points, + } + ) + scored.append((len(registered), points, candidate, candidate_text, registered)) + + registered_count, points, model_dir, candidate_text, registered_images = max( + scored, + key=lambda item: (item[0], item[1]), + ) + if text_model.exists(): + shutil.rmtree(text_model) + shutil.copytree(candidate_text, text_model) + return model_dir, registered_images, int(points), summaries + + +def parse_registered_images(path: Path) -> list[str]: + if not path.is_file(): + return [] + lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()] + data_lines = [line for line in lines if line and not line.startswith("#")] + registered: list[str] = [] + for index in range(0, len(data_lines), 2): + parts = data_lines[index].split(maxsplit=9) + if len(parts) >= 10: + registered.append(parts[9]) + return registered + + +def count_sparse_points(path: Path) -> int: + if not path.is_file(): + return 0 + return sum( + 1 + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() + if line.strip() and not line.startswith("#") + ) + + +def read_best_model_geometry(text_model: Path, work_dir: Path, warnings: list[str]) -> dict[str, Any]: + camera_models = parse_camera_models(text_model / "cameras.txt") + camera_poses = parse_camera_poses(text_model / "images.txt") + sparse_points, sparse_bounds = parse_sparse_points(text_model / "points3D.txt") + camera_path = summarize_camera_path(camera_poses) + coverage_preview = None + if camera_poses and sparse_points: + try: + coverage_preview = write_coverage_preview( + camera_poses, + sparse_points, + work_dir / "sfm_capture_coverage.png", + ).as_posix() + except Exception as exc: + warnings.append(f"failed to write COLMAP coverage preview: {exc}") + return { + "camera_models": camera_models, + "camera_poses": camera_poses, + "camera_path": camera_path, + "sparse_bounds": sparse_bounds, + "coverage_preview": coverage_preview, + } + + +def parse_camera_models(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + cameras: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) < 5: + continue + cameras.append( + { + "camera_id": int(parts[0]), + "model": parts[1], + "width": int(parts[2]), + "height": int(parts[3]), + "params": [float(value) for value in parts[4:]], + } + ) + return cameras + + +def parse_camera_poses(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()] + data_lines = [line for line in lines if line and not line.startswith("#")] + poses: list[dict[str, Any]] = [] + for index in range(0, len(data_lines), 2): + parts = data_lines[index].split(maxsplit=9) + if len(parts) < 10: + continue + qvec = [float(value) for value in parts[1:5]] + tvec = [float(value) for value in parts[5:8]] + position = camera_center_from_qt(qvec, tvec) + poses.append( + { + "image_id": int(parts[0]), + "image_name": parts[9], + "camera_id": int(parts[8]), + "qvec_wxyz": round_list(qvec), + "tvec": round_list(tvec), + "position": round_list(position), + } + ) + return sorted(poses, key=lambda item: item["image_name"]) + + +def camera_center_from_qt(qvec: list[float], tvec: list[float]) -> list[float]: + rotation = qvec_to_rotmat(qvec) + return [ + -sum(rotation[row][col] * tvec[row] for row in range(3)) + for col in range(3) + ] + + +def qvec_to_rotmat(qvec: list[float]) -> list[list[float]]: + w, x, y, z = qvec + return [ + [1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w], + [2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w], + [2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y], + ] + + +def parse_sparse_points(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]: + if not path.is_file(): + return [], {} + points: list[dict[str, Any]] = [] + mins = [math.inf, math.inf, math.inf] + maxs = [-math.inf, -math.inf, -math.inf] + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) < 7: + continue + xyz = [float(parts[1]), float(parts[2]), float(parts[3])] + color = [int(float(parts[4])), int(float(parts[5])), int(float(parts[6]))] + for axis in range(3): + mins[axis] = min(mins[axis], xyz[axis]) + maxs[axis] = max(maxs[axis], xyz[axis]) + if len(points) < 20000: + points.append({"xyz": xyz, "rgb": color}) + if any(not math.isfinite(value) for value in mins + maxs): + return points, {} + center = [(mins[axis] + maxs[axis]) / 2.0 for axis in range(3)] + extent = [maxs[axis] - mins[axis] for axis in range(3)] + robust_mins = [percentile([point["xyz"][axis] for point in points], 0.02) for axis in range(3)] + robust_maxs = [percentile([point["xyz"][axis] for point in points], 0.98) for axis in range(3)] + return points, { + "bounds_min": round_list(mins), + "bounds_max": round_list(maxs), + "bounds_center": round_list(center), + "bounds_extent": round_list(extent), + "robust_bounds_min": round_list(robust_mins), + "robust_bounds_max": round_list(robust_maxs), + "robust_bounds_extent": round_list([robust_maxs[axis] - robust_mins[axis] for axis in range(3)]), + "preview_point_count": len(points), + } + + +def summarize_camera_path(camera_poses: list[dict[str, Any]]) -> dict[str, Any]: + if not camera_poses: + return {} + positions = [pose["position"] for pose in camera_poses] + mins = [min(position[axis] for position in positions) for axis in range(3)] + maxs = [max(position[axis] for position in positions) for axis in range(3)] + path_length = sum( + distance(positions[index - 1], positions[index]) + for index in range(1, len(positions)) + ) + return { + "pose_count": len(camera_poses), + "path_length": round(float(path_length), 6), + "start_image": camera_poses[0]["image_name"], + "end_image": camera_poses[-1]["image_name"], + "bounds_min": round_list(mins), + "bounds_max": round_list(maxs), + "bounds_extent": round_list([maxs[axis] - mins[axis] for axis in range(3)]), + } + + +def write_coverage_preview(camera_poses: list[dict[str, Any]], sparse_points: list[dict[str, Any]], output: Path) -> Path: + all_xyz = [point["xyz"] for point in sparse_points] + [pose["position"] for pose in camera_poses] + extents = [ + max(point[axis] for point in all_xyz) - min(point[axis] for point in all_xyz) + for axis in range(3) + ] + axes = sorted(range(3), key=lambda axis: extents[axis], reverse=True)[:2] + axis_x, axis_y = axes[0], axes[1] + xs = [point[axis_x] for point in all_xyz] + ys = [point[axis_y] for point in all_xyz] + min_x, max_x = robust_axis_bounds(xs) + min_y, max_y = robust_axis_bounds(ys) + pad_x = max((max_x - min_x) * 0.08, 1e-6) + pad_y = max((max_y - min_y) * 0.08, 1e-6) + min_x -= pad_x + max_x += pad_x + min_y -= pad_y + max_y += pad_y + + width, height = 1200, 820 + margin = 70 + image = Image.new("RGB", (width, height), "white") + draw = ImageDraw.Draw(image) + draw.rectangle((margin, margin, width - margin, height - margin), outline=(210, 210, 210)) + + def project(point: list[float]) -> tuple[int, int]: + px = margin + (point[axis_x] - min_x) / max(max_x - min_x, 1e-9) * (width - 2 * margin) + py = height - margin - (point[axis_y] - min_y) / max(max_y - min_y, 1e-9) * (height - 2 * margin) + return clamp_int(px, 0, width - 1), clamp_int(py, 0, height - 1) + + for point in sparse_points: + x, y = project(point["xyz"]) + rgb = point.get("rgb", [150, 150, 150]) + color = tuple(max(45, min(210, int(value))) for value in rgb) + draw.point((x, y), fill=color) + + camera_xy = [project(pose["position"]) for pose in camera_poses] + if len(camera_xy) >= 2: + draw.line(camera_xy, fill=(28, 98, 201), width=3) + for idx, (x, y) in enumerate(camera_xy): + radius = 5 if idx not in {0, len(camera_xy) - 1} else 8 + color = (220, 48, 48) if idx == 0 else (34, 148, 83) if idx == len(camera_xy) - 1 else (28, 98, 201) + draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=color, outline=(255, 255, 255)) + + draw.text((margin, 22), "COLMAP capture coverage: sparse points + registered camera path", fill=(20, 20, 20)) + draw.text((margin, height - 44), f"Axes: {axis_name(axis_x)} / {axis_name(axis_y)} Cameras: {len(camera_poses)} Preview points: {len(sparse_points)}", fill=(70, 70, 70)) + if camera_xy: + sx, sy = camera_xy[0] + ex, ey = camera_xy[-1] + draw.text((sx + 10, sy + 10), "start", fill=(140, 30, 30)) + draw.text((ex + 10, ey + 10), "end", fill=(20, 105, 55)) + output.parent.mkdir(parents=True, exist_ok=True) + image.save(output) + return output + + +def axis_name(axis: int) -> str: + return ("x", "y", "z")[axis] + + +def robust_axis_bounds(values: list[float]) -> tuple[float, float]: + if not values: + return 0.0, 1.0 + low = percentile(values, 0.02) + high = percentile(values, 0.98) + if high <= low: + return min(values), max(values) + return low, high + + +def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(float(value) for value in values) + index = min(max(int(round((len(ordered) - 1) * fraction)), 0), len(ordered) - 1) + return ordered[index] + + +def clamp_int(value: float, low: int, high: int) -> int: + return max(low, min(high, int(round(value)))) + + +def distance(a: list[float], b: list[float]) -> float: + return math.sqrt(sum((a[axis] - b[axis]) ** 2 for axis in range(3))) + + +def round_list(values: list[float], digits: int = 6) -> list[float]: + return [round(float(value), digits) for value in values] + + +def read_database_stats(database: Path) -> dict[str, Any]: + if not database.is_file(): + return {} + try: + connection = sqlite3.connect(database.as_posix()) + try: + image_count = table_count(connection, "images") + keypoint_rows = connection.execute("SELECT rows FROM keypoints").fetchall() + match_rows = connection.execute("SELECT rows FROM matches").fetchall() + geometry_rows = connection.execute("SELECT rows FROM two_view_geometries").fetchall() + finally: + connection.close() + except sqlite3.Error: + return {} + + keypoint_counts = [int(row[0]) for row in keypoint_rows] + match_counts = [int(row[0]) for row in match_rows if int(row[0]) > 0] + geometry_counts = [int(row[0]) for row in geometry_rows if int(row[0]) > 0] + return { + "image_count": image_count, + "keypoint_total": int(sum(keypoint_counts)), + "keypoint_mean": round(float(sum(keypoint_counts) / max(len(keypoint_counts), 1)), 2), + "matched_pair_count": len(match_counts), + "matched_pair_mean": round(float(sum(match_counts) / max(len(match_counts), 1)), 2) if match_counts else 0.0, + "verified_pair_count": len(geometry_counts), + "verified_pair_mean": round(float(sum(geometry_counts) / max(len(geometry_counts), 1)), 2) if geometry_counts else 0.0, + } + + +def table_count(connection: sqlite3.Connection, table: str) -> int: + return int(connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + + +def capture_recommendations( + *, + passed: bool, + registered_ratio: float, + sparse_points: int, + database_stats: dict[str, Any], + warnings: list[str], +) -> list[str]: + if passed: + return ["COLMAP registered enough frames for a pose-initialized reconstruction path."] + recommendations = [] + warning_text = "\n".join(warnings).lower() + verified_pairs = int(database_stats.get("verified_pair_count", 0) or 0) + if "no sparse model" in warning_text or "mapper" in warning_text: + recommendations.append("Capture slower video with stronger sideways parallax; avoid pure forward motion or mostly rotational sweeps.") + if verified_pairs < 3: + recommendations.append("Increase overlap between adjacent views and keep textured surfaces in frame so COLMAP can verify image pairs.") + if registered_ratio < 0.7: + recommendations.append("Collect 30-80 sharp frames around the same room volume before attempting commercial-grade reconstruction.") + if sparse_points < 500: + recommendations.append("Avoid motion blur, reflective glass, blank walls, and very close foreground occluders during capture.") + return recommendations + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_colmap_delaunay_mesh.py b/vendor/gs_playground/scripts/real2sim_colmap_delaunay_mesh.py new file mode 100644 index 0000000..7041446 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_colmap_delaunay_mesh.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Run COLMAP Delaunay meshing from an existing dense workspace.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + + +class ColmapCommandError(RuntimeError): + def __init__(self, command: list[str], returncode: int) -> None: + super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}") + self.command = command + self.returncode = returncode + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run COLMAP Delaunay meshing from an existing dense workspace") + parser.add_argument("--dense-workspace", type=Path, required=True, help="COLMAP dense workspace containing images/stereo/sparse") + parser.add_argument("--output-ply", type=Path, required=True, help="Output Delaunay mesh PLY") + parser.add_argument("--report", type=Path, required=True, help="Output Delaunay report JSON") + parser.add_argument("--colmap-bin", default="colmap") + parser.add_argument("--fused-ply", type=Path, help="Existing fused point cloud; defaults to dense-workspace/fused.ply") + parser.add_argument("--ensure-vis", action="store_true", help="Run stereo_fusion into the workspace when fused.ply.vis is missing") + parser.add_argument("--geom-consistency", action="store_true", default=True) + parser.add_argument("--no-geom-consistency", action="store_false", dest="geom_consistency") + parser.add_argument("--stereo-fusion-min-num-pixels", type=int, default=5) + parser.add_argument("--stereo-fusion-max-reproj-error", type=float, default=2.0) + parser.add_argument("--stereo-fusion-max-depth-error", type=float, default=0.01) + parser.add_argument("--max-proj-dist", type=float, default=12.0) + parser.add_argument("--max-depth-dist", type=float, default=0.035) + parser.add_argument("--visibility-sigma", type=float, default=3.0) + parser.add_argument("--quality-regularization", type=float, default=1.0) + parser.add_argument("--max-side-length-factor", type=float, default=20.0) + parser.add_argument("--max-side-length-percentile", type=float, default=95.0) + parser.add_argument("--min-mesh-faces", type=int, default=100000) + parser.add_argument("--require-pass", action="store_true") + args = parser.parse_args() + + report = run_delaunay_meshing(args) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + if args.require_pass and not report.get("passed", False): + return 1 + return 0 + + +def run_delaunay_meshing(args: argparse.Namespace) -> dict[str, Any]: + colmap = resolve_colmap(args.colmap_bin) + dense_workspace = args.dense_workspace.resolve() + output_ply = args.output_ply.resolve() + fused_ply = (args.fused_ply or dense_workspace / "fused.ply").resolve() + workspace_fused = dense_workspace / "fused.ply" + workspace_vis = dense_workspace / "fused.ply.vis" + commands: list[dict[str, Any]] = [] + warnings: list[str] = [] + + if not dense_workspace.is_dir(): + warnings.append(f"dense workspace is missing: {dense_workspace}") + else: + output_ply.parent.mkdir(parents=True, exist_ok=True) + try: + prepare_workspace_fused( + colmap=colmap, + dense_workspace=dense_workspace, + fused_ply=fused_ply, + workspace_fused=workspace_fused, + workspace_vis=workspace_vis, + args=args, + commands=commands, + ) + run_logged(build_delaunay_command(colmap, dense_workspace, output_ply, args), commands) + except ColmapCommandError as exc: + warnings.append(str(exc)) + + mesh_counts = ply_counts(output_ply) if output_ply.is_file() else {} + mesh_face_count = int(mesh_counts.get("face_count") or 0) + passed = mesh_face_count >= int(args.min_mesh_faces) + if mesh_face_count and not passed: + warnings.append(f"Delaunay mesh face count {mesh_face_count} < {int(args.min_mesh_faces)}") + if not mesh_face_count and not any("COLMAP command failed" in warning for warning in warnings): + warnings.append("COLMAP Delaunay meshing produced no mesh") + + return { + "format": "gs-playground-real2sim-colmap-delaunay-report-v1", + "dense_workspace": dense_workspace.as_posix(), + "fused_ply": workspace_fused.as_posix() if workspace_fused.is_file() else None, + "fused_visibility": workspace_vis.as_posix() if workspace_vis.is_file() else None, + "output_ply": output_ply.as_posix() if output_ply.is_file() else None, + "mesh_vertex_count": int(mesh_counts.get("vertex_count") or 0), + "mesh_face_count": mesh_face_count, + "min_mesh_faces": int(args.min_mesh_faces), + "ensure_vis": bool(args.ensure_vis), + "geom_consistency": bool(args.geom_consistency), + "delaunay_options": { + "max_proj_dist": float(args.max_proj_dist), + "max_depth_dist": float(args.max_depth_dist), + "visibility_sigma": float(args.visibility_sigma), + "quality_regularization": float(args.quality_regularization), + "max_side_length_factor": float(args.max_side_length_factor), + "max_side_length_percentile": float(args.max_side_length_percentile), + }, + "passed": bool(passed), + "warnings": warnings, + "commands": commands, + } + + +def prepare_workspace_fused( + *, + colmap: str, + dense_workspace: Path, + fused_ply: Path, + workspace_fused: Path, + workspace_vis: Path, + args: argparse.Namespace, + commands: list[dict[str, Any]], +) -> None: + if workspace_fused.resolve() != fused_ply.resolve() and fused_ply.is_file(): + workspace_fused.unlink(missing_ok=True) + try: + workspace_fused.symlink_to(fused_ply) + except OSError: + shutil.copy2(fused_ply, workspace_fused) + if workspace_fused.is_file() and workspace_vis.is_file(): + return + if not args.ensure_vis: + raise ColmapCommandError(["missing", workspace_vis.as_posix()], 2) + workspace_fused.unlink(missing_ok=True) + run_logged(build_stereo_fusion_command(colmap, dense_workspace, workspace_fused, args), commands) + + +def build_stereo_fusion_command( + colmap: str, + dense_workspace: Path, + workspace_fused: Path, + args: argparse.Namespace, +) -> list[str]: + command = [ + colmap, + "stereo_fusion", + "--workspace_path", + dense_workspace.as_posix(), + "--workspace_format", + "COLMAP", + "--input_type", + "geometric" if args.geom_consistency else "photometric", + "--output_type", + "PLY", + "--output_path", + workspace_fused.as_posix(), + ] + append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.min_num_pixels"], str(max(int(args.stereo_fusion_min_num_pixels), 1))) + append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.max_reproj_error"], str(max(float(args.stereo_fusion_max_reproj_error), 0.1))) + append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.max_depth_error"], str(max(float(args.stereo_fusion_max_depth_error), 0.0001))) + return command + + +def build_delaunay_command(colmap: str, dense_workspace: Path, output_ply: Path, args: argparse.Namespace) -> list[str]: + command = [ + colmap, + "delaunay_mesher", + "--input_path", + dense_workspace.as_posix(), + "--input_type", + "dense", + "--output_path", + output_ply.as_posix(), + ] + append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_proj_dist"], str(max(float(args.max_proj_dist), 0.1))) + append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_depth_dist"], str(max(float(args.max_depth_dist), 0.0001))) + append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.visibility_sigma"], str(max(float(args.visibility_sigma), 0.1))) + append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.quality_regularization"], str(max(float(args.quality_regularization), 0.0))) + append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_side_length_factor"], str(max(float(args.max_side_length_factor), 1.0))) + append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_side_length_percentile"], str(min(max(float(args.max_side_length_percentile), 1.0), 100.0))) + return command + + +def run_logged(command: list[str], commands: list[dict[str, Any]], *, required: bool = True) -> None: + completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + commands.append( + { + "argv": list(command), + "returncode": completed.returncode, + "output_tail": "\n".join(completed.stdout.splitlines()[-100:]), + } + ) + if completed.returncode != 0 and required: + sys.stderr.write(completed.stdout) + raise ColmapCommandError(command, completed.returncode) + + +def resolve_colmap(colmap_bin: str) -> str: + path = shutil.which(colmap_bin) + if path is not None: + return path + candidate = Path(colmap_bin) + if candidate.is_file(): + return candidate.resolve().as_posix() + raise SystemExit(f"COLMAP binary not found: {colmap_bin}") + + +def append_option_if_supported( + colmap: str, + command: str, + argv: list[str], + option_names: list[str], + value: str, +) -> None: + option = colmap_option(colmap, command, option_names) + if option is not None: + argv.extend([option, value]) + + +def colmap_option(colmap: str, command: str, names: list[str]) -> str | None: + completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + help_text = completed.stdout + for name in names: + if name in help_text: + return name + return None + + +def ply_counts(path: Path) -> dict[str, int]: + counts = {"vertex_count": 0, "face_count": 0} + if not path.is_file(): + return counts + with path.open("rb") as handle: + for raw in handle: + line = raw.decode("ascii", errors="ignore").strip() + if line.startswith("element vertex "): + counts["vertex_count"] = int(line.split()[-1]) + elif line.startswith("element face "): + counts["face_count"] = int(line.split()[-1]) + elif line == "end_header": + break + return counts + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_colmap_dense_reconstruct.py b/vendor/gs_playground/scripts/real2sim_colmap_dense_reconstruct.py new file mode 100644 index 0000000..bfb78df --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_colmap_dense_reconstruct.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""Run COLMAP dense MVS from a passing Real2Sim SfM report.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + + +class ColmapCommandError(RuntimeError): + def __init__(self, command: list[str], returncode: int) -> None: + super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}") + self.command = command + self.returncode = returncode + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run COLMAP dense MVS from a Real2Sim SfM report") + parser.add_argument("--sfm-report", type=Path, required=True, help="SfM report produced by real2sim_colmap_capture_report.py") + parser.add_argument("--output-dir", type=Path, required=True, help="Dense artifact output directory") + parser.add_argument("--report", type=Path, required=True, help="Output dense reconstruction report JSON") + parser.add_argument("--colmap-bin", default="colmap") + parser.add_argument("--max-image-size", type=int, default=960) + parser.add_argument("--min-fused-points", type=int, default=100000) + parser.add_argument("--geom-consistency", action="store_true", default=True) + parser.add_argument("--no-geom-consistency", action="store_false", dest="geom_consistency") + parser.add_argument("--gpu-index", default="") + parser.add_argument("--patch-match-window-radius", type=int, default=5) + parser.add_argument("--patch-match-window-step", type=int, default=1) + parser.add_argument("--patch-match-num-samples", type=int, default=15) + parser.add_argument("--stereo-fusion-min-num-pixels", type=int, default=5) + parser.add_argument("--stereo-fusion-max-reproj-error", type=float, default=2.0) + parser.add_argument("--stereo-fusion-max-depth-error", type=float, default=0.01) + parser.add_argument("--run-poisson", action="store_true") + parser.add_argument("--poisson-depth", type=int, default=10) + parser.add_argument("--require-pass", action="store_true") + args = parser.parse_args() + + report = run_dense_reconstruction(args) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + if args.require_pass and not report.get("passed", False): + return 1 + return 0 + + +def run_dense_reconstruction(args: argparse.Namespace) -> dict[str, Any]: + sfm = json.loads(args.sfm_report.read_text(encoding="utf-8")) + colmap = resolve_colmap(args.colmap_bin) + output_dir = args.output_dir.resolve() + dense_workspace = output_dir / "workspace" + fused_ply = output_dir / "fused.ply" + mesh_ply = output_dir / "mesh_poisson.ply" + commands: list[dict[str, Any]] = [] + warnings: list[str] = [] + + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + frames_dir = Path(str(sfm.get("frames_dir", ""))).resolve() + model_dir = Path(str(sfm.get("model_dir", ""))).resolve() + if not sfm.get("passed", False): + warnings.append("source SfM report did not pass capture gates") + if not frames_dir.is_dir(): + warnings.append(f"SfM frames_dir is missing: {frames_dir}") + if not model_dir.is_dir(): + warnings.append(f"SfM model_dir is missing: {model_dir}") + + if not warnings: + try: + run_logged(build_image_undistorter_command(colmap, frames_dir, model_dir, dense_workspace, args), commands) + run_logged(build_patch_match_command(colmap, dense_workspace, args), commands) + run_logged(build_stereo_fusion_command(colmap, dense_workspace, fused_ply, args), commands) + if args.run_poisson and fused_ply.is_file(): + run_logged(build_poisson_command(colmap, fused_ply, mesh_ply, args), commands, required=False) + except ColmapCommandError as exc: + warnings.append(str(exc)) + + fused_counts = ply_counts(fused_ply) if fused_ply.is_file() else {} + mesh_counts = ply_counts(mesh_ply) if mesh_ply.is_file() else {} + fused_point_count = int(fused_counts.get("vertex_count") or 0) + passed = fused_point_count >= int(args.min_fused_points) + if fused_point_count and not passed: + warnings.append(f"fused point count {fused_point_count} < {int(args.min_fused_points)}") + if not fused_point_count and not any("COLMAP command failed" in warning for warning in warnings): + warnings.append("COLMAP dense reconstruction produced no fused point cloud") + + return { + "format": "gs-playground-real2sim-colmap-dense-report-v1", + "sfm_report": args.sfm_report.resolve().as_posix(), + "frames_dir": frames_dir.as_posix(), + "model_dir": model_dir.as_posix(), + "output_dir": output_dir.as_posix(), + "dense_workspace": dense_workspace.as_posix(), + "fused_ply": fused_ply.as_posix() if fused_ply.is_file() else None, + "mesh_ply": mesh_ply.as_posix() if mesh_ply.is_file() else None, + "max_image_size": int(args.max_image_size), + "geom_consistency": bool(args.geom_consistency), + "gpu_index": str(args.gpu_index), + "fused_point_count": fused_point_count, + "mesh_vertex_count": int(mesh_counts.get("vertex_count") or 0), + "mesh_face_count": int(mesh_counts.get("face_count") or 0), + "min_fused_points": int(args.min_fused_points), + "passed": bool(passed), + "warnings": warnings, + "commands": commands, + } + + +def build_image_undistorter_command( + colmap: str, + frames_dir: Path, + model_dir: Path, + dense_workspace: Path, + args: argparse.Namespace, +) -> list[str]: + command = [ + colmap, + "image_undistorter", + "--image_path", + frames_dir.as_posix(), + "--input_path", + model_dir.as_posix(), + "--output_path", + dense_workspace.as_posix(), + "--output_type", + "COLMAP", + ] + if int(args.max_image_size) > 0: + append_option_if_supported( + colmap, + "image_undistorter", + command, + ["--max_image_size", "--ImageUndistorter.max_image_size"], + str(int(args.max_image_size)), + ) + return command + + +def build_patch_match_command(colmap: str, dense_workspace: Path, args: argparse.Namespace) -> list[str]: + command = [ + colmap, + "patch_match_stereo", + "--workspace_path", + dense_workspace.as_posix(), + "--workspace_format", + "COLMAP", + ] + append_option_if_supported( + colmap, + "patch_match_stereo", + command, + ["--PatchMatchStereo.geom_consistency"], + "true" if args.geom_consistency else "false", + ) + if str(args.gpu_index).strip(): + append_option_if_supported( + colmap, + "patch_match_stereo", + command, + ["--PatchMatchStereo.gpu_index"], + str(args.gpu_index).strip(), + ) + append_option_if_supported( + colmap, + "patch_match_stereo", + command, + ["--PatchMatchStereo.window_radius"], + str(max(int(args.patch_match_window_radius), 1)), + ) + append_option_if_supported( + colmap, + "patch_match_stereo", + command, + ["--PatchMatchStereo.window_step"], + str(max(int(args.patch_match_window_step), 1)), + ) + append_option_if_supported( + colmap, + "patch_match_stereo", + command, + ["--PatchMatchStereo.num_samples"], + str(max(int(args.patch_match_num_samples), 1)), + ) + return command + + +def build_stereo_fusion_command( + colmap: str, + dense_workspace: Path, + fused_ply: Path, + args: argparse.Namespace, +) -> list[str]: + command = [ + colmap, + "stereo_fusion", + "--workspace_path", + dense_workspace.as_posix(), + "--workspace_format", + "COLMAP", + "--input_type", + "geometric" if args.geom_consistency else "photometric", + "--output_path", + fused_ply.as_posix(), + ] + append_option_if_supported( + colmap, + "stereo_fusion", + command, + ["--StereoFusion.min_num_pixels"], + str(max(int(args.stereo_fusion_min_num_pixels), 1)), + ) + append_option_if_supported( + colmap, + "stereo_fusion", + command, + ["--StereoFusion.max_reproj_error"], + str(max(float(args.stereo_fusion_max_reproj_error), 0.1)), + ) + append_option_if_supported( + colmap, + "stereo_fusion", + command, + ["--StereoFusion.max_depth_error"], + str(max(float(args.stereo_fusion_max_depth_error), 0.0001)), + ) + return command + + +def build_poisson_command(colmap: str, fused_ply: Path, mesh_ply: Path, args: argparse.Namespace) -> list[str]: + command = [ + colmap, + "poisson_mesher", + "--input_path", + fused_ply.as_posix(), + "--output_path", + mesh_ply.as_posix(), + ] + append_option_if_supported( + colmap, + "poisson_mesher", + command, + ["--PoissonMeshing.depth"], + str(max(int(args.poisson_depth), 5)), + ) + return command + + +def run_logged(command: list[str], commands: list[dict[str, Any]], *, required: bool = True) -> None: + completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + commands.append( + { + "argv": list(command), + "returncode": completed.returncode, + "output_tail": "\n".join(completed.stdout.splitlines()[-100:]), + } + ) + if completed.returncode != 0 and required: + sys.stderr.write(completed.stdout) + raise ColmapCommandError(command, completed.returncode) + + +def resolve_colmap(colmap_bin: str) -> str: + path = shutil.which(colmap_bin) + if path is not None: + return path + candidate = Path(colmap_bin) + if candidate.is_file(): + return candidate.resolve().as_posix() + raise SystemExit(f"COLMAP binary not found: {colmap_bin}") + + +def colmap_option(colmap: str, command: str, names: list[str]) -> str | None: + completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + help_text = completed.stdout + for name in names: + if name in help_text: + return name + return None + + +def append_option_if_supported( + colmap: str, + command: str, + argv: list[str], + option_names: list[str], + value: str, +) -> None: + option = colmap_option(colmap, command, option_names) + if option is not None: + argv.extend([option, value]) + + +def ply_counts(path: Path) -> dict[str, int]: + counts = {"vertex_count": 0, "face_count": 0} + if not path.is_file(): + return counts + with path.open("rb") as handle: + for raw in handle: + line = raw.decode("ascii", errors="ignore").strip() + if line.startswith("element vertex "): + counts["vertex_count"] = int(line.split()[-1]) + elif line.startswith("element face "): + counts["face_count"] = int(line.split()[-1]) + elif line == "end_header": + break + return counts + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_colmap_texture_bake_obj.py b/vendor/gs_playground/scripts/real2sim_colmap_texture_bake_obj.py new file mode 100644 index 0000000..3b7f414 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_colmap_texture_bake_obj.py @@ -0,0 +1,1198 @@ +#!/usr/bin/env python3 +"""Bake a photo-derived texture for an OBJ using COLMAP registered cameras.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from collections import OrderedDict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image, ImageDraw, ImageFilter + + +@dataclass(frozen=True) +class CameraModel: + camera_id: int + model: str + width: int + height: int + params: tuple[float, ...] + + +@dataclass(frozen=True) +class RegisteredImage: + image_id: int + qvec: tuple[float, float, float, float] + tvec: tuple[float, float, float] + camera_id: int + image_name: str + + +@dataclass(frozen=True) +class ObjFace: + vertices: tuple[int, int, int] + texcoords: tuple[int, int, int] + + +@dataclass(frozen=True) +class ObjMesh: + vertices: np.ndarray + colors: np.ndarray + texcoords: np.ndarray + faces: tuple[ObjFace, ...] + + +@dataclass(frozen=True) +class ProjectionCamera: + image_name: str + image_path: Path + width: int + height: int + rotation: np.ndarray + translation: np.ndarray + center: np.ndarray + model: CameraModel + + +@dataclass(frozen=True) +class CameraVisibilityMap: + width: int + height: int + depth: np.ndarray + + +def main() -> int: + parser = argparse.ArgumentParser(description="Bake an OBJ texture from COLMAP registered views") + parser.add_argument("--input-obj", type=Path, required=True) + parser.add_argument("--output-obj", type=Path, required=True) + parser.add_argument("--output-texture", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--sfm-report", type=Path, help="SfM report with best_model_text_dir and frames_dir") + parser.add_argument("--colmap-text", type=Path, help="COLMAP text model directory containing cameras.txt/images.txt") + parser.add_argument("--images-dir", type=Path, help="Directory containing registered image files") + parser.add_argument("--atlas-size", type=int, default=4096) + parser.add_argument("--max-cameras", type=int, default=72) + parser.add_argument("--max-faces", type=int, default=0, help="Evenly sample faces for baking; 0 means all faces") + parser.add_argument("--max-output-faces", type=int, default=0, help="Evenly sample output mesh faces before UV unwrapping; 0 means all faces") + parser.add_argument("--chunk-size", type=int, default=20000) + parser.add_argument("--texture-fill-mode", choices=["face", "warp"], default="face", help="Use flat face colors or affine-warp source image triangles into the UV atlas") + parser.add_argument("--color-blend-cameras", type=int, default=1, help="Blend flat fallback colors from this many top-scoring projected cameras") + parser.add_argument("--visibility-depth-tolerance", type=float, default=0.0, help="Reject projected faces behind the per-camera mesh z-buffer; <=0 disables") + parser.add_argument("--visibility-grid-size", type=int, default=320, help="Longest side of the approximate per-camera visibility z-buffer") + parser.add_argument( + "--max-warped-faces", + type=int, + default=0, + help="When using --texture-fill-mode warp, only affine-warp the largest UV triangles; 0 means try every projected face", + ) + parser.add_argument("--unwrap-uv", action="store_true", help="Use xatlas to create non-overlapping texture UVs before baking") + parser.add_argument("--require-unwrap-uv", action="store_true", help="Fail if --unwrap-uv is requested but xatlas is unavailable") + parser.add_argument("--postprocess-median-radius", type=int, default=0, help="Median-filter baked texels inside the UV mask") + parser.add_argument("--bleed-iterations", type=int, default=0, help="Expand baked texels outside UV islands to reduce texture seams") + parser.add_argument("--background", default="120,120,120", help="RGB background for unfilled texture pixels") + args = parser.parse_args() + + report = bake_obj_texture( + input_obj=args.input_obj, + output_obj=args.output_obj, + output_texture=args.output_texture, + report_path=args.report, + sfm_report=args.sfm_report, + colmap_text=args.colmap_text, + images_dir=args.images_dir, + atlas_size=max(int(args.atlas_size), 64), + max_cameras=max(int(args.max_cameras), 1), + max_faces=max(int(args.max_faces), 0), + max_output_faces=max(int(args.max_output_faces), 0), + chunk_size=max(int(args.chunk_size), 512), + texture_fill_mode=args.texture_fill_mode, + color_blend_cameras=max(int(args.color_blend_cameras), 1), + visibility_depth_tolerance=max(float(args.visibility_depth_tolerance), 0.0), + visibility_grid_size=max(int(args.visibility_grid_size), 32), + max_warped_faces=max(int(args.max_warped_faces), 0), + unwrap_uv=bool(args.unwrap_uv), + require_unwrap_uv=bool(args.require_unwrap_uv), + postprocess_median_radius=max(int(args.postprocess_median_radius), 0), + bleed_iterations=max(int(args.bleed_iterations), 0), + background=parse_rgb(args.background), + ) + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + + +def bake_obj_texture( + *, + input_obj: Path, + output_obj: Path, + output_texture: Path, + report_path: Path, + sfm_report: Path | None = None, + colmap_text: Path | None = None, + images_dir: Path | None = None, + atlas_size: int = 4096, + max_cameras: int = 72, + max_faces: int = 0, + max_output_faces: int = 0, + chunk_size: int = 20000, + texture_fill_mode: str = "face", + color_blend_cameras: int = 1, + visibility_depth_tolerance: float = 0.0, + visibility_grid_size: int = 320, + max_warped_faces: int = 0, + unwrap_uv: bool = False, + require_unwrap_uv: bool = False, + postprocess_median_radius: int = 0, + bleed_iterations: int = 0, + background: tuple[int, int, int] = (120, 120, 120), +) -> dict[str, Any]: + resolved_text, resolved_images = resolve_colmap_inputs( + sfm_report=sfm_report, + colmap_text=colmap_text, + images_dir=images_dir, + ) + mesh = read_obj_mesh(input_obj) + mesh, mesh_sampling_report = maybe_sample_mesh_faces(mesh, max_output_faces=max_output_faces) + mesh, unwrap_report = maybe_unwrap_mesh_uvs( + mesh, + requested=unwrap_uv, + require=require_unwrap_uv, + ) + cameras = load_projection_cameras(resolved_text, resolved_images, max_cameras=max_cameras) + selected_indices = selected_face_indices(len(mesh.faces), max_faces=max_faces) + texture_report = bake_texture_image( + mesh=mesh, + cameras=cameras, + face_indices=selected_indices, + output_texture=output_texture, + atlas_size=atlas_size, + chunk_size=chunk_size, + texture_fill_mode=texture_fill_mode, + color_blend_cameras=color_blend_cameras, + visibility_depth_tolerance=visibility_depth_tolerance, + visibility_grid_size=visibility_grid_size, + max_warped_faces=max_warped_faces, + postprocess_median_radius=postprocess_median_radius, + bleed_iterations=bleed_iterations, + background=background, + ) + output_obj.parent.mkdir(parents=True, exist_ok=True) + output_texture = output_texture.resolve() + write_textured_obj( + source_mesh=mesh, + output_obj=output_obj, + texture_name=output_texture.name, + ) + report = { + "format": "gs-playground-real2sim-colmap-texture-bake-report-v1", + "input_obj": input_obj.resolve().as_posix(), + "output_obj": output_obj.resolve().as_posix(), + "output_texture": output_texture.as_posix(), + "colmap_text": resolved_text.resolve().as_posix(), + "images_dir": resolved_images.resolve().as_posix(), + "vertex_count": int(mesh.vertices.shape[0]), + "face_count": int(len(mesh.faces)), + "texcoord_count": int(mesh.texcoords.shape[0]), + "mesh_sampling": mesh_sampling_report, + "uv_unwrap": unwrap_report, + "registered_camera_count": int(len(cameras)), + "selected_camera_count": int(len(cameras)), + "selected_face_count": int(len(selected_indices)), + **texture_report, + } + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return report + + +def resolve_colmap_inputs( + *, + sfm_report: Path | None, + colmap_text: Path | None, + images_dir: Path | None, +) -> tuple[Path, Path]: + if sfm_report is not None: + data = json.loads(sfm_report.read_text(encoding="utf-8")) + if colmap_text is None: + value = data.get("best_model_text_dir") + if value: + colmap_text = Path(str(value)) + if images_dir is None: + value = data.get("frames_dir") + if value: + images_dir = Path(str(value)) + if colmap_text is None or not colmap_text.is_dir(): + raise SystemExit(f"COLMAP text model directory not found: {colmap_text}") + if images_dir is None or not images_dir.is_dir(): + raise SystemExit(f"registered images directory not found: {images_dir}") + return colmap_text, images_dir + + +def read_obj_mesh(path: Path) -> ObjMesh: + vertices: list[tuple[float, float, float]] = [] + colors: list[tuple[float, float, float]] = [] + texcoords: list[tuple[float, float]] = [] + faces: list[ObjFace] = [] + for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + if raw.startswith("v "): + parts = raw.split() + if len(parts) < 4: + continue + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + if len(parts) >= 7: + colors.append((float(parts[4]), float(parts[5]), float(parts[6]))) + else: + colors.append((0.5, 0.5, 0.5)) + elif raw.startswith("vt "): + parts = raw.split() + if len(parts) >= 3: + texcoords.append((float(parts[1]), float(parts[2]))) + elif raw.startswith("f "): + refs = [parse_face_ref(token) for token in raw.split()[1:]] + if len(refs) < 3: + continue + for idx in range(1, len(refs) - 1): + tri = (refs[0], refs[idx], refs[idx + 1]) + vi = tuple(item[0] for item in tri) + vti = tuple(item[1] if item[1] is not None else item[0] for item in tri) + faces.append(ObjFace(vertices=vi, texcoords=vti)) + if not vertices or not faces: + raise SystemExit(f"OBJ has no usable vertices/faces: {path}") + vertex_array = np.asarray(vertices, dtype=np.float32) + color_array = np.asarray(colors, dtype=np.float32) + if texcoords: + texcoord_array = np.asarray(texcoords, dtype=np.float32) + else: + texcoord_array = planar_texcoords(vertex_array) + faces = [ + ObjFace(vertices=face.vertices, texcoords=face.vertices) + for face in faces + ] + return ObjMesh( + vertices=vertex_array, + colors=np.clip(color_array, 0.0, 1.0), + texcoords=np.clip(texcoord_array, 0.0, 1.0), + faces=tuple(faces), + ) + + +def maybe_unwrap_mesh_uvs( + mesh: ObjMesh, + *, + requested: bool, + require: bool, +) -> tuple[ObjMesh, dict[str, Any]]: + report: dict[str, Any] = { + "requested": bool(requested), + "required": bool(require), + "applied": False, + "method": "input_obj_uv", + "input_vertex_count": int(mesh.vertices.shape[0]), + "input_texcoord_count": int(mesh.texcoords.shape[0]), + "input_face_count": int(len(mesh.faces)), + } + if not requested: + return mesh, report + try: + import xatlas # type: ignore[import-not-found] + except Exception as exc: + report["error"] = f"xatlas unavailable: {exc}" + if require: + raise SystemExit(report["error"]) + return mesh, report + + faces = np.asarray([face.vertices for face in mesh.faces], dtype=np.uint32) + try: + vmapping, indices, texcoords = xatlas.parametrize( + np.asarray(mesh.vertices, dtype=np.float32), + faces, + ) + except Exception as exc: + report["error"] = f"xatlas unwrap failed: {exc}" + if require: + raise SystemExit(report["error"]) + return mesh, report + + vmapping = np.asarray(vmapping, dtype=np.int64) + indices = np.asarray(indices, dtype=np.int64) + texcoords = np.asarray(texcoords, dtype=np.float32) + if indices.ndim != 2 or indices.shape[1] != 3 or texcoords.ndim != 2 or texcoords.shape[1] != 2: + report["error"] = "xatlas returned an unexpected mesh layout" + if require: + raise SystemExit(report["error"]) + return mesh, report + + unwrapped = ObjMesh( + vertices=mesh.vertices[np.clip(vmapping, 0, mesh.vertices.shape[0] - 1)], + colors=mesh.colors[np.clip(vmapping, 0, mesh.colors.shape[0] - 1)], + texcoords=np.clip(texcoords, 0.0, 1.0), + faces=tuple( + ObjFace( + vertices=(int(face[0]), int(face[1]), int(face[2])), + texcoords=(int(face[0]), int(face[1]), int(face[2])), + ) + for face in indices + ), + ) + report.update( + { + "applied": True, + "method": "xatlas", + "output_vertex_count": int(unwrapped.vertices.shape[0]), + "output_texcoord_count": int(unwrapped.texcoords.shape[0]), + "output_face_count": int(len(unwrapped.faces)), + "duplicated_vertex_count": int(max(0, unwrapped.vertices.shape[0] - mesh.vertices.shape[0])), + } + ) + return unwrapped, report + + +def maybe_sample_mesh_faces(mesh: ObjMesh, *, max_output_faces: int) -> tuple[ObjMesh, dict[str, Any]]: + report: dict[str, Any] = { + "requested_max_output_faces": int(max_output_faces), + "applied": False, + "method": "all_faces", + "input_face_count": int(len(mesh.faces)), + "output_face_count": int(len(mesh.faces)), + } + if max_output_faces <= 0 or max_output_faces >= len(mesh.faces): + return mesh, report + indices = selected_face_indices(len(mesh.faces), max_faces=max_output_faces) + sampled_faces = tuple(mesh.faces[int(index)] for index in indices) + sampled = ObjMesh( + vertices=mesh.vertices, + colors=mesh.colors, + texcoords=mesh.texcoords, + faces=sampled_faces, + ) + report.update( + { + "applied": True, + "method": "uniform_face_sampling", + "output_face_count": int(len(sampled.faces)), + } + ) + return sampled, report + + +def parse_face_ref(token: str) -> tuple[int, int | None]: + parts = token.split("/") + vertex = parse_obj_index(parts[0]) + texcoord = parse_obj_index(parts[1]) if len(parts) >= 2 and parts[1] else None + return vertex, texcoord + + +def parse_obj_index(value: str) -> int: + index = int(value) + if index <= 0: + raise SystemExit("negative OBJ indices are not supported for texture baking") + return index - 1 + + +def planar_texcoords(vertices: np.ndarray) -> np.ndarray: + extents = vertices.max(axis=0) - vertices.min(axis=0) + axes = np.argsort(extents)[-2:] + selected = vertices[:, axes] + low = selected.min(axis=0) + span = np.maximum(selected.max(axis=0) - low, 1e-9) + return (selected - low) / span + + +def read_colmap_cameras(path: Path) -> dict[int, CameraModel]: + cameras: dict[int, CameraModel] = {} + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + camera_id = int(parts[0]) + cameras[camera_id] = CameraModel( + camera_id=camera_id, + model=parts[1].upper(), + width=int(parts[2]), + height=int(parts[3]), + params=tuple(float(value) for value in parts[4:]), + ) + return cameras + + +def read_colmap_images(path: Path) -> list[RegisteredImage]: + images: list[RegisteredImage] = [] + lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()] + idx = 0 + while idx < len(lines): + line = lines[idx] + idx += 1 + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) < 10: + continue + images.append( + RegisteredImage( + image_id=int(parts[0]), + qvec=tuple(float(value) for value in parts[1:5]), # type: ignore[arg-type] + tvec=tuple(float(value) for value in parts[5:8]), # type: ignore[arg-type] + camera_id=int(parts[8]), + image_name=" ".join(parts[9:]), + ) + ) + idx += 1 + return images + + +def load_projection_cameras(colmap_text: Path, images_dir: Path, *, max_cameras: int) -> list[ProjectionCamera]: + camera_models = read_colmap_cameras(colmap_text / "cameras.txt") + registered = read_colmap_images(colmap_text / "images.txt") + if max_cameras > 0 and len(registered) > max_cameras: + indices = np.linspace(0, len(registered) - 1, max_cameras).round().astype(int) + registered = [registered[int(index)] for index in indices] + cameras: list[ProjectionCamera] = [] + for image in registered: + model = camera_models.get(image.camera_id) + image_path = images_dir / image.image_name + if model is None or not image_path.is_file(): + continue + rotation = np.asarray(qvec_to_rotmat(image.qvec), dtype=np.float32) + translation = np.asarray(image.tvec, dtype=np.float32) + center = -rotation.T @ translation + cameras.append( + ProjectionCamera( + image_name=image.image_name, + image_path=image_path, + width=model.width, + height=model.height, + rotation=rotation, + translation=translation, + center=center, + model=model, + ) + ) + if not cameras: + raise SystemExit("no registered COLMAP image could be loaded for texture baking") + return cameras + + +def selected_face_indices(face_count: int, *, max_faces: int) -> np.ndarray: + if max_faces <= 0 or max_faces >= face_count: + return np.arange(face_count, dtype=np.int64) + return np.linspace(0, face_count - 1, max_faces).round().astype(np.int64) + + +def build_camera_visibility_maps( + *, + mesh: ObjMesh, + cameras: list[ProjectionCamera], + face_indices: np.ndarray, + grid_size: int, +) -> list[CameraVisibilityMap]: + if len(face_indices) == 0: + return [] + faces = [mesh.faces[int(index)] for index in face_indices] + vi = np.asarray([face.vertices for face in faces], dtype=np.int64) + centroids = mesh.vertices[vi].mean(axis=1) + maps: list[CameraVisibilityMap] = [] + for camera in cameras: + scale = max(float(camera.width), float(camera.height), 1.0) + width = max(int(round(camera.width / scale * grid_size)), 8) + height = max(int(round(camera.height / scale * grid_size)), 8) + depth = np.full((height, width), np.inf, dtype=np.float32) + cam_points = centroids @ camera.rotation.T + camera.translation + projected = project_camera(camera.model, cam_points) + if projected is not None: + u, v = projected + z = cam_points[:, 2] + finite = np.isfinite(z) & np.isfinite(u) & np.isfinite(v) + valid = finite & (z > 1e-6) & (u >= 0) & (u < camera.width) & (v >= 0) & (v < camera.height) + if np.any(valid): + gx = np.clip((u[valid] / max(camera.width, 1) * width).astype(np.int64), 0, width - 1) + gy = np.clip((v[valid] / max(camera.height, 1) * height).astype(np.int64), 0, height - 1) + flat = gy * width + gx + np.minimum.at(depth.reshape(-1), flat, z[valid].astype(np.float32)) + maps.append(CameraVisibilityMap(width=width, height=height, depth=depth)) + return maps + + +def camera_visibility_mask( + *, + visibility: CameraVisibilityMap | None, + camera: ProjectionCamera, + u: np.ndarray, + v: np.ndarray, + z: np.ndarray, + tolerance: float, +) -> np.ndarray: + if visibility is None or tolerance <= 0.0: + return np.ones(z.shape, dtype=bool) + gx = np.clip((u / max(camera.width, 1) * visibility.width).astype(np.int64), 0, visibility.width - 1) + gy = np.clip((v / max(camera.height, 1) * visibility.height).astype(np.int64), 0, visibility.height - 1) + nearest = visibility.depth[gy, gx].astype(np.float64) + slack = np.maximum(float(tolerance), np.abs(nearest) * float(tolerance)) + return np.isfinite(nearest) & (z.astype(np.float64) <= nearest + slack) + + +def warp_allowed_face_indices( + *, + mesh: ObjMesh, + face_indices: np.ndarray, + atlas_size: int, + max_warped_faces: int, +) -> set[int] | None: + if max_warped_faces <= 0 or max_warped_faces >= len(face_indices): + return None + areas: list[tuple[float, int]] = [] + for index in face_indices: + face = mesh.faces[int(index)] + texcoords = mesh.texcoords[np.clip(face.texcoords, 0, mesh.texcoords.shape[0] - 1)] + area = polygon_area(uv_polygon_pixels(texcoords, atlas_size)) + if area >= 0.25: + areas.append((float(area), int(index))) + if len(areas) <= max_warped_faces: + return {index for _area, index in areas} + areas.sort(reverse=True) + return {index for _area, index in areas[:max_warped_faces]} + + +def bake_texture_image( + *, + mesh: ObjMesh, + cameras: list[ProjectionCamera], + face_indices: np.ndarray, + output_texture: Path, + atlas_size: int, + chunk_size: int, + texture_fill_mode: str, + color_blend_cameras: int, + visibility_depth_tolerance: float, + visibility_grid_size: int, + max_warped_faces: int, + postprocess_median_radius: int, + bleed_iterations: int, + background: tuple[int, int, int], +) -> dict[str, Any]: + texture = Image.new("RGB", (atlas_size, atlas_size), background) + mask = Image.new("L", (atlas_size, atlas_size), 0) + draw = ImageDraw.Draw(texture) + mask_draw = ImageDraw.Draw(mask) + image_cache: OrderedDict[str, np.ndarray] = OrderedDict() + pil_image_cache: OrderedDict[str, Image.Image] = OrderedDict() + projected_faces = 0 + fallback_faces = 0 + drawn_faces = 0 + warped_faces = 0 + warp_fallback_faces = 0 + warp_skipped_faces = 0 + warp_allowed_faces = warp_allowed_face_indices( + mesh=mesh, + face_indices=face_indices, + atlas_size=atlas_size, + max_warped_faces=max_warped_faces, + ) + visibility_maps = build_camera_visibility_maps( + mesh=mesh, + cameras=cameras, + face_indices=face_indices, + grid_size=visibility_grid_size, + ) if visibility_depth_tolerance > 0.0 else None + for start in range(0, len(face_indices), chunk_size): + chunk_indices = face_indices[start : start + chunk_size] + faces = [mesh.faces[int(index)] for index in chunk_indices] + vi = np.asarray([face.vertices for face in faces], dtype=np.int64) + vti = np.asarray([face.texcoords for face in faces], dtype=np.int64) + points = mesh.vertices[vi] + colors = mesh.colors[vi].mean(axis=1) + centroids = points.mean(axis=1) + normals = face_normals(points) + best_camera, best_uv, best_score = select_best_camera( + cameras, + centroids, + normals, + visibility_maps=visibility_maps, + visibility_depth_tolerance=visibility_depth_tolerance, + ) + sampled_colors = np.clip((colors * 255.0).round(), 0, 255).astype(np.uint8) + valid = best_camera >= 0 + if np.any(valid): + projected_faces += int(np.count_nonzero(valid)) + if color_blend_cameras > 1: + sampled_colors[valid] = blend_projected_face_colors( + cameras=cameras, + centroids=centroids[valid], + normals=normals[valid], + image_cache=image_cache, + max_cameras=color_blend_cameras, + visibility_maps=visibility_maps, + visibility_depth_tolerance=visibility_depth_tolerance, + ) + else: + for camera_idx in np.unique(best_camera[valid]): + face_mask = best_camera == camera_idx + camera = cameras[int(camera_idx)] + image = cached_image(image_cache, camera.image_path) + sampled_colors[face_mask] = sample_image(image, best_uv[face_mask]) + fallback_faces += int(np.count_nonzero(~valid)) + uv = mesh.texcoords[np.clip(vti, 0, mesh.texcoords.shape[0] - 1)] + for local_idx, tri_uv in enumerate(uv): + polygon = uv_polygon_pixels(tri_uv, atlas_size) + if polygon_area(polygon) < 0.25: + continue + filled_by_warp = False + can_warp = ( + texture_fill_mode == "warp" + and best_camera[local_idx] >= 0 + and (warp_allowed_faces is None or int(chunk_indices[local_idx]) in warp_allowed_faces) + ) + if can_warp: + camera = cameras[int(best_camera[local_idx])] + src_tri = project_triangle_pixels(camera, points[local_idx]) + if src_tri is not None: + image = cached_pil_image(pil_image_cache, camera.image_path) + filled_by_warp = paste_warped_triangle( + texture=texture, + image=image, + dest_polygon=polygon, + source_triangle=src_tri, + ) + if filled_by_warp: + warped_faces += 1 + else: + if texture_fill_mode == "warp" and best_camera[local_idx] >= 0: + if warp_allowed_faces is not None and int(chunk_indices[local_idx]) not in warp_allowed_faces: + warp_skipped_faces += 1 + else: + warp_fallback_faces += 1 + color = tuple(int(value) for value in sampled_colors[local_idx]) + draw.polygon(polygon, fill=color) + mask_draw.polygon(polygon, fill=255) + drawn_faces += 1 + mask_array = np.asarray(mask, dtype=np.uint8) + coverage = float(np.count_nonzero(mask_array)) / float(mask_array.size) + selected_count = max(int(len(face_indices)), 1) + projected_ratio = float(projected_faces) / float(selected_count) + drawn_ratio = float(drawn_faces) / float(selected_count) + postprocess_report = postprocess_texture( + texture=texture, + mask=mask, + median_radius=postprocess_median_radius, + bleed_iterations=bleed_iterations, + ) + output_texture.parent.mkdir(parents=True, exist_ok=True) + texture.save(output_texture) + return { + "atlas_size": int(atlas_size), + "projected_face_count": int(projected_faces), + "fallback_face_count": int(fallback_faces), + "drawn_face_count": int(drawn_faces), + "texture_fill_mode": texture_fill_mode, + "color_blend_cameras": int(color_blend_cameras), + "visibility_depth_tolerance": float(visibility_depth_tolerance), + "visibility_grid_size": int(visibility_grid_size), + "visibility_camera_count": int(len(visibility_maps)) if visibility_maps is not None else 0, + "max_warped_faces": int(max_warped_faces), + "warp_candidate_face_count": int(len(warp_allowed_faces)) if warp_allowed_faces is not None else int(len(face_indices)), + "warped_face_count": int(warped_faces), + "warp_fallback_face_count": int(warp_fallback_faces), + "warp_skipped_face_count": int(warp_skipped_faces), + "texture_coverage_ratio": round(coverage, 6), + "atlas_pixel_coverage_ratio": round(coverage, 6), + "projected_face_ratio": round(projected_ratio, 6), + "textured_face_ratio": round(drawn_ratio, 6), + "texture_quality_breakdown": { + "mesh_face_coverage": round(drawn_ratio, 6), + "camera_projected_face_coverage": round(projected_ratio, 6), + "atlas_pixel_coverage": round(coverage, 6), + "interpretation": "atlas_pixel_coverage measures UV atlas occupancy; mesh_face_coverage measures how much reconstructed mesh received a baked texture.", + }, + "texture_postprocess": postprocess_report, + } + + +def postprocess_texture( + *, + texture: Image.Image, + mask: Image.Image, + median_radius: int, + bleed_iterations: int, +) -> dict[str, Any]: + report = { + "median_radius": int(max(median_radius, 0)), + "median_applied": False, + "bleed_iterations_requested": int(max(bleed_iterations, 0)), + "bleed_iterations_applied": 0, + "bleed_filled_pixels": 0, + } + if median_radius > 0: + size = max(3, int(median_radius) * 2 + 1) + filtered = texture.filter(ImageFilter.MedianFilter(size=size)) + texture.paste(filtered, mask=mask) + report["median_applied"] = True + report["median_filter_size"] = int(size) + if bleed_iterations > 0: + filled_texture, filled_mask, filled_pixels, applied = bleed_texture_edges( + texture, + mask, + iterations=int(bleed_iterations), + ) + texture.paste(filled_texture) + mask.paste(filled_mask) + report["bleed_iterations_applied"] = int(applied) + report["bleed_filled_pixels"] = int(filled_pixels) + return report + + +def bleed_texture_edges( + texture: Image.Image, + mask: Image.Image, + *, + iterations: int, +) -> tuple[Image.Image, Image.Image, int, int]: + colors = np.asarray(texture.convert("RGB"), dtype=np.float32).copy() + valid = np.asarray(mask, dtype=np.uint8) > 0 + total_filled = 0 + applied = 0 + shifts = ( + (-1, -1), + (-1, 0), + (-1, 1), + (0, -1), + (0, 1), + (1, -1), + (1, 0), + (1, 1), + ) + for _ in range(max(int(iterations), 0)): + accum = np.zeros_like(colors, dtype=np.float32) + counts = np.zeros(valid.shape, dtype=np.float32) + for dy, dx in shifts: + shifted_valid = shifted_bool(valid, dy=dy, dx=dx) + shifted_colors = shifted_rgb(colors, dy=dy, dx=dx) + accum += shifted_colors * shifted_valid[:, :, None] + counts += shifted_valid.astype(np.float32) + fill = (~valid) & (counts > 0) + filled = int(np.count_nonzero(fill)) + if filled <= 0: + break + colors[fill] = accum[fill] / counts[fill, None] + valid[fill] = True + total_filled += filled + applied += 1 + out_texture = Image.fromarray(np.clip(colors, 0, 255).astype(np.uint8), "RGB") + out_mask = Image.fromarray((valid.astype(np.uint8) * 255), "L") + return out_texture, out_mask, total_filled, applied + + +def shifted_bool(values: np.ndarray, *, dy: int, dx: int) -> np.ndarray: + result = np.zeros_like(values, dtype=bool) + src_y0 = max(0, -dy) + src_y1 = values.shape[0] - max(0, dy) + src_x0 = max(0, -dx) + src_x1 = values.shape[1] - max(0, dx) + dst_y0 = max(0, dy) + dst_y1 = values.shape[0] - max(0, -dy) + dst_x0 = max(0, dx) + dst_x1 = values.shape[1] - max(0, -dx) + result[dst_y0:dst_y1, dst_x0:dst_x1] = values[src_y0:src_y1, src_x0:src_x1] + return result + + +def shifted_rgb(values: np.ndarray, *, dy: int, dx: int) -> np.ndarray: + result = np.zeros_like(values, dtype=np.float32) + src_y0 = max(0, -dy) + src_y1 = values.shape[0] - max(0, dy) + src_x0 = max(0, -dx) + src_x1 = values.shape[1] - max(0, dx) + dst_y0 = max(0, dy) + dst_y1 = values.shape[0] - max(0, -dy) + dst_x0 = max(0, dx) + dst_x1 = values.shape[1] - max(0, -dx) + result[dst_y0:dst_y1, dst_x0:dst_x1] = values[src_y0:src_y1, src_x0:src_x1] + return result + + +def face_normals(points: np.ndarray) -> np.ndarray: + normals = np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]) + norms = np.linalg.norm(normals, axis=1, keepdims=True) + return normals / np.maximum(norms, 1e-9) + + +def select_best_camera( + cameras: list[ProjectionCamera], + centroids: np.ndarray, + normals: np.ndarray, + *, + visibility_maps: list[CameraVisibilityMap] | None = None, + visibility_depth_tolerance: float = 0.0, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + count = centroids.shape[0] + best_camera = np.full(count, -1, dtype=np.int32) + best_uv = np.zeros((count, 2), dtype=np.float32) + best_score = np.full(count, -np.inf, dtype=np.float32) + for camera_idx, camera in enumerate(cameras): + cam_points = centroids @ camera.rotation.T + camera.translation + projected = project_camera(camera.model, cam_points) + if projected is None: + continue + u, v = projected + z = cam_points[:, 2] + finite = np.isfinite(z) & np.isfinite(u) & np.isfinite(v) + valid = finite & (z > 1e-6) & (u >= 1) & (u < camera.width - 1) & (v >= 1) & (v < camera.height - 1) + if visibility_maps is not None and camera_idx < len(visibility_maps): + visible = camera_visibility_mask( + visibility=visibility_maps[camera_idx], + camera=camera, + u=u, + v=v, + z=z, + tolerance=visibility_depth_tolerance, + ) + valid &= visible + if not np.any(valid): + continue + valid_idx = np.nonzero(valid)[0] + rays = camera.center.astype(np.float64)[None, :] - centroids[valid_idx].astype(np.float64) + ray_norms = np.linalg.norm(rays, axis=1) + normal_score = np.abs( + np.sum(normals[valid_idx].astype(np.float64) * rays, axis=1) / np.maximum(ray_norms, 1e-9) + ) + center_score = np.full(count, -np.inf, dtype=np.float32) + du = (u[valid_idx] - (camera.width * 0.5)) / max(camera.width * 0.5, 1.0) + dv = (v[valid_idx] - (camera.height * 0.5)) / max(camera.height * 0.5, 1.0) + center_score[valid_idx] = (1.0 - np.sqrt(du * du + dv * dv)).astype(np.float32) + score = np.full(count, -np.inf, dtype=np.float32) + score[valid_idx] = (0.70 * normal_score + 0.30 * center_score[valid_idx]).astype(np.float32) + update = score > best_score + if not np.any(update): + continue + best_score[update] = score[update] + best_camera[update] = camera_idx + best_uv[update, 0] = u[update] + best_uv[update, 1] = v[update] + return best_camera, best_uv, best_score + + +def blend_projected_face_colors( + *, + cameras: list[ProjectionCamera], + centroids: np.ndarray, + normals: np.ndarray, + image_cache: OrderedDict[str, np.ndarray], + max_cameras: int, + visibility_maps: list[CameraVisibilityMap] | None = None, + visibility_depth_tolerance: float = 0.0, +) -> np.ndarray: + count = centroids.shape[0] + top_k = max(int(max_cameras), 1) + top_camera = np.full((count, top_k), -1, dtype=np.int32) + top_uv = np.zeros((count, top_k, 2), dtype=np.float32) + top_score = np.full((count, top_k), -np.inf, dtype=np.float32) + for camera_idx, camera in enumerate(cameras): + cam_points = centroids @ camera.rotation.T + camera.translation + projected = project_camera(camera.model, cam_points) + if projected is None: + continue + u, v = projected + z = cam_points[:, 2] + finite = np.isfinite(z) & np.isfinite(u) & np.isfinite(v) + valid = finite & (z > 1e-6) & (u >= 1) & (u < camera.width - 1) & (v >= 1) & (v < camera.height - 1) + if visibility_maps is not None and camera_idx < len(visibility_maps): + visible = camera_visibility_mask( + visibility=visibility_maps[camera_idx], + camera=camera, + u=u, + v=v, + z=z, + tolerance=visibility_depth_tolerance, + ) + valid &= visible + if not np.any(valid): + continue + valid_idx = np.nonzero(valid)[0] + rays = camera.center.astype(np.float64)[None, :] - centroids[valid_idx].astype(np.float64) + ray_norms = np.linalg.norm(rays, axis=1) + normal_score = np.abs( + np.sum(normals[valid_idx].astype(np.float64) * rays, axis=1) / np.maximum(ray_norms, 1e-9) + ) + du = (u[valid_idx] - (camera.width * 0.5)) / max(camera.width * 0.5, 1.0) + dv = (v[valid_idx] - (camera.height * 0.5)) / max(camera.height * 0.5, 1.0) + center_score = (1.0 - np.sqrt(du * du + dv * dv)).astype(np.float32) + score = (0.70 * normal_score + 0.30 * center_score).astype(np.float32) + for local, face_index in enumerate(valid_idx): + weakest = int(np.argmin(top_score[face_index])) + if score[local] <= top_score[face_index, weakest]: + continue + top_score[face_index, weakest] = score[local] + top_camera[face_index, weakest] = camera_idx + top_uv[face_index, weakest, 0] = u[face_index] + top_uv[face_index, weakest, 1] = v[face_index] + + accum = np.zeros((count, 3), dtype=np.float64) + weights = np.zeros((count,), dtype=np.float64) + for slot in range(top_k): + slot_valid = top_camera[:, slot] >= 0 + if not np.any(slot_valid): + continue + for camera_idx in np.unique(top_camera[slot_valid, slot]): + face_mask = slot_valid & (top_camera[:, slot] == camera_idx) + camera = cameras[int(camera_idx)] + image = cached_image(image_cache, camera.image_path) + colors = sample_image(image, top_uv[face_mask, slot, :]).astype(np.float64) + score = np.maximum(top_score[face_mask, slot].astype(np.float64), 1e-3) + accum[face_mask] += colors * score[:, None] + weights[face_mask] += score + missing = weights <= 0 + if np.any(missing): + accum[missing] = 127.0 + weights[missing] = 1.0 + return np.clip(np.rint(accum / weights[:, None]), 0, 255).astype(np.uint8) + + +def project_camera(model: CameraModel, cam_points: np.ndarray) -> tuple[np.ndarray, np.ndarray] | None: + z = cam_points[:, 2].astype(np.float64) + x = np.full(z.shape, np.nan, dtype=np.float64) + y = np.full(z.shape, np.nan, dtype=np.float64) + valid_z = np.isfinite(z) & (z > 1e-9) + x[valid_z] = cam_points[valid_z, 0].astype(np.float64) / z[valid_z] + y[valid_z] = cam_points[valid_z, 1].astype(np.float64) / z[valid_z] + params = model.params + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + if model.model == "SIMPLE_PINHOLE": + f, cx, cy = params[:3] + return f * x + cx, f * y + cy + if model.model == "PINHOLE": + fx, fy, cx, cy = params[:4] + return fx * x + cx, fy * y + cy + if model.model in {"SIMPLE_RADIAL", "RADIAL"}: + f, cx, cy = params[:3] + k1 = params[3] if len(params) > 3 else 0.0 + k2 = params[4] if len(params) > 4 else 0.0 + r2 = x * x + y * y + radial = 1.0 + k1 * r2 + k2 * r2 * r2 + return f * x * radial + cx, f * y * radial + cy + if model.model == "OPENCV": + fx, fy, cx, cy = params[:4] + k1 = params[4] if len(params) > 4 else 0.0 + k2 = params[5] if len(params) > 5 else 0.0 + p1 = params[6] if len(params) > 6 else 0.0 + p2 = params[7] if len(params) > 7 else 0.0 + r2 = x * x + y * y + radial = 1.0 + k1 * r2 + k2 * r2 * r2 + x_dist = x * radial + 2.0 * p1 * x * y + p2 * (r2 + 2.0 * x * x) + y_dist = y * radial + p1 * (r2 + 2.0 * y * y) + 2.0 * p2 * x * y + return fx * x_dist + cx, fy * y_dist + cy + return None + + +def cached_image(cache: OrderedDict[str, np.ndarray], path: Path, *, max_entries: int = 6) -> np.ndarray: + key = path.as_posix() + if key in cache: + cache.move_to_end(key) + return cache[key] + image = np.asarray(Image.open(path).convert("RGB"), dtype=np.uint8) + cache[key] = image + cache.move_to_end(key) + while len(cache) > max_entries: + cache.popitem(last=False) + return image + + +def cached_pil_image(cache: OrderedDict[str, Image.Image], path: Path, *, max_entries: int = 4) -> Image.Image: + key = path.as_posix() + if key in cache: + cache.move_to_end(key) + return cache[key] + image = Image.open(path).convert("RGB") + cache[key] = image + cache.move_to_end(key) + while len(cache) > max_entries: + _old_key, old_image = cache.popitem(last=False) + old_image.close() + return image + + +def sample_image(image: np.ndarray, uv_pixels: np.ndarray) -> np.ndarray: + h, w = image.shape[:2] + u = np.clip(np.rint(uv_pixels[:, 0]).astype(np.int64), 0, w - 1) + v = np.clip(np.rint(uv_pixels[:, 1]).astype(np.int64), 0, h - 1) + return image[v, u, :] + + +def project_triangle_pixels(camera: ProjectionCamera, points: np.ndarray) -> list[tuple[float, float]] | None: + cam_points = points @ camera.rotation.T + camera.translation + projected = project_camera(camera.model, cam_points) + if projected is None: + return None + u, v = projected + z = cam_points[:, 2] + valid = ( + np.isfinite(z) + & np.isfinite(u) + & np.isfinite(v) + & (z > 1e-6) + & (u >= 0) + & (u < camera.width) + & (v >= 0) + & (v < camera.height) + ) + if not bool(np.all(valid)): + return None + triangle = [(float(u[idx]), float(v[idx])) for idx in range(3)] + if triangle_area_float(triangle) < 0.5: + return None + return triangle + + +def paste_warped_triangle( + *, + texture: Image.Image, + image: Image.Image, + dest_polygon: list[tuple[int, int]], + source_triangle: list[tuple[float, float]], +) -> bool: + if polygon_area(dest_polygon) < 0.25 or triangle_area_float(source_triangle) < 0.5: + return False + min_x = max(min(x for x, _y in dest_polygon), 0) + max_x = min(max(x for x, _y in dest_polygon), texture.width - 1) + min_y = max(min(y for _x, y in dest_polygon), 0) + max_y = min(max(y for _x, y in dest_polygon), texture.height - 1) + if max_x <= min_x or max_y <= min_y: + return False + local_dest = [(x - min_x, y - min_y) for x, y in dest_polygon] + coeffs = affine_dest_to_source(local_dest, source_triangle) + if coeffs is None: + return False + width = max_x - min_x + 1 + height = max_y - min_y + 1 + patch = image.transform( + (width, height), + image_affine_transform_kind(), + coeffs, + resample=image_bilinear_resample_kind(), + ) + tri_mask = Image.new("L", (width, height), 0) + ImageDraw.Draw(tri_mask).polygon(local_dest, fill=255) + texture.paste(patch, (min_x, min_y), tri_mask) + return True + + +def affine_dest_to_source( + dest_triangle: list[tuple[float, float]], + source_triangle: list[tuple[float, float]], +) -> tuple[float, float, float, float, float, float] | None: + matrix = np.asarray( + [[x, y, 1.0] for x, y in dest_triangle], + dtype=np.float64, + ) + src_x = np.asarray([x for x, _y in source_triangle], dtype=np.float64) + src_y = np.asarray([y for _x, y in source_triangle], dtype=np.float64) + try: + ax, bx, cx = np.linalg.solve(matrix, src_x) + ay, by, cy = np.linalg.solve(matrix, src_y) + except np.linalg.LinAlgError: + return None + coeffs = (float(ax), float(bx), float(cx), float(ay), float(by), float(cy)) + if not all(math.isfinite(value) for value in coeffs): + return None + return coeffs + + +def image_affine_transform_kind() -> Any: + transform = getattr(Image, "Transform", None) + if transform is not None and hasattr(transform, "AFFINE"): + return transform.AFFINE + return Image.AFFINE + + +def image_bilinear_resample_kind() -> Any: + resampling = getattr(Image, "Resampling", None) + if resampling is not None and hasattr(resampling, "BILINEAR"): + return resampling.BILINEAR + return Image.BILINEAR + + +def uv_polygon_pixels(uv: np.ndarray, atlas_size: int) -> list[tuple[int, int]]: + u = np.clip(uv[:, 0], 0.0, 1.0) + v = np.clip(uv[:, 1], 0.0, 1.0) + x = np.rint(u * (atlas_size - 1)).astype(int) + y = np.rint((1.0 - v) * (atlas_size - 1)).astype(int) + return [(int(px), int(py)) for px, py in zip(x, y)] + + +def polygon_area(points: list[tuple[int, int]]) -> float: + if len(points) < 3: + return 0.0 + area = 0.0 + for idx, (x0, y0) in enumerate(points): + x1, y1 = points[(idx + 1) % len(points)] + area += x0 * y1 - x1 * y0 + return abs(area) * 0.5 + + +def triangle_area_float(points: list[tuple[float, float]]) -> float: + if len(points) < 3: + return 0.0 + (x0, y0), (x1, y1), (x2, y2) = points[:3] + return abs((x0 * (y1 - y2) + x1 * (y2 - y0) + x2 * (y0 - y1)) * 0.5) + + +def write_textured_obj(*, source_mesh: ObjMesh, output_obj: Path, texture_name: str) -> None: + material_name = safe_token(output_obj.stem) + "_baked" + material_file = output_obj.with_suffix(".mtl") + with material_file.open("w", encoding="utf-8") as dst: + dst.write("# Generated Real2Sim COLMAP texture bake material\n") + dst.write(f"newmtl {material_name}\n") + dst.write("Ka 1.000000 1.000000 1.000000\n") + dst.write("Kd 1.000000 1.000000 1.000000\n") + dst.write("Ks 0.030000 0.030000 0.030000\n") + dst.write("Ns 8.000000\n") + dst.write("d 1.000000\n") + dst.write("illum 2\n") + dst.write(f"map_Kd {texture_name}\n") + with output_obj.open("w", encoding="utf-8") as dst: + dst.write("# Real2Sim COLMAP photo-textured scene mesh\n") + dst.write(f"mtllib {material_file.name}\n") + dst.write("o real2sim_colmap_textured_scene\n") + for vertex, color in zip(source_mesh.vertices, source_mesh.colors): + dst.write( + f"v {vertex[0]:.9g} {vertex[1]:.9g} {vertex[2]:.9g} " + f"{color[0]:.6f} {color[1]:.6f} {color[2]:.6f}\n" + ) + for texcoord in source_mesh.texcoords: + dst.write(f"vt {texcoord[0]:.7g} {texcoord[1]:.7g}\n") + dst.write(f"usemtl {material_name}\n") + for face in source_mesh.faces: + dst.write( + "f " + f"{face.vertices[0] + 1}/{face.texcoords[0] + 1} " + f"{face.vertices[1] + 1}/{face.texcoords[1] + 1} " + f"{face.vertices[2] + 1}/{face.texcoords[2] + 1}\n" + ) + + +def qvec_to_rotmat(qvec: tuple[float, float, float, float]) -> list[list[float]]: + w, x, y, z = qvec + return [ + [1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * w * z, 2 * z * x + 2 * w * y], + [2 * x * y + 2 * w * z, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * w * x], + [2 * z * x - 2 * w * y, 2 * y * z + 2 * w * x, 1 - 2 * x * x - 2 * y * y], + ] + + +def parse_rgb(value: str) -> tuple[int, int, int]: + parts = [int(part.strip()) for part in value.split(",")] + if len(parts) != 3: + raise SystemExit("--background must be R,G,B") + return tuple(max(0, min(255, part)) for part in parts) # type: ignore[return-value] + + +def safe_token(value: str) -> str: + token = re.sub(r"[^A-Za-z0-9_]+", "_", value.strip()) + return token.strip("_") or "scene" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_download_sam3d_checkpoints.sh b/vendor/gs_playground/scripts/real2sim_download_sam3d_checkpoints.sh new file mode 100755 index 0000000..c1376a8 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_download_sam3d_checkpoints.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Download SAM-3D Objects checkpoints into the layout expected by Real2Sim. +# +# Official paper-equivalent source: +# REAL2SIM_SAM3D_CHECKPOINT_REPO=facebook/sam-3d-objects +# +# Public mirror fallback used only when the official gated repo is unavailable: +# REAL2SIM_SAM3D_CHECKPOINT_REPO=licesma/sam-3d-objects-weights + +set -euo pipefail + +if [[ -z "${REAL2SIM_SAM3D_ROOT:-}" ]]; then + echo "REAL2SIM_SAM3D_ROOT is required. Source scripts/real2sim_wuying_paper_env.sh first." >&2 + exit 2 +fi + +repo="${REAL2SIM_SAM3D_CHECKPOINT_REPO:-facebook/sam-3d-objects}" +max_workers="${REAL2SIM_SAM3D_DOWNLOAD_WORKERS:-1}" +dest="${REAL2SIM_SAM3D_CHECKPOINT_DIR:-${REAL2SIM_SAM3D_ROOT}/checkpoints/hf}" +hf_bin="${REAL2SIM_HF_BIN:-hf}" +force="${REAL2SIM_SAM3D_CHECKPOINT_FORCE:-0}" + +if [[ -e "${dest}/pipeline.yaml" && "${force}" != "1" ]]; then + echo "SAM-3D checkpoints already exist at ${dest}. Set REAL2SIM_SAM3D_CHECKPOINT_FORCE=1 to replace." >&2 + exit 0 +fi + +mkdir -p "${REAL2SIM_SAM3D_ROOT}/checkpoints" +download_dir="$(mktemp -d "${REAL2SIM_SAM3D_ROOT}/checkpoints/.sam3d-download.XXXXXX")" +cleanup() { + rm -rf "${download_dir}" +} +trap cleanup EXIT + +"${hf_bin}" download \ + "${repo}" \ + --repo-type model \ + --local-dir "${download_dir}" \ + --max-workers "${max_workers}" + +source_dir="" +if [[ -f "${download_dir}/checkpoints/pipeline.yaml" ]]; then + source_dir="${download_dir}/checkpoints" +elif [[ -f "${download_dir}/hf/pipeline.yaml" ]]; then + source_dir="${download_dir}/hf" +else + echo "Downloaded ${repo}, but no checkpoints/pipeline.yaml or hf/pipeline.yaml was found." >&2 + find "${download_dir}" -maxdepth 3 -type f | sort >&2 + exit 1 +fi + +tmp_dest="${dest}.tmp.$$" +rm -rf "${tmp_dest}" +mkdir -p "$(dirname "${dest}")" +cp -a "${source_dir}" "${tmp_dest}" +if [[ -e "${dest}" || -L "${dest}" ]]; then + rm -rf "${dest}" +fi +mv "${tmp_dest}" "${dest}" + +cat > "${dest}.SOURCE.txt" < int: + parser = argparse.ArgumentParser(description="Export registered COLMAP frames as a pose-init 3DGS training dataset") + parser.add_argument("--sfm-report", type=Path, required=True, help="COLMAP capture report JSON") + parser.add_argument("--output-dir", type=Path, required=True, help="Output pose-init dataset directory") + parser.add_argument("--allow-failed-sfm", action="store_true", help="Export even when sfm_report.passed is false") + args = parser.parse_args() + + summary = export_pose_init_dataset( + sfm_report=args.sfm_report, + output_dir=args.output_dir, + allow_failed_sfm=bool(args.allow_failed_sfm), + ) + print(json.dumps(summary, indent=2, ensure_ascii=False)) + return 0 + + +def export_pose_init_dataset(*, sfm_report: Path, output_dir: Path, allow_failed_sfm: bool = False) -> dict[str, Any]: + report = read_json(sfm_report) + if not report.get("passed") and not allow_failed_sfm: + raise SystemExit("SFM report did not pass; rerun with --allow-failed-sfm to export anyway") + + frames_dir = Path(str(report.get("frames_dir", ""))) + if not frames_dir.is_dir(): + raise SystemExit(f"SFM report frames_dir does not exist: {frames_dir}") + text_model = Path(str(report.get("best_model_text_dir", ""))) + if not text_model.is_dir(): + raise SystemExit(f"SFM report best_model_text_dir does not exist: {text_model}") + + camera_models = {int(item["camera_id"]): item for item in report.get("camera_models", [])} + camera_poses = list(report.get("camera_poses", [])) + if not camera_models: + raise SystemExit("SFM report contains no camera models") + if not camera_poses: + raise SystemExit("SFM report contains no registered camera poses") + + if output_dir.exists(): + shutil.rmtree(output_dir) + image_dir = output_dir / "images" + colmap_text_dir = output_dir / "colmap_text" + image_dir.mkdir(parents=True, exist_ok=True) + colmap_text_dir.mkdir(parents=True, exist_ok=True) + + frames: list[dict[str, Any]] = [] + copied_images = 0 + for pose in sorted(camera_poses, key=lambda item: str(item.get("image_name", ""))): + image_name = str(pose.get("image_name", "")) + source_image = frames_dir / image_name + if not source_image.is_file(): + raise SystemExit(f"registered COLMAP frame is missing: {source_image}") + dest_image = image_dir / source_image.name + shutil.copy2(source_image, dest_image) + copied_images += 1 + + camera_id = int(pose["camera_id"]) + camera = camera_models.get(camera_id) + if camera is None: + raise SystemExit(f"pose references unknown camera_id {camera_id}: {image_name}") + intrinsics = camera_intrinsics(camera) + colmap_c2w = colmap_camera_to_world(pose["qvec_wxyz"], pose["tvec"]) + nerfstudio_c2w = colmap_c2w_to_nerfstudio(colmap_c2w) + frames.append( + { + "file_path": f"images/{dest_image.name}", + "transform_matrix": round_matrix(nerfstudio_c2w), + "colmap_transform_matrix": round_matrix(colmap_c2w), + "fl_x": intrinsics["fl_x"], + "fl_y": intrinsics["fl_y"], + "cx": intrinsics["cx"], + "cy": intrinsics["cy"], + "w": intrinsics["w"], + "h": intrinsics["h"], + "camera_model": intrinsics["camera_model"], + "real2sim_pose": { + "image_id": pose.get("image_id"), + "camera_id": camera_id, + "qvec_wxyz": pose.get("qvec_wxyz"), + "tvec": pose.get("tvec"), + "position_colmap_world": pose.get("position"), + }, + } + ) + + sparse_ply = write_sparse_points_ply(text_model / "points3D.txt", output_dir / "colmap_sparse_points.ply") + copied_colmap_files = copy_colmap_text_model(text_model, colmap_text_dir) + transforms = { + "format": "gs-playground-real2sim-pose-init-transforms-v1", + "camera_convention": "nerfstudio/OpenGL camera-to-world; raw COLMAP camera-to-world is preserved per frame", + "source_sfm_report": sfm_report.resolve().as_posix(), + "frame_count": len(frames), + "registered_image_count": int(report.get("registered_image_count", len(frames))), + "sparse_point_count": int(report.get("sparse_point_count", 0) or 0), + "camera_path": report.get("camera_path", {}), + "sparse_bounds": report.get("sparse_bounds", {}), + "database_stats": report.get("database_stats", {}), + "frames": frames, + } + (output_dir / "transforms.json").write_text(json.dumps(transforms, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + write_readme(output_dir, transforms, sparse_ply=sparse_ply) + + summary = { + "format": "gs-playground-real2sim-pose-init-dataset-v1", + "root": output_dir.resolve().as_posix(), + "transforms": "transforms.json", + "images": "images", + "image_count": copied_images, + "colmap_text": "colmap_text", + "colmap_text_files": copied_colmap_files, + "sparse_points_ply": sparse_ply.name if sparse_ply is not None else None, + "sparse_point_count": int(report.get("sparse_point_count", 0) or 0), + "registered_ratio": report.get("registered_ratio"), + "ready_for_training": bool(report.get("passed") and copied_images > 0 and sparse_ply is not None), + } + (output_dir / "pose_init_dataset.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return summary + + +def read_json(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise SystemExit(f"expected JSON object: {path}") + return payload + + +def camera_intrinsics(camera: dict[str, Any]) -> dict[str, Any]: + model = str(camera.get("model", "")).upper() + width = int(camera["width"]) + height = int(camera["height"]) + params = [float(value) for value in camera.get("params", [])] + if model == "SIMPLE_PINHOLE": + f, cx, cy = params[:3] + fx = fy = f + elif model == "PINHOLE": + fx, fy, cx, cy = params[:4] + elif model in {"SIMPLE_RADIAL", "RADIAL"}: + f, cx, cy = params[:3] + fx = fy = f + elif model in {"OPENCV", "OPENCV_FISHEYE", "FULL_OPENCV"}: + fx, fy, cx, cy = params[:4] + else: + raise SystemExit(f"unsupported COLMAP camera model for pose-init export: {model}") + return { + "camera_model": model, + "w": width, + "h": height, + "fl_x": round(float(fx), 6), + "fl_y": round(float(fy), 6), + "cx": round(float(cx), 6), + "cy": round(float(cy), 6), + } + + +def colmap_camera_to_world(qvec: list[float], tvec: list[float]) -> list[list[float]]: + rotation = qvec_to_rotmat([float(value) for value in qvec]) + t = [float(value) for value in tvec] + rt = transpose(rotation) + center = [-sum(rt[row][col] * t[col] for col in range(3)) for row in range(3)] + return [ + [rt[0][0], rt[0][1], rt[0][2], center[0]], + [rt[1][0], rt[1][1], rt[1][2], center[1]], + [rt[2][0], rt[2][1], rt[2][2], center[2]], + [0.0, 0.0, 0.0, 1.0], + ] + + +def colmap_c2w_to_nerfstudio(matrix: list[list[float]]) -> list[list[float]]: + converted = [list(row) for row in matrix] + for row in range(3): + converted[row][1] *= -1.0 + converted[row][2] *= -1.0 + return converted + + +def qvec_to_rotmat(qvec: list[float]) -> list[list[float]]: + w, x, y, z = qvec + return [ + [1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w], + [2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w], + [2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y], + ] + + +def transpose(matrix: list[list[float]]) -> list[list[float]]: + return [[matrix[col][row] for col in range(3)] for row in range(3)] + + +def round_matrix(matrix: list[list[float]], digits: int = 8) -> list[list[float]]: + return [[round(float(value), digits) for value in row] for row in matrix] + + +def write_sparse_points_ply(points_txt: Path, output: Path) -> Path | None: + points: list[tuple[float, float, float, int, int, int]] = [] + if not points_txt.is_file(): + return None + for line in points_txt.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) < 7: + continue + x, y, z = (float(parts[index]) for index in range(1, 4)) + r, g, b = (clamp_color(float(parts[index])) for index in range(4, 7)) + points.append((x, y, z, r, g, b)) + if not points: + return None + with output.open("w", encoding="ascii") as handle: + handle.write("ply\n") + handle.write("format ascii 1.0\n") + handle.write(f"element vertex {len(points)}\n") + handle.write("property float x\n") + handle.write("property float y\n") + handle.write("property float z\n") + handle.write("property uchar red\n") + handle.write("property uchar green\n") + handle.write("property uchar blue\n") + handle.write("end_header\n") + for x, y, z, r, g, b in points: + handle.write(f"{x:.8f} {y:.8f} {z:.8f} {r} {g} {b}\n") + return output + + +def clamp_color(value: float) -> int: + return max(0, min(255, int(round(value)))) + + +def copy_colmap_text_model(text_model: Path, output: Path) -> list[str]: + copied: list[str] = [] + for name in COLMAP_TEXT_FILES: + source = text_model / name + if source.is_file(): + shutil.copy2(source, output / name) + copied.append(name) + return copied + + +def write_readme(output_dir: Path, transforms: dict[str, Any], *, sparse_ply: Path | None) -> None: + lines = [ + "# Real2Sim Pose-Init Dataset", + "", + "This directory is generated from a COLMAP capture report that passed the Real2Sim capture gate.", + "It is intended as the bridge from fast AnySplat preview reconstruction to a pose-initialized 3DGS training path.", + "", + "Contents:", + "", + "- `images/`: registered video frames copied from the SfM frame set.", + "- `transforms.json`: Nerfstudio/OpenGL-style camera-to-world transforms plus raw COLMAP pose metadata.", + "- `colmap_text/`: the selected COLMAP text model (`cameras.txt`, `images.txt`, `points3D.txt`).", + "- `colmap_sparse_points.ply`: colored sparse point cloud exported from `points3D.txt`.", + "", + "Summary:", + "", + f"- frames: {transforms.get('frame_count')}", + f"- registered images: {transforms.get('registered_image_count')}", + f"- sparse points: {transforms.get('sparse_point_count')}", + f"- sparse PLY: {sparse_ply.name if sparse_ply is not None else 'missing'}", + "", + "A downstream trainer should initialize from these poses and sparse points instead of random camera/point initialization.", + "", + ] + (output_dir / "README.md").write_text("\n".join(lines), encoding="utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/README.md b/vendor/gs_playground/scripts/real2sim_external/README.md new file mode 100644 index 0000000..2e32387 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/README.md @@ -0,0 +1,115 @@ +# Real2Sim External Model Adapters + +These scripts connect the GS-Playground Real2Sim stage contract to public model +implementations. They are optional: the built-in backend still runs without any +of these dependencies. + +## Quality Upgrade Path + +1. `groundingdino_hf_detect.py` + - Purpose: replace heuristic connected components with open-vocabulary boxes. + - Source: official GroundingDINO / Grounded-SAM-2 Hugging Face path. + - Main dependency: `transformers`, `Pillow`, `torch`. + - Default model: `IDEA-Research/grounding-dino-tiny`. + +2. `sam2_segment.py` + - Purpose: replace coarse masks with SAM2 box-prompted masks. + - Source: official `facebookresearch/sam2`. + - Required env: + - `REAL2SIM_SAM2_CHECKPOINT=/path/to/sam2.1_hiera_large.pt` + - `REAL2SIM_SAM2_MODEL_CFG=configs/sam2.1/sam2.1_hiera_l.yaml` + +3. `instance_refine.py` + - Purpose: match the paper's visual-only instance de-duplication stage after + SAM/SAM2. + - Method: composite detection/segmentation confidence, mask IoU, mask + inclusion, and boundary overlap. + +4. `lama_inpaint.py` + - Purpose: replace median-fill background removal with sequential LaMa + inpainting. + - Runtime used here: `simple-lama-inpainting`, a lightweight LaMa wrapper + listed by the official LaMa repository. + - Method: one-object-at-a-time mask expansion and sequential background + update. Re-detection of newly exposed regions must be supplied by the + configured detection/segmentation loop for fully strict paper parity. + The strict config uses `REAL2SIM_GROUNDING_DINO_COMMAND` and + `REAL2SIM_SAM2_COMMAND` for this nested loop. + +5. `depth_anything_depth.py` + - Purpose: replace flat pseudo-depth detail with a monocular Depth Anything + V2 depth prior before building the fallback 3DGS assets. + - Main dependency: `transformers`, `Pillow`, `torch`. + - Default model: `depth-anything/Depth-Anything-V2-Small-hf`. + +6. `anysplat_scene.py` + - Purpose: replace pseudo-depth scene Gaussians with AnySplat scene Gaussians. + - Source: official AnySplat repo and Hugging Face model `lhjiang/anysplat`. + - Output: official `export_ply` scene Gaussians, decoder depth, camera + metadata, and a depth-derived scene mesh. + - Required env: + - `REAL2SIM_ANYSPLAT_ROOT=/path/to/AnySplat` + - Best input: multiple nearby views or a short video, not a single repeated + frame. + +7. `sam3d_objects_reconstruct.py` + - Purpose: replace object box meshes and object pseudo-Gaussians with SAM 3D + Objects outputs. + - Source: official Meta `facebookresearch/sam-3d-objects`. + - Output: object Gaussian PLY from the SAM-3D `save_ply` output, an + exportable object mesh, and pose/scale metadata. The script fails if no + mesh is returned unless `--allow-fallback-mesh` is passed for debugging. + - Required env: + - `REAL2SIM_SAM3D_ROOT=/path/to/sam-3d-objects` + - `REAL2SIM_SAM3D_CONFIG=/path/to/checkpoints/hf/pipeline.yaml` + - Checkpoints are gated on Hugging Face and require accepted access. The + upstream setup currently recommends an NVIDIA GPU with at least 32 GB VRAM. + - Use `--cameras {cameras_json} --match-camera-preprocess` when the scene + was reconstructed by AnySplat, so SAM-3D receives the same + aspect-preserving center crop and mask geometry as the scene camera. + - Use `--canonicalize-gaussian-to-mesh` to rotate the object Gaussian PLY + into the SAM-3D mesh canonical frame using only geometric extent axes. It + updates positions, normals, and splat quaternions without source-image + photometric scoring. + - Use `--remove-video-overlay` only for screenshots/video thumbnails with a + central play button; it inpaints detected bright, low-saturation UI pixels + near the image center before SAM-3D inference. + +8. `depth_mask_pose_align.py` + - Purpose: align SAM-3D objects back into the AnySplat scene using rendered + object Gaussian PLY depth/mask, scene depth, and camera intrinsics. + Mesh rendering is used only as a fallback when the object Gaussian asset is + unavailable or invalid. + - Output: an updated `{aligned_assets_json}` manifest with refined + `pose.position`, `pose.quat_wxyz`, and `pose.scale` values plus alignment + metadata including the selected `render_source`, candidate axis search + label, mask IoU, centroid error, depth error, and quality summary. + If the object Gaussian directory contains SAM-3D metadata showing + `gaussian_canonicalization.status=ok`, the aligner keeps the canonical + rotation and uses only a mask/depth position-scale seed instead of + source-view axis search. + +9. Speedy-Splat pruning + - Purpose: match the paper's 3DGS pruning stage. + - Configure `pruning.mode=speedy_splat` and set + `pruning.external_command` or `REAL2SIM_SPEEDY_SPLAT_COMMAND`. + - The pipeline fails if the command is missing in Speedy-Splat mode. + - The adapter keeps Speedy-Splat's official score computation, but applies + an exact-count tie-safe prune mask so all-zero or heavily tied object + scores cannot delete every Gaussian in a canonical local object asset. + +## Example + +```bash +python -m real2sim run \ + --backend hybrid \ + --config configs/real2sim.external.example.json \ + --input media/teaser.png \ + --output /tmp/gs_real2sim_hybrid \ + --prompt "robot. person. table. chair. object." +``` + +Use `--backend external` only after AnySplat and SAM 3D Objects are installed, +their checkpoints are available, and the machine meets the SAM-3D memory +requirements. Use `python -m real2sim run ... --dry-run --backend hybrid` to +check which stage commands are configured before launching a long model run. diff --git a/vendor/gs_playground/scripts/real2sim_external/anysplat_scene.py b/vendor/gs_playground/scripts/real2sim_external/anysplat_scene.py new file mode 100644 index 0000000..0a1e263 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/anysplat_scene.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""AnySplat scene reconstruction adapter for Real2Sim. + +This adapter uses the official AnySplat model/API and exports a real 3DGS PLY +through AnySplat's `export_ply` helper. Video inputs can provide real nearby +frames through `--frames-dir`; single-image inputs are kept as a single view +instead of pretending that duplicated frames are independent observations. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.contracts import write_json +from real2sim.image_io import write_gray + + +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"} + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run AnySplat scene reconstruction for Real2Sim") + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--ply", type=Path, required=True) + parser.add_argument("--mesh", type=Path, required=True) + parser.add_argument("--cameras", type=Path, required=True) + parser.add_argument("--depth", type=Path, required=True) + parser.add_argument("--source-preview", type=Path, default=None, help="Optional source-camera RGB render preview PNG") + parser.add_argument("--frames-dir", type=Path, default=None, help="Optional extracted video frames directory") + parser.add_argument("--anysplat-root", type=Path, default=os.environ.get("REAL2SIM_ANYSPLAT_ROOT")) + parser.add_argument("--model-id", default=os.environ.get("REAL2SIM_ANYSPLAT_MODEL_ID", "lhjiang/anysplat")) + parser.add_argument("--max-views", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MAX_VIEWS", "8"))) + parser.add_argument("--mesh-grid", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MESH_GRID", "48"))) + parser.add_argument("--shift-and-scale", action="store_true", help="Normalize exported Gaussian positions with AnySplat exporter") + args = parser.parse_args() + + if args.anysplat_root is None: + raise SystemExit("AnySplat root is required via --anysplat-root or REAL2SIM_ANYSPLAT_ROOT") + sys.path.insert(0, args.anysplat_root.as_posix()) + + import torch + from src.model.model.anysplat import AnySplat + from src.model.ply_export import export_ply + from src.utils.image import process_image + from real2sim.builtin import write_depth_mesh_obj + + args.output_dir.mkdir(parents=True, exist_ok=True) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = AnySplat.from_pretrained(args.model_id).to(device) + model.eval() + for param in model.parameters(): + param.requires_grad = False + + image_paths = collect_image_paths(args.image, args.frames_dir, max_views=args.max_views) + images = torch.stack([process_image(path.as_posix()) for path in image_paths], dim=0).unsqueeze(0).to(device) + b, views, _, height, width = images.shape + with torch.inference_mode(): + gaussians, pose = model.inference((images + 1) * 0.5) + + export_ply( + gaussians.means[0], + gaussians.scales[0], + gaussians.rotations[0], + gaussians.harmonics[0], + gaussians.opacities[0], + args.ply, + shift_and_scale=args.shift_and_scale, + save_sh_dc_only=True, + ) + + extrinsic = pose["extrinsic"] + intrinsic = pose["intrinsic"] + near = torch.ones(b, views, device=device) * 0.1 + far = torch.ones(b, views, device=device) * 100.0 + with torch.inference_mode(): + rendered = model.decoder.forward(gaussians, extrinsic, intrinsic.float(), near, far, (height, width), "depth") + + preview_metrics = None + if args.source_preview is not None: + source_rgb_tensor = ((images[0, 0].detach().cpu() + 1.0) * 0.5).clamp(0.0, 1.0) + rendered_rgb_tensor = rendered.color[0, 0].detach().cpu().clamp(0.0, 1.0) + preview_metrics = save_source_preview(args.source_preview, rendered_rgb_tensor, source_rgb_tensor) + + depth = extract_first_depth_map(rendered.depth) + normalized_depth = normalize_depth(depth) + write_gray(args.depth, normalized_depth) + + intrinsics = denormalize_intrinsics(intrinsic[0, 0].detach().cpu().numpy(), width, height) + scene_rgb = (((images[0, 0].detach().cpu().permute(1, 2, 0).numpy()) + 1.0) * 0.5).clip(0.0, 1.0) + write_depth_mesh_obj(args.mesh, scene_rgb, normalized_depth, intrinsics, grid=max(8, int(args.mesh_grid))) + + write_json( + args.cameras, + { + "backend": "anysplat", + "model_id": args.model_id, + "image_paths": [path.as_posix() for path in image_paths], + "image_shape": [height, width], + "intrinsics": intrinsics, + "source_preview": args.source_preview.as_posix() if args.source_preview is not None else None, + "source_preview_metrics": preview_metrics, + "pred_context_pose": to_jsonable(pose), + }, + ) + write_json( + args.output_dir / "anysplat_metadata.json", + { + "backend": "anysplat", + "model_id": args.model_id, + "num_views": len(image_paths), + "max_views": args.max_views, + "mesh_grid": max(8, int(args.mesh_grid)), + "shift_and_scale": args.shift_and_scale, + "source_preview": args.source_preview.as_posix() if args.source_preview is not None else None, + "source_preview_metrics": preview_metrics, + }, + ) + return 0 + + +def save_source_preview(output: Path, rendered_rgb, source_rgb) -> dict[str, float]: + from PIL import Image + + rendered = tensor_rgb_to_numpy(rendered_rgb) + source = tensor_rgb_to_numpy(source_rgb) + mae = float(np.mean(np.abs(rendered.astype(np.float32) / 255.0 - source.astype(np.float32) / 255.0))) + mse = float(np.mean((rendered.astype(np.float32) / 255.0 - source.astype(np.float32) / 255.0) ** 2)) + psnr = float(99.0 if mse <= 1e-12 else -10.0 * np.log10(mse)) + output.parent.mkdir(parents=True, exist_ok=True) + Image.fromarray(rendered).save(output) + return {"mae": mae, "mse": mse, "psnr": psnr} + + +def tensor_rgb_to_numpy(rgb) -> np.ndarray: + if hasattr(rgb, "detach"): + rgb = rgb.detach().cpu().numpy() + arr = np.asarray(rgb, dtype=np.float32) + if arr.ndim == 3 and arr.shape[0] == 3: + arr = np.transpose(arr, (1, 2, 0)) + if arr.ndim != 3 or arr.shape[-1] != 3: + raise SystemExit(f"RGB preview tensor must have shape 3xHxW or HxWx3, got {arr.shape}") + return (arr.clip(0.0, 1.0) * 255.0).round().astype(np.uint8) + + +def collect_image_paths(image: Path, frames_dir: Path | None, *, max_views: int) -> list[Path]: + if frames_dir is not None and str(frames_dir) not in {"", ".", "None", "__none__"} and frames_dir.is_dir(): + frames = sorted(path for path in frames_dir.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES) + if frames: + return sample_evenly(frames, max_views=max(1, max_views)) + return [image] + + +def sample_evenly(paths: list[Path], *, max_views: int) -> list[Path]: + if len(paths) <= max_views: + return paths + if max_views == 1: + return [paths[0]] + indices = np.linspace(0, len(paths) - 1, num=max_views).round().astype(int) + return [paths[int(idx)] for idx in sorted(set(indices))] + + +def normalize_depth(depth: np.ndarray) -> np.ndarray: + finite = np.asarray(depth, dtype=np.float32) + valid = np.isfinite(finite) + if not valid.any(): + return np.zeros_like(finite, dtype=np.float32) + lo = float(np.quantile(finite[valid], 0.02)) + hi = float(np.quantile(finite[valid], 0.98)) + if hi <= lo: + return np.zeros_like(finite, dtype=np.float32) + return np.clip((finite - lo) / (hi - lo), 0.0, 1.0).astype(np.float32) + + +def extract_first_depth_map(depth) -> np.ndarray: + if hasattr(depth, "detach"): + depth = depth.detach().cpu().numpy() + arr = np.asarray(depth, dtype=np.float32) + if arr.ndim == 5 and arr.shape[-1] == 1: + arr = arr[..., 0] + if arr.ndim == 4: + arr = arr[0, 0] + elif arr.ndim == 3: + arr = arr[0] + elif arr.ndim != 2: + raise SystemExit(f"AnySplat decoder depth must be 2D, 3D, or 4D, got shape {arr.shape}") + if arr.ndim != 2: + raise SystemExit(f"AnySplat decoder depth did not reduce to a 2D map, got shape {arr.shape}") + return arr.astype(np.float32, copy=False) + + +def denormalize_intrinsics(intrinsic: np.ndarray, width: int, height: int) -> dict[str, float | int]: + intr = np.asarray(intrinsic, dtype=np.float32) + return { + "width": int(width), + "height": int(height), + "fx": float(intr[0, 0] * width), + "fy": float(intr[1, 1] * height), + "cx": float(intr[0, 2] * width), + "cy": float(intr[1, 2] * height), + } + + +def to_jsonable(value): + if isinstance(value, dict): + return {str(k): to_jsonable(v) for k, v in value.items()} + if hasattr(value, "detach"): + return value.detach().cpu().tolist() + if isinstance(value, (list, tuple)): + return [to_jsonable(v) for v in value] + return value + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/depth_anything_depth.py b/vendor/gs_playground/scripts/real2sim_external/depth_anything_depth.py new file mode 100644 index 0000000..fb00526 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/depth_anything_depth.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Depth Anything V2 adapter for the Real2Sim hybrid backend. + +The adapter writes a normalized PGM/PNG depth map that the built-in asset +generator can fuse with its geometry prior. The default model is the small +Transformers checkpoint so it can run on the Wuying 12 GB GPU. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.image_io import write_gray + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run Depth Anything V2 monocular depth for Real2Sim") + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--model", + default=os.environ.get("REAL2SIM_DEPTH_MODEL", "depth-anything/Depth-Anything-V2-Small-hf"), + help="Hugging Face depth-estimation model id", + ) + parser.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda"]) + parser.add_argument("--invert", dest="invert", action="store_true", help="Invert relative depth before writing") + parser.add_argument("--no-invert", dest="invert", action="store_false") + parser.set_defaults(invert=True) + args = parser.parse_args() + + depth = estimate_depth(args.image, args.model, args.device) + if args.invert: + depth = 1.0 - depth + write_gray(args.output, depth) + return 0 + + +def estimate_depth(image_path: Path, model_id: str, device: str) -> np.ndarray: + try: + import torch + from PIL import Image + from transformers import pipeline + except Exception as exc: # pragma: no cover - external runtime guard + raise SystemExit( + "Depth Anything requires torch, pillow, and transformers. " + "Install them in the Real2Sim environment before enabling depth_estimation." + ) from exc + + image = Image.open(image_path).convert("RGB") + device_index = -1 + if device == "cuda" or (device == "auto" and torch.cuda.is_available()): + device_index = 0 + + pipe = pipeline(task="depth-estimation", model=model_id, device=device_index) + result = pipe(image) + raw_depth = result.get("predicted_depth") + if raw_depth is not None: + depth = raw_depth.detach().float().cpu().numpy() + depth = np.squeeze(depth) + else: + depth = np.asarray(result["depth"], dtype=np.float32) + if depth.ndim == 3: + depth = depth[:, :, 0] + return normalize_depth(depth) + + +def normalize_depth(depth: np.ndarray) -> np.ndarray: + depth = depth.astype(np.float32) + lo = float(np.percentile(depth, 1.0)) + hi = float(np.percentile(depth, 99.0)) + if hi <= lo: + return np.zeros_like(depth, dtype=np.float32) + return np.clip((depth - lo) / (hi - lo), 0.0, 1.0).astype(np.float32) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/depth_mask_pose_align.py b/vendor/gs_playground/scripts/real2sim_external/depth_mask_pose_align.py new file mode 100644 index 0000000..d95d9e6 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/depth_mask_pose_align.py @@ -0,0 +1,1180 @@ +#!/usr/bin/env python3 +"""Depth/mask pose alignment for Real2Sim object assets. + +This optional stage refines object pose and scale after AnySplat scene +reconstruction and SAM-3D object reconstruction. It uses the same image-space +mask that was passed to SAM-3D, the scene depth map, and camera intrinsics to +place each object in the reconstructed scene coordinate frame. +""" + +from __future__ import annotations + +import argparse +import itertools +import struct +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.contracts import load_instances, read_json, write_json +from real2sim.image_io import read_gray + +MIN_RENDER_AREA_RATIO = 0.2 +MAX_RENDER_AREA_RATIO = 4.0 +MIN_RENDER_MASK_IOU = 0.005 +MAX_GAUSSIAN_ALIGNMENT_POINTS = 50000 +ALIGNMENT_REFINEMENT_ITERATIONS = 3 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Refine object poses from masks, depth, and camera intrinsics") + parser.add_argument("--assets", type=Path, required=True, help="Input Real2Sim manifest JSON") + parser.add_argument("--instances", type=Path, required=True, help="Instance JSON with mask paths") + parser.add_argument("--depth", type=Path, required=True, help="Scene depth map") + parser.add_argument("--cameras", type=Path, required=True, help="Camera metadata JSON") + parser.add_argument("--output", type=Path, required=True, help="Output aligned manifest JSON") + parser.add_argument("--work-dir", type=Path, default=None, help="Root used for relative asset paths") + parser.add_argument("--depth-min", type=float, default=0.4) + parser.add_argument("--depth-max", type=float, default=2.4) + parser.add_argument("--z-offset", type=float, default=0.55) + args = parser.parse_args() + + work_dir = args.work_dir or args.assets.parent.parent + manifest = read_json(args.assets) + refined = refine_assets( + manifest, + instances_json=args.instances, + depth_path=args.depth, + cameras_path=args.cameras, + work_dir=work_dir, + depth_range=(args.depth_min, args.depth_max), + z_offset=args.z_offset, + ) + write_json(args.output, refined) + return 0 + + +def refine_assets( + manifest: dict[str, Any], + *, + instances_json: Path, + depth_path: Path, + cameras_path: Path, + work_dir: Path, + depth_range: tuple[float, float], + z_offset: float, +) -> dict[str, Any]: + refined = dict(manifest) + objects = [dict(obj) for obj in manifest.get("objects", [])] + refined["objects"] = objects + + alignment_meta: dict[str, Any] = { + "backend": "depth_mask_pose_align", + "depth": relative_or_absolute(depth_path, work_dir), + "cameras": relative_or_absolute(cameras_path, work_dir), + "instances": relative_or_absolute(instances_json, work_dir), + "aligned_objects": [], + "skipped_objects": [], + } + + if not depth_path.exists() or not cameras_path.exists() or not instances_json.exists(): + alignment_meta["status"] = "skipped_missing_global_artifact" + return with_alignment_metadata(refined, alignment_meta) + + depth = read_gray(depth_path) + intrinsics = load_intrinsics(cameras_path, depth.shape) + instances = {instance.instance_id: instance for instance in load_instances(instances_json)} + + for obj in objects: + name = str(obj.get("name", obj.get("label", "object"))) + instance_id = str(obj.get("source_instance_id") or name) + instance = instances.get(instance_id) + if instance is None or instance.mask_path is None or not instance.mask_path.exists(): + alignment_meta["skipped_objects"].append({"name": name, "reason": "missing_instance_mask"}) + continue + + mask = read_gray(instance.mask_path) > 0.5 + if mask.shape != depth.shape: + mask = resize_mask_to_camera_shape(mask, depth.shape) + pose = dict(obj.get("pose") or {}) + gaussian_path = resolve_optional_path(obj.get("gaussian_ply") or obj.get("ply"), work_dir) + render_failures: list[dict[str, Any]] = [] + if gaussian_path is not None and gaussian_path.exists(): + aligned_pose, render_meta = align_pose_by_rendered_gaussian( + gaussian_path, + pose, + target_mask=mask, + background_depth=depth, + intrinsics=intrinsics, + depth_range=depth_range, + z_offset=z_offset, + ) + if aligned_pose is not None: + pose.update(aligned_pose) + obj["pose"] = pose + alignment_entry = { + "name": name, + "source_instance_id": instance_id, + "mask": relative_or_absolute(instance.mask_path, work_dir), + **render_meta, + } + alignment_meta["aligned_objects"].append(alignment_entry) + continue + render_failures.append(render_meta) + + mesh_path = resolve_optional_path(obj.get("mesh_obj") or obj.get("mesh"), work_dir) + if mesh_path is not None and mesh_path.exists(): + aligned_pose, render_meta = align_pose_by_rendered_mesh( + mesh_path, + pose, + target_mask=mask, + background_depth=depth, + intrinsics=intrinsics, + depth_range=depth_range, + z_offset=z_offset, + ) + if aligned_pose is not None: + pose.update(aligned_pose) + obj["pose"] = pose + alignment_entry = { + "name": name, + "source_instance_id": instance_id, + "mask": relative_or_absolute(instance.mask_path, work_dir), + **render_meta, + } + alignment_meta["aligned_objects"].append(alignment_entry) + continue + render_failures.append(render_meta) + + aligned_pose, fallback_meta = align_pose_by_mask_depth( + pose, + target_mask=mask, + background_depth=depth, + intrinsics=intrinsics, + depth_range=depth_range, + z_offset=z_offset, + render_failures=render_failures, + ) + if aligned_pose is None: + alignment_meta["skipped_objects"].append( + { + "name": name, + "reason": "empty_mask_depth_overlap", + "render_failures": render_failures, + **fallback_meta, + } + ) + continue + + pose.update(aligned_pose) + obj["pose"] = pose + alignment_meta["aligned_objects"].append( + { + "name": name, + "source_instance_id": instance_id, + "mask": relative_or_absolute(instance.mask_path, work_dir), + **fallback_meta, + } + ) + + alignment_meta["quality_summary"] = summarize_alignment_quality(alignment_meta["aligned_objects"]) + alignment_meta["status"] = "ok" + return with_alignment_metadata(refined, alignment_meta) + + +def align_pose_by_mask_depth( + pose: dict[str, Any], + *, + target_mask: np.ndarray, + background_depth: np.ndarray, + intrinsics: dict[str, float], + depth_range: tuple[float, float], + z_offset: float, + render_failures: list[dict[str, Any]] | None = None, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + points = mask_depth_points(target_mask, background_depth, intrinsics, depth_range=depth_range, z_offset=z_offset) + meta: dict[str, Any] = { + "render_source": "mask_depth_fallback", + "target_mask_pixels": int(target_mask.sum()), + "mask_depth_points": int(points.shape[0]), + "render_failures": render_failures or [], + } + if points.size == 0: + return None, meta + + position = np.median(points, axis=0) + extent = np.maximum(points.max(axis=0) - points.min(axis=0), np.array([0.03, 0.03, 0.03], dtype=np.float32)) + output_pose = { + "position": vector_json(position), + "quat_wxyz": [1.0, 0.0, 0.0, 0.0], + "scale": vector_json(extent), + } + meta["quat_source"] = "identity_mask_depth_fallback" + meta["scale_source"] = "mask_depth_extent" + if render_failures: + meta["fallback_reason"] = "rendered_asset_mask_empty" + elif any(key in pose for key in ("gaussian_ply", "mesh_obj", "mesh")): + meta["fallback_reason"] = "renderable_asset_missing" + else: + meta["fallback_reason"] = "no_renderable_asset" + return output_pose, meta + + +def align_pose_by_rendered_mesh( + mesh_path: Path, + pose: dict[str, Any], + *, + target_mask: np.ndarray, + background_depth: np.ndarray, + intrinsics: dict[str, float], + depth_range: tuple[float, float], + z_offset: float = 0.55, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + vertices, faces = load_obj_mesh(mesh_path) + if vertices.size == 0: + return None, {"render_source": "mesh_obj", "target_mask_pixels": int(target_mask.sum()), "rendered_mask_pixels": 0} + return align_pose_by_rendered_asset( + lambda position, scale, rotation: render_mesh_arrays_depth_mask(vertices, faces, position, scale, rotation, intrinsics, target_mask.shape), + pose, + target_mask=target_mask, + background_depth=background_depth, + intrinsics=intrinsics, + depth_range=depth_range, + render_source="mesh_obj", + local_points=vertices, + z_offset=z_offset, + ) + + +def align_pose_by_rendered_gaussian( + gaussian_path: Path, + pose: dict[str, Any], + *, + target_mask: np.ndarray, + background_depth: np.ndarray, + intrinsics: dict[str, float], + depth_range: tuple[float, float], + z_offset: float = 0.55, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + points = load_gaussian_points(gaussian_path) + if points.size == 0: + return None, {"render_source": "gaussian_ply", "target_mask_pixels": int(target_mask.sum()), "rendered_mask_pixels": 0} + original_point_count = int(points.shape[0]) + points = downsample_alignment_points(points, max_points=MAX_GAUSSIAN_ALIGNMENT_POINTS) + canonicalization = load_gaussian_canonicalization_metadata(gaussian_path) + aligned_pose, render_meta = align_pose_by_rendered_asset( + lambda position, scale, rotation: render_gaussian_depth_mask(points, position, scale, rotation, intrinsics, target_mask.shape), + pose, + target_mask=target_mask, + background_depth=background_depth, + intrinsics=intrinsics, + depth_range=depth_range, + render_source="gaussian_ply", + local_points=points, + z_offset=z_offset, + ) + render_meta["gaussian_points"] = original_point_count + render_meta["gaussian_alignment_points"] = int(points.shape[0]) + if canonicalization is not None: + render_meta["gaussian_canonicalization"] = canonicalization + return aligned_pose, render_meta + + +def align_pose_by_rendered_asset( + render_depth_mask, + pose: dict[str, Any], + *, + target_mask: np.ndarray, + background_depth: np.ndarray, + intrinsics: dict[str, float], + depth_range: tuple[float, float], + render_source: str, + local_points: np.ndarray | None = None, + z_offset: float = 0.55, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + position = np.asarray(pose.get("position", [0.0, 1.0, 0.55]), dtype=np.float32) + scale = np.asarray(pose.get("scale", [1.0, 1.0, 1.0]), dtype=np.float32) + if scale.size == 1: + scale = np.repeat(scale, 3) + rotation = rotation_matrix_from_quat_wxyz(pose.get("quat_wxyz", [1.0, 0.0, 0.0, 0.0])) + target_pixels = int(target_mask.sum()) + if target_pixels <= 0: + return None, {"render_source": render_source, "target_mask_pixels": target_pixels, "rendered_mask_pixels": 0} + + target_depth = denormalized_depth_median(background_depth, target_mask, depth_range) + target_seed = estimate_mask_depth_pose(target_mask, background_depth, intrinsics, depth_range=depth_range, z_offset=z_offset) + attempts: list[dict[str, Any]] = [ + align_render_candidate( + render_depth_mask, + position, + scale, + rotation, + target_mask=target_mask, + target_depth=target_depth, + intrinsics=intrinsics, + label="pose", + ) + ] + + candidate_search = "pose_only" + if should_run_axis_search(attempts[0], target_mask.shape): + for candidate in axis_alignment_candidates(rotation): + seed_scale = candidate_extent_scale(local_points, target_seed["extent"], candidate["rotation"], scale) + attempts.append( + align_render_candidate( + render_depth_mask, + target_seed["position"], + seed_scale, + candidate["rotation"], + target_mask=target_mask, + target_depth=target_depth, + intrinsics=intrinsics, + label=candidate["label"], + ) + ) + candidate_search = "target_seed_axis_search" + + best = best_alignment_attempt(attempts) + if best is None or int(best.get("rerendered_mask_pixels", 0)) <= 0: + return ( + None, + { + "render_source": render_source, + "target_mask_pixels": target_pixels, + "rendered_mask_pixels": int(attempts[0].get("rendered_mask_pixels", 0)) if attempts else 0, + "candidate_count": len(attempts), + "candidate_attempts_top": compact_attempts(attempts), + }, + ) + rejection_reason = render_alignment_rejection_reason(best) + if rejection_reason is not None: + return ( + None, + { + "render_source": render_source, + "target_mask_pixels": target_pixels, + "rendered_mask_pixels": int(best.get("rendered_mask_pixels", 0)), + "rerendered_mask_pixels": int(best.get("rerendered_mask_pixels", 0)), + "mask_iou": round(float(best.get("mask_iou", 0.0)), 7), + "area_ratio": round(float(best.get("area_ratio", 0.0)), 7), + "centroid_error_px": round(float(best.get("centroid_error_px", 0.0)), 7), + "alignment_score": round(float(best.get("alignment_score", 0.0)), 7), + "refinement_iterations": int(best.get("refinement_iterations", 0)), + "rejection_reason": rejection_reason, + "candidate_count": len(attempts), + "candidate_attempts_top": compact_attempts(attempts), + }, + ) + + output_pose = { + "position": vector_json(best["position"]), + "quat_wxyz": vector_json(quat_wxyz_from_rotation_matrix(best["rotation"])), + "scale": vector_json(best["scale"]), + } + meta = { + "render_source": render_source, + "target_mask_pixels": target_pixels, + "rendered_mask_pixels": int(best["rendered_mask_pixels"]), + "rerendered_mask_pixels": int(best["rerendered_mask_pixels"]), + "target_depth": round(float(target_depth), 7), + "rendered_depth": round(float(best["rendered_depth"]), 7), + "rerendered_depth": round(float(best["rerendered_depth"]), 7), + "scale_factor": round(float(best["scale_factor"]), 7), + "mask_iou": round(float(best["mask_iou"]), 7), + "centroid_error_px": round(float(best["centroid_error_px"]), 7), + "depth_abs_error": round(float(best["depth_abs_error"]), 7), + "area_ratio": round(float(best["area_ratio"]), 7), + "alignment_score": round(float(best["alignment_score"]), 7), + "refinement_iterations": int(best.get("refinement_iterations", 0)), + "candidate_search": candidate_search if len(attempts) > 1 else "pose_only", + "candidate_label": best["label"], + "candidate_count": len(attempts), + "candidate_attempts_top": compact_attempts(attempts), + } + return output_pose, meta + + +def align_render_candidate( + render_depth_mask, + position: np.ndarray, + scale: np.ndarray, + rotation: np.ndarray, + *, + target_mask: np.ndarray, + target_depth: float, + intrinsics: dict[str, float], + label: str, +) -> dict[str, Any]: + position = np.asarray(position, dtype=np.float32).reshape(3).copy() + scale = normalize_scale(scale) + initial_scale = scale.copy() + rotation = np.asarray(rotation, dtype=np.float32).reshape(3, 3) + rendered_mask, rendered_depth = render_depth_mask(position, scale, rotation) + rendered_pixels = int(rendered_mask.sum()) + target_pixels = int(target_mask.sum()) + rendered_depth_median = float(np.median(rendered_depth[rendered_mask])) if rendered_pixels > 0 else 0.0 + attempt: dict[str, Any] = { + "label": label, + "position": position, + "scale": scale, + "rotation": rotation, + "target_mask_pixels": target_pixels, + "rendered_mask_pixels": rendered_pixels, + "rerendered_mask_pixels": 0, + "rendered_depth": rendered_depth_median, + "rerendered_depth": 0.0, + "scale_factor": 1.0, + "refinement_iterations": 0, + "mask_iou": 0.0, + "centroid_error_px": float(max(target_mask.shape)), + "depth_abs_error": float("inf"), + "area_ratio": 0.0, + "alignment_score": -float("inf"), + } + if rendered_pixels <= 0 or target_pixels <= 0: + return attempt + + target_center_yx = mask_centroid(target_mask) + scale_floor = np.array([1e-4, 1e-4, 1e-4], dtype=np.float32) + rerendered_mask = rendered_mask + rerendered_depth = rendered_depth + completed_iterations = 0 + best_state: dict[str, Any] | None = None + for _idx in range(ALIGNMENT_REFINEMENT_ITERATIONS): + current_pixels = int(rerendered_mask.sum()) + if current_pixels <= 0: + break + current_depth = float(np.median(rerendered_depth[rerendered_mask])) + scale_step = float(np.sqrt(target_pixels / float(current_pixels))) + if not np.isfinite(scale_step) or scale_step <= 0: + scale_step = 1.0 + scale = np.maximum(scale * scale_step, scale_floor) + position[1] += target_depth - current_depth + + rendered_center_yx = mask_centroid(rerendered_mask) + if target_center_yx is not None and rendered_center_yx is not None: + dy = float(target_center_yx[0] - rendered_center_yx[0]) + dx = float(target_center_yx[1] - rendered_center_yx[1]) + fx = max(float(intrinsics["fx"]), 1e-6) + fy = max(float(intrinsics["fy"]), 1e-6) + position[0] += dx / fx * target_depth + position[2] -= dy / fy * target_depth + + rerendered_mask, rerendered_depth = render_depth_mask(position, scale, rotation) + completed_iterations += 1 + metrics = render_alignment_metrics(target_mask, rerendered_mask, rerendered_depth, target_depth) + if best_state is None or metrics["alignment_score"] > best_state["metrics"]["alignment_score"]: + best_state = { + "position": position.copy(), + "scale": scale.copy(), + "rerendered_mask_pixels": int(rerendered_mask.sum()), + "metrics": metrics, + "refinement_iterations": completed_iterations, + } + + if best_state is None: + metrics = render_alignment_metrics(target_mask, rerendered_mask, rerendered_depth, target_depth) + best_state = { + "position": position.copy(), + "scale": scale.copy(), + "rerendered_mask_pixels": int(rerendered_mask.sum()), + "metrics": metrics, + "refinement_iterations": completed_iterations, + } + metrics = best_state["metrics"] + initial_scale_values = np.maximum(initial_scale, scale_floor) + best_scale = np.asarray(best_state["scale"], dtype=np.float32) + total_scale = float(np.median(best_scale / initial_scale_values)) + attempt.update( + { + "position": best_state["position"], + "scale": best_scale, + "rerendered_mask_pixels": int(best_state["rerendered_mask_pixels"]), + "rerendered_depth": metrics["rerendered_depth"], + "scale_factor": total_scale if np.isfinite(total_scale) else 1.0, + "refinement_iterations": int(best_state["refinement_iterations"]), + **metrics, + } + ) + return attempt + + +def best_alignment_attempt(attempts: list[dict[str, Any]]) -> dict[str, Any] | None: + valid = [attempt for attempt in attempts if int(attempt.get("rerendered_mask_pixels", 0)) > 0] + if not valid: + return None + return max(valid, key=lambda attempt: float(attempt.get("alignment_score", -float("inf")))) + + +def render_alignment_rejection_reason(attempt: dict[str, Any]) -> str | None: + area_ratio = float(attempt.get("area_ratio", 0.0)) + if not np.isfinite(area_ratio) or area_ratio < MIN_RENDER_AREA_RATIO or area_ratio > MAX_RENDER_AREA_RATIO: + return "rendered_area_ratio_out_of_range" + mask_iou = float(attempt.get("mask_iou", 0.0)) + if not np.isfinite(mask_iou) or mask_iou < MIN_RENDER_MASK_IOU: + return "rendered_mask_iou_too_low" + return None + + +def render_alignment_metrics( + target_mask: np.ndarray, + rendered_mask: np.ndarray, + rendered_depth: np.ndarray, + target_depth: float, +) -> dict[str, float]: + rendered_pixels = int(rendered_mask.sum()) + target_pixels = int(target_mask.sum()) + iou = mask_iou(target_mask, rendered_mask) + centroid_error = centroid_distance(target_mask, rendered_mask) + rerendered_depth = float(np.median(rendered_depth[rendered_mask])) if rendered_pixels > 0 else 0.0 + depth_abs_error = abs(float(target_depth) - rerendered_depth) if rendered_pixels > 0 else float("inf") + area_ratio = float(rendered_pixels / max(target_pixels, 1)) + area_penalty = abs(np.log(max(area_ratio, 1e-6))) + score = iou * 100.0 - centroid_error * 0.05 - depth_abs_error * 5.0 - area_penalty + return { + "mask_iou": float(iou), + "centroid_error_px": float(centroid_error), + "rerendered_depth": float(rerendered_depth), + "depth_abs_error": float(depth_abs_error), + "area_ratio": float(area_ratio), + "alignment_score": float(score), + } + + +def should_run_axis_search(initial_attempt: dict[str, Any], shape: tuple[int, int]) -> bool: + if int(initial_attempt.get("rerendered_mask_pixels", 0)) <= 0: + return True + if float(initial_attempt.get("mask_iou", 0.0)) < 0.2: + return True + return float(initial_attempt.get("centroid_error_px", 0.0)) > max(shape) * 0.15 + + +def estimate_mask_depth_pose( + target_mask: np.ndarray, + background_depth: np.ndarray, + intrinsics: dict[str, float], + *, + depth_range: tuple[float, float], + z_offset: float, +) -> dict[str, np.ndarray]: + points = mask_depth_points(target_mask, background_depth, intrinsics, depth_range=depth_range, z_offset=z_offset) + if points.size == 0: + depth = denormalized_depth_median(background_depth, target_mask, depth_range) + return { + "position": np.asarray([0.0, depth, z_offset], dtype=np.float32), + "extent": np.asarray([0.05, 0.05, 0.05], dtype=np.float32), + } + extent = np.maximum(points.max(axis=0) - points.min(axis=0), np.array([0.03, 0.03, 0.03], dtype=np.float32)) + return {"position": np.median(points, axis=0).astype(np.float32), "extent": extent.astype(np.float32)} + + +def candidate_extent_scale( + local_points: np.ndarray | None, + target_extent: np.ndarray, + rotation: np.ndarray, + fallback_scale: np.ndarray, +) -> np.ndarray: + fallback = normalize_scale(fallback_scale) + if local_points is None or local_points.size == 0: + return fallback + local_extent = np.maximum(local_points.max(axis=0) - local_points.min(axis=0), np.array([1e-4, 1e-4, 1e-4], dtype=np.float32)) + rotated_extent = np.maximum(np.abs(rotation) @ local_extent, np.array([1e-4, 1e-4, 1e-4], dtype=np.float32)) + ratios = np.asarray(target_extent, dtype=np.float32).reshape(3) / rotated_extent + ratios = ratios[np.isfinite(ratios) & (ratios > 0)] + if ratios.size == 0: + return fallback + uniform = float(np.median(ratios)) + if not np.isfinite(uniform) or uniform <= 0: + return fallback + return np.repeat(max(uniform, 1e-4), 3).astype(np.float32) + + +def axis_alignment_candidates(base_rotation: np.ndarray) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [{"label": "pose_rotation", "rotation": np.asarray(base_rotation, dtype=np.float32).reshape(3, 3)}] + seen = {rotation_key(candidates[0]["rotation"])} + axis_names = ("x", "y", "z") + for perm in itertools.permutations(range(3)): + for signs in itertools.product((-1.0, 1.0), repeat=3): + rotation = np.zeros((3, 3), dtype=np.float32) + for world_axis, local_axis in enumerate(perm): + rotation[world_axis, local_axis] = signs[world_axis] + if np.linalg.det(rotation) < 0.5: + continue + key = rotation_key(rotation) + if key in seen: + continue + seen.add(key) + label = "axis_" + "".join(axis_names[idx] for idx in perm) + "_" + "".join("p" if sign > 0 else "n" for sign in signs) + candidates.append({"label": label, "rotation": rotation}) + return candidates + + +def load_gaussian_canonicalization_metadata(gaussian_path: Path) -> dict[str, Any] | None: + metadata_path = gaussian_path.parent / "sam3d_metadata.json" + if not metadata_path.exists(): + return None + try: + metadata = read_json(metadata_path) + except Exception: + return {"status": "unreadable", "metadata": relative_or_absolute(metadata_path, gaussian_path.parent)} + canonicalization = metadata.get("gaussian_canonicalization") if isinstance(metadata, dict) else None + if not isinstance(canonicalization, dict): + return None + keep = { + "enabled": canonicalization.get("enabled"), + "status": canonicalization.get("status"), + "changed": canonicalization.get("changed"), + "method": canonicalization.get("method"), + "gaussian_axes_to_mesh_axes": canonicalization.get("gaussian_axes_to_mesh_axes"), + "mesh_axis_signs": canonicalization.get("mesh_axis_signs"), + } + return {key: value for key, value in keep.items() if value is not None} + + +def compact_attempts(attempts: list[dict[str, Any]], *, limit: int = 5) -> list[dict[str, Any]]: + ranked = sorted(attempts, key=lambda attempt: float(attempt.get("alignment_score", -float("inf"))), reverse=True) + compact = [] + for attempt in ranked[:limit]: + compact.append( + { + "label": attempt.get("label"), + "rendered_mask_pixels": int(attempt.get("rendered_mask_pixels", 0)), + "rerendered_mask_pixels": int(attempt.get("rerendered_mask_pixels", 0)), + "mask_iou": round(float(attempt.get("mask_iou", 0.0)), 7), + "centroid_error_px": round(float(attempt.get("centroid_error_px", 0.0)), 7), + "depth_abs_error": round(float(attempt.get("depth_abs_error", 0.0)), 7) + if np.isfinite(float(attempt.get("depth_abs_error", 0.0))) + else None, + "refinement_iterations": int(attempt.get("refinement_iterations", 0)), + "alignment_score": round(float(attempt.get("alignment_score", -float("inf"))), 7) + if np.isfinite(float(attempt.get("alignment_score", -float("inf")))) + else None, + } + ) + return compact + + +def render_gaussian_depth_mask( + points: np.ndarray, + position: np.ndarray, + scale: np.ndarray, + rotation: np.ndarray | None, + intrinsics: dict[str, float], + shape: tuple[int, int], + *, + radius_pixels: int = 2, +) -> tuple[np.ndarray, np.ndarray]: + if points.size == 0: + return np.zeros(shape, dtype=bool), np.zeros(shape, dtype=np.float32) + transformed = transform_local_points(points, position, scale, rotation) + return render_points_depth_mask(transformed, intrinsics, shape, radius_pixels=radius_pixels) + + +def render_mesh_depth_mask( + mesh_path: Path, + position: np.ndarray, + scale: np.ndarray, + rotation: np.ndarray | None, + intrinsics: dict[str, float], + shape: tuple[int, int], +) -> tuple[np.ndarray, np.ndarray]: + vertices, faces = load_obj_mesh(mesh_path) + return render_mesh_arrays_depth_mask(vertices, faces, position, scale, rotation, intrinsics, shape) + + +def render_mesh_arrays_depth_mask( + vertices: np.ndarray, + faces: list[tuple[int, int, int]], + position: np.ndarray, + scale: np.ndarray, + rotation: np.ndarray | None, + intrinsics: dict[str, float], + shape: tuple[int, int], +) -> tuple[np.ndarray, np.ndarray]: + if vertices.size == 0: + return np.zeros(shape, dtype=bool), np.zeros(shape, dtype=np.float32) + transformed = transform_local_points(vertices, position, scale, rotation) + if not faces: + return render_points_depth_mask(transformed, intrinsics, shape, radius_pixels=1) + height, width = shape + fx = max(float(intrinsics["fx"]), 1e-6) + fy = max(float(intrinsics["fy"]), 1e-6) + cx = float(intrinsics["cx"]) + cy = float(intrinsics["cy"]) + depth = np.full(shape, np.inf, dtype=np.float32) + pixels = np.zeros((transformed.shape[0], 2), dtype=np.float32) + z = np.maximum(transformed[:, 1], 1e-6) + pixels[:, 0] = transformed[:, 0] / z * fx + cx + pixels[:, 1] = cy - (transformed[:, 2] / z * fy) + for face in faces: + rasterize_triangle(pixels[list(face)], z[list(face)], depth) + mask = np.isfinite(depth) + depth[~mask] = 0.0 + return mask, depth + + +def transform_local_points( + points: np.ndarray, + position: np.ndarray, + scale: np.ndarray, + rotation: np.ndarray | None, +) -> np.ndarray: + local = np.asarray(points, dtype=np.float32) * normalize_scale(scale).reshape(1, 3) + rot = np.eye(3, dtype=np.float32) if rotation is None else np.asarray(rotation, dtype=np.float32).reshape(3, 3) + return local @ rot.T + np.asarray(position, dtype=np.float32).reshape(1, 3) + + +def render_points_depth_mask( + points: np.ndarray, + intrinsics: dict[str, float], + shape: tuple[int, int], + *, + radius_pixels: int, +) -> tuple[np.ndarray, np.ndarray]: + height, width = shape + fx = max(float(intrinsics["fx"]), 1e-6) + fy = max(float(intrinsics["fy"]), 1e-6) + cx = float(intrinsics["cx"]) + cy = float(intrinsics["cy"]) + depth = np.full(shape, np.inf, dtype=np.float32) + z = np.maximum(points[:, 1], 1e-6) + px = points[:, 0] / z * fx + cx + py = cy - (points[:, 2] / z * fy) + radius = max(0, int(radius_pixels)) + for x_value, y_value, z_value in zip(px, py, z): + x_center = int(round(float(x_value))) + y_center = int(round(float(y_value))) + if x_center < -radius or x_center >= width + radius or y_center < -radius or y_center >= height + radius: + continue + for y in range(max(0, y_center - radius), min(height, y_center + radius + 1)): + for x in range(max(0, x_center - radius), min(width, x_center + radius + 1)): + if (x - x_center) ** 2 + (y - y_center) ** 2 <= radius ** 2: + depth[y, x] = min(depth[y, x], float(z_value)) + mask = np.isfinite(depth) + depth[~mask] = 0.0 + return mask, depth + + +def rasterize_triangle(points: np.ndarray, z_values: np.ndarray, depth: np.ndarray) -> None: + height, width = depth.shape + min_x = max(0, int(np.floor(float(points[:, 0].min())))) + max_x = min(width - 1, int(np.ceil(float(points[:, 0].max())))) + min_y = max(0, int(np.floor(float(points[:, 1].min())))) + max_y = min(height - 1, int(np.ceil(float(points[:, 1].max())))) + if max_x < min_x or max_y < min_y: + return + a, b, c = points.astype(np.float32) + denom = (b[1] - c[1]) * (a[0] - c[0]) + (c[0] - b[0]) * (a[1] - c[1]) + if abs(float(denom)) < 1e-8: + return + for y in range(min_y, max_y + 1): + for x in range(min_x, max_x + 1): + px = x + 0.5 + py = y + 0.5 + w0 = ((b[1] - c[1]) * (px - c[0]) + (c[0] - b[0]) * (py - c[1])) / denom + w1 = ((c[1] - a[1]) * (px - c[0]) + (a[0] - c[0]) * (py - c[1])) / denom + w2 = 1.0 - w0 - w1 + if w0 >= -1e-5 and w1 >= -1e-5 and w2 >= -1e-5: + z = float(w0 * z_values[0] + w1 * z_values[1] + w2 * z_values[2]) + if z < depth[y, x]: + depth[y, x] = z + + +def load_obj_mesh(path: Path) -> tuple[np.ndarray, list[tuple[int, int, int]]]: + vertices: list[tuple[float, float, float]] = [] + faces: list[tuple[int, int, int]] = [] + with path.open("r", encoding="utf-8", errors="ignore") as f: + for line in f: + if line.startswith("v "): + parts = line.split() + if len(parts) >= 4: + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + elif line.startswith("f "): + raw = [] + for token in line.split()[1:]: + index = token.split("/")[0] + if index: + raw.append(int(index) - 1) + if len(raw) >= 3: + for idx in range(1, len(raw) - 1): + faces.append((raw[0], raw[idx], raw[idx + 1])) + return np.asarray(vertices, dtype=np.float32), faces + + +PLY_SCALAR_FORMATS = { + "char": "b", + "int8": "b", + "uchar": "B", + "uint8": "B", + "short": "h", + "int16": "h", + "ushort": "H", + "uint16": "H", + "int": "i", + "int32": "i", + "uint": "I", + "uint32": "I", + "float": "f", + "float32": "f", + "double": "d", + "float64": "d", +} + + +def load_gaussian_points(path: Path) -> np.ndarray: + data = path.read_bytes() + header_marker = b"end_header" + marker_index = data.find(header_marker) + if marker_index < 0: + return np.zeros((0, 3), dtype=np.float32) + body_start = data.find(b"\n", marker_index) + if body_start < 0: + return np.zeros((0, 3), dtype=np.float32) + header = data[: body_start + 1].decode("ascii", errors="ignore").splitlines() + body = data[body_start + 1 :] + fmt = "ascii" + properties: list[tuple[str, str]] = [] + vertex_count = 0 + current_element = "" + for line in header: + parts = line.strip().split() + if len(parts) >= 3 and parts[0] == "format": + fmt = parts[1] + elif len(parts) >= 3 and parts[0] == "element": + current_element = parts[1] + if current_element == "vertex": + vertex_count = int(parts[2]) + elif len(parts) >= 3 and parts[0] == "property" and current_element == "vertex": + if parts[1] == "list": + return np.zeros((0, 3), dtype=np.float32) + properties.append((parts[-1], parts[1])) + if vertex_count <= 0 or not properties: + return np.zeros((0, 3), dtype=np.float32) + names = [name for name, _kind in properties] + if all(name in names for name in ("x", "y", "z")): + indices = [names.index("x"), names.index("y"), names.index("z")] + else: + indices = [0, 1, 2] + if len(properties) <= max(indices): + return np.zeros((0, 3), dtype=np.float32) + if fmt == "ascii": + points = load_ascii_ply_points(body, vertex_count, indices) + elif fmt in {"binary_little_endian", "binary_big_endian"}: + points = load_binary_ply_points(body, vertex_count, properties, indices, little_endian=fmt == "binary_little_endian") + else: + points = [] + return np.asarray(points, dtype=np.float32) + + +def downsample_alignment_points(points: np.ndarray, *, max_points: int = MAX_GAUSSIAN_ALIGNMENT_POINTS) -> np.ndarray: + points = np.asarray(points, dtype=np.float32) + if max_points <= 0 or points.shape[0] <= max_points: + return points + indices = np.linspace(0, points.shape[0] - 1, int(max_points), dtype=np.int64) + return points[indices] + + +def load_ascii_ply_points(body: bytes, vertex_count: int, indices: list[int]) -> list[tuple[float, float, float]]: + points = [] + for line in body.decode("ascii", errors="ignore").splitlines(): + if len(points) >= vertex_count: + break + stripped = line.strip() + if not stripped: + continue + try: + row = [float(value) for value in stripped.split()] + except ValueError: + continue + if len(row) > max(indices): + points.append((row[indices[0]], row[indices[1]], row[indices[2]])) + return points + + +def load_binary_ply_points( + body: bytes, + vertex_count: int, + properties: list[tuple[str, str]], + indices: list[int], + *, + little_endian: bool, +) -> list[tuple[float, float, float]]: + try: + row_format = ("<" if little_endian else ">") + "".join(PLY_SCALAR_FORMATS[kind] for _name, kind in properties) + except KeyError: + return [] + row_size = struct.calcsize(row_format) + if row_size <= 0: + return [] + count = min(vertex_count, len(body) // row_size) + points = [] + for idx in range(count): + row = struct.unpack_from(row_format, body, idx * row_size) + points.append((float(row[indices[0]]), float(row[indices[1]]), float(row[indices[2]]))) + return points + + +def denormalized_depth_median(depth: np.ndarray, mask: np.ndarray, depth_range: tuple[float, float]) -> float: + values = np.asarray(depth[mask], dtype=np.float32) + values = values[np.isfinite(values)] + if values.size == 0: + return float(sum(depth_range) * 0.5) + depth_min, depth_max = depth_range + return float(depth_min + np.median(np.clip(values, 0.0, 1.0)) * (depth_max - depth_min)) + + +def mask_centroid(mask: np.ndarray) -> tuple[float, float] | None: + ys, xs = np.where(mask) + if ys.size == 0: + return None + return (float(ys.mean()), float(xs.mean())) + + +def mask_iou(a: np.ndarray, b: np.ndarray) -> float: + intersection = int(np.logical_and(a, b).sum()) + union = int(np.logical_or(a, b).sum()) + return float(intersection / union) if union > 0 else 0.0 + + +def centroid_distance(a: np.ndarray, b: np.ndarray) -> float: + a_center = mask_centroid(a) + b_center = mask_centroid(b) + if a_center is None or b_center is None: + return float(max(a.shape)) + return float(np.hypot(a_center[0] - b_center[0], a_center[1] - b_center[1])) + + +def normalize_scale(scale: np.ndarray | list[float] | tuple[float, ...]) -> np.ndarray: + values = np.asarray(scale, dtype=np.float32).reshape(-1) + if values.size == 0: + values = np.ones(3, dtype=np.float32) + if values.size == 1: + values = np.repeat(values, 3) + values = values[:3] if values.size >= 3 else np.pad(values, (0, 3 - values.size), constant_values=1.0) + return np.maximum(values.astype(np.float32), np.array([1e-4, 1e-4, 1e-4], dtype=np.float32)) + + +def rotation_matrix_from_quat_wxyz(raw_quat: Any) -> np.ndarray: + quat = np.asarray(raw_quat, dtype=np.float32).reshape(-1) + if quat.size < 4: + quat = np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32) + w, x, y, z = quat[:4] + norm = float(np.linalg.norm([w, x, y, z])) + if norm <= 1e-8: + return np.eye(3, dtype=np.float32) + w, x, y, z = (w / norm, x / norm, y / norm, z / norm) + return np.asarray( + [ + [1.0 - 2.0 * (y * y + z * z), 2.0 * (x * y - z * w), 2.0 * (x * z + y * w)], + [2.0 * (x * y + z * w), 1.0 - 2.0 * (x * x + z * z), 2.0 * (y * z - x * w)], + [2.0 * (x * z - y * w), 2.0 * (y * z + x * w), 1.0 - 2.0 * (x * x + y * y)], + ], + dtype=np.float32, + ) + + +def quat_wxyz_from_rotation_matrix(rotation: np.ndarray) -> np.ndarray: + matrix = np.asarray(rotation, dtype=np.float64).reshape(3, 3) + trace = float(np.trace(matrix)) + if trace > 0.0: + s = np.sqrt(trace + 1.0) * 2.0 + quat = np.asarray( + [ + 0.25 * s, + (matrix[2, 1] - matrix[1, 2]) / s, + (matrix[0, 2] - matrix[2, 0]) / s, + (matrix[1, 0] - matrix[0, 1]) / s, + ], + dtype=np.float64, + ) + else: + axis = int(np.argmax(np.diag(matrix))) + if axis == 0: + s = np.sqrt(1.0 + matrix[0, 0] - matrix[1, 1] - matrix[2, 2]) * 2.0 + quat = np.asarray( + [ + (matrix[2, 1] - matrix[1, 2]) / s, + 0.25 * s, + (matrix[0, 1] + matrix[1, 0]) / s, + (matrix[0, 2] + matrix[2, 0]) / s, + ], + dtype=np.float64, + ) + elif axis == 1: + s = np.sqrt(1.0 + matrix[1, 1] - matrix[0, 0] - matrix[2, 2]) * 2.0 + quat = np.asarray( + [ + (matrix[0, 2] - matrix[2, 0]) / s, + (matrix[0, 1] + matrix[1, 0]) / s, + 0.25 * s, + (matrix[1, 2] + matrix[2, 1]) / s, + ], + dtype=np.float64, + ) + else: + s = np.sqrt(1.0 + matrix[2, 2] - matrix[0, 0] - matrix[1, 1]) * 2.0 + quat = np.asarray( + [ + (matrix[1, 0] - matrix[0, 1]) / s, + (matrix[0, 2] + matrix[2, 0]) / s, + (matrix[1, 2] + matrix[2, 1]) / s, + 0.25 * s, + ], + dtype=np.float64, + ) + norm = float(np.linalg.norm(quat)) + if norm <= 1e-8: + return np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32) + quat = quat / norm + if quat[0] < 0: + quat = -quat + return quat.astype(np.float32) + + +def rotation_key(rotation: np.ndarray) -> tuple[int, ...]: + return tuple(int(round(float(value) * 1000.0)) for value in np.asarray(rotation).reshape(-1)) + + +def summarize_alignment_quality(entries: list[dict[str, Any]]) -> dict[str, Any]: + rendered_entries = [entry for entry in entries if entry.get("render_source") in {"gaussian_ply", "mesh_obj"}] + fallback_entries = [entry for entry in entries if entry.get("render_source") == "mask_depth_fallback"] + summary: dict[str, Any] = { + "aligned_count": len(entries), + "rendered_asset_count": len(rendered_entries), + "mask_depth_fallback_count": len(fallback_entries), + } + for key in ("mask_iou", "centroid_error_px", "depth_abs_error", "area_ratio"): + values = [float(entry[key]) for entry in rendered_entries if entry.get(key) is not None and np.isfinite(float(entry[key]))] + if values: + summary[f"mean_{key}"] = round(float(np.mean(values)), 7) + summary[f"min_{key}"] = round(float(np.min(values)), 7) + summary[f"max_{key}"] = round(float(np.max(values)), 7) + return summary + + +def load_intrinsics(cameras_path: Path, shape: tuple[int, int]) -> dict[str, float]: + height, width = shape + default = { + "width": float(width), + "height": float(height), + "fx": float(max(width, height) * 0.85), + "fy": float(max(width, height) * 0.85), + "cx": float(width) / 2.0, + "cy": float(height) / 2.0, + } + raw = read_json(cameras_path) + intrinsics = raw.get("intrinsics", raw) if isinstance(raw, dict) else raw + if isinstance(intrinsics, dict): + out = dict(default) + for key in ("width", "height", "fx", "fy", "cx", "cy"): + if intrinsics.get(key) is not None: + out[key] = float(intrinsics[key]) + return out + + arr = np.asarray(intrinsics, dtype=np.float32) + if arr.shape[0] < 3 or arr.shape[1] < 3: + return default + fx = float(arr[0, 0]) + fy = float(arr[1, 1]) + cx = float(arr[0, 2]) + cy = float(arr[1, 2]) + if max(abs(fx), abs(fy), abs(cx), abs(cy)) <= 4.0: + fx *= width + fy *= height + cx *= width + cy *= height + return {"width": float(width), "height": float(height), "fx": fx, "fy": fy, "cx": cx, "cy": cy} + + +def mask_depth_points( + mask: np.ndarray, + depth: np.ndarray, + intrinsics: dict[str, float], + *, + depth_range: tuple[float, float], + z_offset: float, +) -> np.ndarray: + ys, xs = np.where(mask) + if ys.size == 0: + return np.zeros((0, 3), dtype=np.float32) + values = np.asarray(depth[ys, xs], dtype=np.float32) + valid = np.isfinite(values) + if not valid.any(): + return np.zeros((0, 3), dtype=np.float32) + ys = ys[valid].astype(np.float32) + xs = xs[valid].astype(np.float32) + values = np.clip(values[valid], 0.0, 1.0) + depth_min, depth_max = depth_range + z_depth = depth_min + values * (depth_max - depth_min) + fx = max(float(intrinsics["fx"]), 1e-6) + fy = max(float(intrinsics["fy"]), 1e-6) + cx = float(intrinsics["cx"]) + cy = float(intrinsics["cy"]) + xw = (xs - cx) / fx * z_depth + yw = z_depth + zw = (cy - ys) / fy * z_depth + z_offset + return np.stack([xw, yw, zw], axis=1).astype(np.float32) + + +def resize_mask_nearest(mask: np.ndarray, shape: tuple[int, int]) -> np.ndarray: + height, width = shape + src_h, src_w = mask.shape + y_idx = np.clip(np.round(np.linspace(0, src_h - 1, height)).astype(int), 0, src_h - 1) + x_idx = np.clip(np.round(np.linspace(0, src_w - 1, width)).astype(int), 0, src_w - 1) + return mask[y_idx][:, x_idx] + + +def resize_mask_to_camera_shape(mask: np.ndarray, shape: tuple[int, int]) -> np.ndarray: + """Match AnySplat process_image: aspect-preserving cover resize, center crop.""" + if mask.shape == shape: + return mask + src_h, src_w = mask.shape + dst_h, dst_w = shape + if src_h <= 0 or src_w <= 0 or dst_h <= 0 or dst_w <= 0: + return np.zeros(shape, dtype=bool) + scale = max(dst_w / float(src_w), dst_h / float(src_h)) + resized_h = max(dst_h, int(round(src_h * scale))) + resized_w = max(dst_w, int(round(src_w * scale))) + resized = resize_mask_nearest(mask, (resized_h, resized_w)) + top = max(0, (resized_h - dst_h) // 2) + left = max(0, (resized_w - dst_w) // 2) + cropped = resized[top : top + dst_h, left : left + dst_w] + if cropped.shape == shape: + return cropped + out = np.zeros(shape, dtype=cropped.dtype) + out[: cropped.shape[0], : cropped.shape[1]] = cropped + return out + + +def with_alignment_metadata(manifest: dict[str, Any], alignment_meta: dict[str, Any]) -> dict[str, Any]: + metadata = dict(manifest.get("metadata") or {}) + metadata["pose_alignment"] = alignment_meta + manifest["metadata"] = metadata + return manifest + + +def vector_json(values: np.ndarray) -> list[float]: + return [round(float(value), 7) for value in np.asarray(values, dtype=np.float32).reshape(-1).tolist()] + + +def relative_or_absolute(path: Path, root: Path) -> str: + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + return path.resolve().as_posix() + + +def resolve_optional_path(value: Any, root: Path) -> Path | None: + if not value: + return None + path = Path(str(value)) + return path if path.is_absolute() else (root / path).resolve() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/groundingdino_hf_detect.py b/vendor/gs_playground/scripts/real2sim_external/groundingdino_hf_detect.py new file mode 100644 index 0000000..39b96e3 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/groundingdino_hf_detect.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""GroundingDINO detection adapter for Real2Sim. + +This uses the Hugging Face GroundingDINO interface used by the official +Grounded-SAM-2 demos. It writes the Real2Sim detections.json contract. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.contracts import write_json + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run GroundingDINO via Hugging Face and write detections.json") + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--model-id", default="IDEA-Research/grounding-dino-tiny") + parser.add_argument("--box-threshold", type=float, default=0.25) + parser.add_argument("--text-threshold", type=float, default=0.25) + parser.add_argument("--nms-threshold", type=float, default=0.6) + parser.add_argument("--max-detections", type=int, default=24) + parser.add_argument("--device", default="auto", choices=["auto", "cuda", "cpu"]) + args = parser.parse_args() + + import torch + from PIL import Image + from transformers import AutoModelForZeroShotObjectDetection, AutoProcessor + + device = "cuda" if args.device == "auto" and torch.cuda.is_available() else args.device + if device == "auto": + device = "cpu" + + image = Image.open(args.image).convert("RGB") + processor = AutoProcessor.from_pretrained(args.model_id) + model = AutoModelForZeroShotObjectDetection.from_pretrained(args.model_id).to(device) + + detections = [] + queries = prompt_queries(args.prompt) + for prompt_index, prompt in enumerate(queries): + inputs = processor(images=image, text=prompt, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model(**inputs) + + result = processor.post_process_grounded_object_detection( + outputs, + inputs.input_ids, + threshold=args.box_threshold, + text_threshold=args.text_threshold, + target_sizes=[image.size[::-1]], + )[0] + + boxes = result.get("boxes", []) + scores = result.get("scores", []) + labels = result.get("labels", []) + for idx, box in enumerate(boxes): + label = labels[idx] if idx < len(labels) else prompt.rstrip(".") + score = float(scores[idx].detach().cpu().item()) if idx < len(scores) else None + bbox = [float(v) for v in box.detach().cpu().tolist()] + detections.append( + { + "id": f"det_{prompt_index:03d}_{idx:03d}", + "label": str(label).strip().rstrip(".") or prompt.rstrip("."), + "score": score, + "bbox_xyxy": bbox, + "prompt_query": prompt, + "prompt_index": prompt_index, + } + ) + + detections = nms_detections(detections, args.nms_threshold)[: max(0, args.max_detections)] + write_json( + args.output, + { + "detections": detections, + "metadata": { + "backend": "groundingdino_hf", + "model_id": args.model_id, + "prompt": args.prompt, + "prompt_queries": queries, + "detection_scheme": "prompt_wise_independent", + "box_threshold": args.box_threshold, + "text_threshold": args.text_threshold, + "nms_threshold": args.nms_threshold, + "max_detections": args.max_detections, + }, + }, + ) + return 0 + + +def normalize_prompt(prompt: str) -> str: + text = " ".join(prompt.strip().lower().split()) + if not text: + text = "object" + return text if text.endswith(".") else f"{text}." + + +def prompt_queries(prompt: str) -> list[str]: + parts = [part.strip() for part in re.split(r"[.,;\n]+", prompt) if part.strip()] + if not parts: + parts = ["object"] + return [normalize_prompt(part) for part in parts] + + +def nms_detections(detections: list[dict], threshold: float) -> list[dict]: + pending = sorted(detections, key=lambda item: float(item.get("score") or 0.0), reverse=True) + kept: list[dict] = [] + while pending: + current = pending.pop(0) + kept.append(current) + pending = [ + candidate + for candidate in pending + if current.get("label") != candidate.get("label") + or iou(current["bbox_xyxy"], candidate["bbox_xyxy"]) < threshold + ] + for idx, item in enumerate(kept): + item["id"] = f"det_{idx:03d}" + return kept + + +def iou(a: list[float], b: list[float]) -> float: + ax0, ay0, ax1, ay1 = a + bx0, by0, bx1, by1 = b + ix0, iy0 = max(ax0, bx0), max(ay0, by0) + ix1, iy1 = min(ax1, bx1), min(ay1, by1) + iw, ih = max(0.0, ix1 - ix0), max(0.0, iy1 - iy0) + inter = iw * ih + area_a = max(0.0, ax1 - ax0) * max(0.0, ay1 - ay0) + area_b = max(0.0, bx1 - bx0) * max(0.0, by1 - by0) + union = area_a + area_b - inter + return inter / union if union > 0 else 0.0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/instance_refine.py b/vendor/gs_playground/scripts/real2sim_external/instance_refine.py new file mode 100644 index 0000000..02d6c59 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/instance_refine.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Paper-style visual instance refinement for Real2Sim. + +The GS-Playground paper avoids semantic-similarity de-duplication after +open-vocabulary detection. This stage uses only visual mask criteria and a +composite detection/segmentation confidence score. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.contracts import read_json, write_json +from real2sim.image_io import read_gray, read_rgb, write_gray + + +def main() -> int: + parser = argparse.ArgumentParser(description="Refine SAM/SAM2 instances using visual de-duplication") + parser.add_argument("--instances", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--image", type=Path, default=None, help="Optional source RGB image for accessory mask recovery") + parser.add_argument("--mask-iou-threshold", type=float, default=0.75) + parser.add_argument("--inclusion-threshold", type=float, default=0.92) + parser.add_argument("--boundary-overlap-threshold", type=float, default=0.85) + parser.add_argument("--merge-saturated-accessories", action="store_true") + parser.add_argument("--accessory-search-margin-ratio", type=float, default=0.25) + parser.add_argument("--accessory-saturation-threshold", type=float, default=0.45) + parser.add_argument("--accessory-contact-passes", type=int, default=4) + parser.add_argument("--accessory-max-area-growth", type=float, default=0.15) + parser.add_argument("--accessory-min-component-area-ratio", type=float, default=0.001) + parser.add_argument("--accessory-max-components", type=int, default=8) + args = parser.parse_args() + + raw = read_json(args.instances) + entries = raw.get("instances", raw) if isinstance(raw, dict) else raw + image = read_rgb(args.image) if args.image is not None and args.image.exists() else None + refined = refine_instance_entries( + entries, + args.instances.parent, + image=image, + merge_saturated_accessories=args.merge_saturated_accessories, + mask_iou_threshold=args.mask_iou_threshold, + inclusion_threshold=args.inclusion_threshold, + boundary_overlap_threshold=args.boundary_overlap_threshold, + accessory_search_margin_ratio=args.accessory_search_margin_ratio, + accessory_saturation_threshold=args.accessory_saturation_threshold, + accessory_contact_passes=args.accessory_contact_passes, + accessory_max_area_growth=args.accessory_max_area_growth, + accessory_min_component_area_ratio=args.accessory_min_component_area_ratio, + accessory_max_components=args.accessory_max_components, + ) + metadata = dict(raw.get("metadata", {})) if isinstance(raw, dict) else {} + metadata["instance_refinement"] = { + "backend": "visual_mask_dedup", + "input_instances": len(entries), + "output_instances": len(refined), + "mask_iou_threshold": args.mask_iou_threshold, + "inclusion_threshold": args.inclusion_threshold, + "boundary_overlap_threshold": args.boundary_overlap_threshold, + "merge_saturated_accessories": args.merge_saturated_accessories, + "accessory_search_margin_ratio": args.accessory_search_margin_ratio, + "accessory_saturation_threshold": args.accessory_saturation_threshold, + "accessory_contact_passes": args.accessory_contact_passes, + "accessory_max_area_growth": args.accessory_max_area_growth, + "accessory_min_component_area_ratio": args.accessory_min_component_area_ratio, + "accessory_max_components": args.accessory_max_components, + "accessory_pixels_added": sum(int(entry.get("accessory_pixels_added", 0)) for entry in refined), + } + write_json(args.output, {"instances": refined, "metadata": metadata}) + return 0 + + +def refine_instance_entries( + entries: list[dict[str, Any]], + base_dir: Path, + *, + image: np.ndarray | None = None, + merge_saturated_accessories: bool = False, + mask_iou_threshold: float = 0.75, + inclusion_threshold: float = 0.92, + boundary_overlap_threshold: float = 0.85, + accessory_search_margin_ratio: float = 0.25, + accessory_saturation_threshold: float = 0.45, + accessory_contact_passes: int = 4, + accessory_max_area_growth: float = 0.15, + accessory_min_component_area_ratio: float = 0.001, + accessory_max_components: int = 8, +) -> list[dict[str, Any]]: + prepared = [] + for index, entry in enumerate(entries): + mask_path = entry.get("mask_path", entry.get("mask")) + if not mask_path: + continue + path = Path(str(mask_path)) + path = path if path.is_absolute() else (base_dir / path).resolve() + if not path.exists(): + continue + mask = read_gray(path) > 0.5 + if merge_saturated_accessories and image is not None and image.shape[:2] == mask.shape: + mask, merge_event = merge_saturated_accessories_into_mask( + mask, + image, + bbox_xyxy=entry.get("bbox_xyxy"), + search_margin_ratio=accessory_search_margin_ratio, + saturation_threshold=accessory_saturation_threshold, + contact_passes=accessory_contact_passes, + max_area_growth=accessory_max_area_growth, + min_component_area_ratio=accessory_min_component_area_ratio, + max_components=accessory_max_components, + ) + if merge_event["added_pixels"] > 0: + write_gray(path, mask.astype(np.float32)) + entry = dict(entry) + entry["accessory_pixels_added"] = merge_event["added_pixels"] + entry["accessory_components_added"] = merge_event["components_added"] + score = composite_score(entry) + item = dict(entry) + item["composite_score"] = score + item["_input_index"] = index + prepared.append((item, mask)) + + prepared.sort(key=lambda pair: (-float(pair[0]["composite_score"]), int(pair[0]["_input_index"]))) + kept: list[tuple[dict[str, Any], np.ndarray]] = [] + for item, mask in prepared: + duplicate = False + for kept_item, kept_mask in kept: + if is_visual_duplicate( + mask, + kept_mask, + mask_iou_threshold=mask_iou_threshold, + inclusion_threshold=inclusion_threshold, + boundary_overlap_threshold=boundary_overlap_threshold, + ): + duplicate = True + break + if not duplicate: + kept.append((item, mask)) + + kept.sort(key=lambda pair: int(pair[0]["_input_index"])) + output = [] + for idx, (item, _) in enumerate(kept): + cleaned = dict(item) + cleaned.pop("_input_index", None) + cleaned["id"] = str(cleaned.get("id", cleaned.get("instance_id", f"inst_{idx:03d}"))) + output.append(cleaned) + return output + + +def composite_score(entry: dict[str, Any]) -> float: + detection = entry.get("detection_score", entry.get("score", 0.0)) + segmentation = entry.get("segmentation_score", entry.get("mask_score", detection)) + detection = 0.0 if detection is None else float(detection) + segmentation = 0.0 if segmentation is None else float(segmentation) + return 0.5 * detection + 0.5 * segmentation + + +def is_visual_duplicate( + mask: np.ndarray, + reference: np.ndarray, + *, + mask_iou_threshold: float, + inclusion_threshold: float, + boundary_overlap_threshold: float, +) -> bool: + if mask.shape != reference.shape: + return False + intersection = int((mask & reference).sum()) + area = int(mask.sum()) + reference_area = int(reference.sum()) + if area == 0 or reference_area == 0: + return False + union = area + reference_area - intersection + if union > 0 and intersection / union >= mask_iou_threshold: + return True + smaller = mask if area <= reference_area else reference + larger = reference if area <= reference_area else mask + inclusion = int((smaller & larger).sum()) / float(int(smaller.sum())) + boundary_overlap = int((boundary(smaller) & larger).sum()) / max(1, int(boundary(smaller).sum())) + return inclusion >= inclusion_threshold and boundary_overlap >= boundary_overlap_threshold + + +def boundary(mask: np.ndarray) -> np.ndarray: + eroded = mask.copy() + padded = np.pad(mask, 1, mode="constant", constant_values=False) + eroded &= padded[:-2, 1:-1] + eroded &= padded[2:, 1:-1] + eroded &= padded[1:-1, :-2] + eroded &= padded[1:-1, 2:] + return mask & ~eroded + + +def merge_saturated_accessories_into_mask( + mask: np.ndarray, + image: np.ndarray, + *, + bbox_xyxy: Any | None = None, + search_margin_ratio: float = 0.25, + saturation_threshold: float = 0.45, + contact_passes: int = 4, + max_area_growth: float = 0.15, + min_component_area_ratio: float = 0.001, + max_components: int = 8, +) -> tuple[np.ndarray, dict[str, int]]: + base = np.asarray(mask, dtype=bool) + if not base.any(): + return base, {"added_pixels": 0, "components_added": 0} + + search = expanded_search_region(base, bbox_xyxy, margin_ratio=search_margin_ratio) + rgb = np.asarray(image, dtype=np.float32) + if rgb.max(initial=0.0) > 1.5: + rgb = rgb / 255.0 + saturation = rgb_saturation(rgb) + candidates = (saturation >= saturation_threshold) & search & ~base + if not candidates.any(): + return base, {"added_pixels": 0, "components_added": 0} + + contact = dilate(base, max(0, int(contact_passes))) + max_added = max(1, int(base.sum() * max(0.0, max_area_growth))) + min_component_area = max(1, int(np.ceil(base.sum() * max(0.0, min_component_area_ratio)))) + merged = base.copy() + added_pixels = 0 + components_added = 0 + components = connected_components(candidates) + components.sort(key=lambda comp: int(comp.sum()), reverse=True) + for component in components: + if max_components > 0 and components_added >= max_components: + break + area = int(component.sum()) + if area < min_component_area or added_pixels + area > max_added: + continue + if not (component & contact).any(): + continue + merged |= component + contact = dilate(merged, max(0, int(contact_passes))) + added_pixels += area + components_added += 1 + return merged, {"added_pixels": added_pixels, "components_added": components_added} + + +def expanded_search_region(mask: np.ndarray, bbox_xyxy: Any | None, *, margin_ratio: float) -> np.ndarray: + h, w = mask.shape + y0, y1, x0, x1 = mask_bounds(mask) + if bbox_xyxy is not None: + try: + bx0, by0, bx1, by1 = [float(v) for v in bbox_xyxy] + x0 = min(x0, int(np.floor(bx0))) + y0 = min(y0, int(np.floor(by0))) + x1 = max(x1, int(np.ceil(bx1))) + y1 = max(y1, int(np.ceil(by1))) + except (TypeError, ValueError): + pass + box_w = max(1, x1 - x0 + 1) + box_h = max(1, y1 - y0 + 1) + margin_x = max(1, int(round(box_w * max(0.0, margin_ratio)))) + margin_y = max(1, int(round(box_h * max(0.0, margin_ratio)))) + x0 = max(0, x0 - margin_x) + x1 = min(w - 1, x1 + margin_x) + y0 = max(0, y0 - margin_y) + y1 = min(h - 1, y1 + margin_y) + region = np.zeros((h, w), dtype=bool) + region[y0 : y1 + 1, x0 : x1 + 1] = True + return region + + +def mask_bounds(mask: np.ndarray) -> tuple[int, int, int, int]: + ys, xs = np.where(mask) + if ys.size == 0: + return 0, 0, 0, 0 + return int(ys.min()), int(ys.max()), int(xs.min()), int(xs.max()) + + +def rgb_saturation(rgb: np.ndarray) -> np.ndarray: + clipped = np.clip(rgb[:, :, :3], 0.0, 1.0) + maxc = clipped.max(axis=2) + minc = clipped.min(axis=2) + return (maxc - minc) / np.maximum(maxc, 1e-6) + + +def connected_components(mask: np.ndarray) -> list[np.ndarray]: + h, w = mask.shape + visited = np.zeros_like(mask, dtype=bool) + components: list[np.ndarray] = [] + ys, xs = np.where(mask) + for sy, sx in zip(ys.tolist(), xs.tolist()): + if visited[sy, sx] or not mask[sy, sx]: + continue + stack = [(sy, sx)] + visited[sy, sx] = True + pixels = [] + while stack: + y, x = stack.pop() + pixels.append((y, x)) + for ny in range(y - 1, y + 2): + for nx in range(x - 1, x + 2): + if ny == y and nx == x: + continue + if 0 <= ny < h and 0 <= nx < w and mask[ny, nx] and not visited[ny, nx]: + visited[ny, nx] = True + stack.append((ny, nx)) + component = np.zeros_like(mask, dtype=bool) + for y, x in pixels: + component[y, x] = True + components.append(component) + return components + + +def dilate(mask: np.ndarray, passes: int) -> np.ndarray: + out = np.asarray(mask, dtype=bool).copy() + for _ in range(max(0, passes)): + padded = np.pad(out, 1, mode="constant", constant_values=False) + expanded = out.copy() + for dy in range(3): + for dx in range(3): + expanded |= padded[dy : dy + out.shape[0], dx : dx + out.shape[1]] + out = expanded + return out + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/lama_inpaint.py b/vendor/gs_playground/scripts/real2sim_external/lama_inpaint.py new file mode 100644 index 0000000..edf643c --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/lama_inpaint.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Sequential LaMa inpainting adapter for Real2Sim. + +Preferred runtime is the lightweight simple-lama-inpainting package, which is +listed by the official LaMa repository as a third-party app. This adapter keeps +the Real2Sim contract stable: image + instances.json -> background image. +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.commands import format_command +from real2sim.contracts import load_instances, write_json +from real2sim.image_io import read_gray, read_rgb, write_gray, write_rgb + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run LaMa object removal from Real2Sim instance masks") + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--instances", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--dilate", type=int, default=7) + parser.add_argument("--prompt", default="object") + parser.add_argument("--work-dir", type=Path, default=None) + parser.add_argument("--redetect-command", default="") + parser.add_argument("--segment-command", default="") + parser.add_argument("--redetect-command-env", default="") + parser.add_argument("--segment-command-env", default="") + parser.add_argument("--require-redetect-loop", action="store_true") + parser.add_argument("--max-recovery-passes", type=int, default=1) + parser.add_argument("--max-area-growth", type=float, default=0.5) + args = parser.parse_args() + + instances = load_instances(args.instances) + image = read_rgb(args.image) + redetect_command = args.redetect_command or (os.environ.get(args.redetect_command_env, "") if args.redetect_command_env else "") + segment_command = args.segment_command or (os.environ.get(args.segment_command_env, "") if args.segment_command_env else "") + require_recovery_commands(redetect_command, segment_command, strict=args.require_redetect_loop) + + try: + from PIL import Image + from simple_lama_inpainting import SimpleLama + except Exception as exc: + raise SystemExit( + "simple-lama-inpainting is not installed. Install it in the external " + "model environment or use the built-in backend." + ) from exc + + lama = SimpleLama() + current = image + steps = [] + recovery_events = [] + work_dir = args.work_dir or args.output.parent + instance_masks = { + instance.instance_id: read_gray(instance.mask_path) > 0.5 + for instance in instances + if instance.mask_path and instance.mask_path.exists() + } + ordered = sorted( + [instance for instance in instances if instance.mask_path], + key=lambda inst: (-(inst.score if inst.score is not None else 0.0), inst.instance_id), + ) + for instance in ordered: + mask = read_gray(instance.mask_path) > 0.5 + if mask.shape != current.shape[:2]: + continue + expanded = dilate(mask, args.dilate) + pil_image = Image.fromarray((np.clip(current, 0, 1) * 255).astype(np.uint8)) + pil_mask = Image.fromarray((expanded.astype(np.uint8) * 255), mode="L") + result = lama(pil_image, pil_mask) + current = np.asarray(result.convert("RGB"), dtype=np.float32) / 255.0 + if redetect_command and segment_command: + recovery_events.extend( + recover_occluded_regions( + current, + instances=instances, + instance_masks=instance_masks, + instances_json=args.instances, + prompt=args.prompt, + work_dir=work_dir, + pass_index=len(steps), + redetect_command=redetect_command, + segment_command=segment_command, + max_area_growth=args.max_area_growth, + ) + ) + steps.append( + { + "instance_id": instance.instance_id, + "label": instance.label, + "mask_pixels": int(mask.sum()), + "expanded_mask_pixels": int(expanded.sum()), + } + ) + write_rgb(args.output, current) + write_json( + args.output.with_suffix(args.output.suffix + ".json"), + { + "backend": "simple_lama_sequential", + "strategy": "one_object_at_a_time_mask_expansion", + "steps": steps, + "recovery_events": recovery_events, + "redetect_loop": bool(redetect_command and segment_command), + }, + ) + return 0 + + +def require_recovery_commands(redetect_command: str, segment_command: str, *, strict: bool) -> None: + if strict and (not redetect_command or not segment_command): + raise SystemExit( + "paper-strict sequential inpainting requires both a re-detection command " + "and a segmentation command for newly exposed region recovery" + ) + + +def recover_occluded_regions( + image: np.ndarray, + *, + instances, + instance_masks: dict[str, np.ndarray], + instances_json: Path, + prompt: str, + work_dir: Path, + pass_index: int, + redetect_command, + segment_command, + max_area_growth: float, +) -> list[dict]: + recovery_dir = work_dir / "03_inpainting" / "recovery" / f"pass_{pass_index:03d}" + recovery_dir.mkdir(parents=True, exist_ok=True) + image_path = recovery_dir / "image.ppm" + detections_json = recovery_dir / "detections.json" + masks_dir = recovery_dir / "masks" + recovered_instances_json = recovery_dir / "instances.json" + write_rgb(image_path, image) + + base_context = { + "python": sys.executable, + "image": image_path, + "input": image_path, + "prompt": prompt, + "detections_json": detections_json, + "instances_json": recovered_instances_json, + "masks_dir": masks_dir, + } + run_nested_command(redetect_command, base_context) + run_nested_command(segment_command, base_context) + + existing = {instance.instance_id: instance for instance in instances} + events = [] + for candidate in load_instances(recovered_instances_json): + if candidate.mask_path is None or not candidate.mask_path.exists(): + continue + candidate_mask = read_gray(candidate.mask_path) > 0.5 + for instance in instances: + if instance.label != candidate.label or instance.mask_path is None: + continue + base_mask = instance_masks.get(instance.instance_id) + if base_mask is None: + continue + merged = merge_recovered_mask(base_mask, candidate_mask, same_label=True, max_area_growth=max_area_growth) + if int(merged.sum()) == int(base_mask.sum()): + continue + write_gray(instance.mask_path, merged.astype(np.float32)) + instance_masks[instance.instance_id] = merged + events.append( + { + "target_instance_id": instance.instance_id, + "candidate_instance_id": candidate.instance_id, + "label": instance.label, + "old_mask_pixels": int(base_mask.sum()), + "new_mask_pixels": int(merged.sum()), + } + ) + break + return events + + +def run_nested_command(command, context: dict) -> None: + argv = format_command(command, context) + completed = subprocess.run(argv, cwd=REPO_ROOT, check=False) + if completed.returncode != 0: + raise SystemExit(f"nested recovery command failed with exit code {completed.returncode}: {argv}") + + +def merge_recovered_mask(base: np.ndarray, candidate: np.ndarray, *, same_label: bool, max_area_growth: float) -> np.ndarray: + if not same_label or base.shape != candidate.shape: + return base + new_region = candidate & ~base + if not new_region.any(): + return base + if int(new_region.sum()) > int(base.sum()) * max_area_growth: + return base + if not (dilate(base, 1) & new_region).any(): + return base + return base | new_region + + +def dilate(mask: np.ndarray, passes: int) -> np.ndarray: + out = mask.copy() + for _ in range(max(0, passes)): + padded = np.pad(out, 1, mode="constant", constant_values=False) + out = ( + padded[1:-1, 1:-1] + | padded[:-2, 1:-1] + | padded[2:, 1:-1] + | padded[1:-1, :-2] + | padded[1:-1, 2:] + ) + return out + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/sam2_segment.py b/vendor/gs_playground/scripts/real2sim_external/sam2_segment.py new file mode 100644 index 0000000..7b9bc74 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/sam2_segment.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""SAM2 segmentation adapter for Real2Sim. + +Consumes detections.json boxes and writes instances.json plus per-instance PGM +masks. This uses Meta's official SAM2 image predictor API. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.contracts import read_json, write_json +from real2sim.image_io import write_gray + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run SAM2 masks from GroundingDINO boxes") + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--detections", type=Path, required=True) + parser.add_argument("--masks-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--checkpoint", default=os.environ.get("REAL2SIM_SAM2_CHECKPOINT", "")) + parser.add_argument("--model-cfg", default=os.environ.get("REAL2SIM_SAM2_MODEL_CFG", "configs/sam2.1/sam2.1_hiera_l.yaml")) + parser.add_argument("--device", default="auto", choices=["auto", "cuda", "cpu"]) + args = parser.parse_args() + + if not args.checkpoint: + raise SystemExit("SAM2 checkpoint is required via --checkpoint or REAL2SIM_SAM2_CHECKPOINT") + + import numpy as np + import torch + from PIL import Image + from sam2.build_sam import build_sam2 + from sam2.sam2_image_predictor import SAM2ImagePredictor + + detections_raw = read_json(args.detections) + detections = detections_raw.get("detections", detections_raw) if isinstance(detections_raw, dict) else detections_raw + if not detections: + write_json(args.output, {"instances": [], "metadata": {"backend": "sam2"}}) + return 0 + + boxes = np.array([entry.get("bbox_xyxy", entry.get("bbox")) for entry in detections], dtype=np.float32) + device = "cuda" if args.device == "auto" and torch.cuda.is_available() else args.device + if device == "auto": + device = "cpu" + + image = np.array(Image.open(args.image).convert("RGB")) + model = build_sam2(args.model_cfg, args.checkpoint, device=device) + predictor = SAM2ImagePredictor(model) + args.masks_dir.mkdir(parents=True, exist_ok=True) + + with torch.inference_mode(): + if device == "cuda": + with torch.autocast("cuda", dtype=torch.bfloat16): + masks, scores, _ = predict_masks(predictor, image, boxes) + else: + masks, scores, _ = predict_masks(predictor, image, boxes) + + masks = normalize_masks(masks) + instances = [] + for idx, (detection, mask) in enumerate(zip(detections, masks)): + instance_id = str(detection.get("id", f"det_{idx:03d}")) + mask_path = args.masks_dir / f"{safe_name(instance_id)}.pgm" + write_gray(mask_path, mask.astype(np.float32)) + segmentation_score = None + if scores is not None and idx < len(scores): + segmentation_score = float(np.asarray(scores[idx]).reshape(-1)[0]) + instances.append( + instance_record( + detection, + mask_path=mask_path, + output_parent=args.output.parent, + segmentation_score=segmentation_score, + ) + ) + + write_json(args.output, {"instances": instances, "metadata": {"backend": "sam2", "model_cfg": args.model_cfg}}) + return 0 + + +def predict_masks(predictor, image, boxes): + predictor.set_image(image) + return predictor.predict(point_coords=None, point_labels=None, box=boxes, multimask_output=False) + + +def normalize_masks(masks): + import numpy as np + + masks = np.asarray(masks) + if masks.ndim == 4: + masks = masks[:, 0] + if masks.ndim == 2: + masks = masks[None] + return masks > 0 + + +def instance_record(detection: dict, *, mask_path: Path, output_parent: Path, segmentation_score: float | None) -> dict: + instance_id = str(detection.get("id", "instance")) + detection_score = detection.get("score") + score = float(detection_score) if detection_score is not None else segmentation_score + record = { + "id": instance_id, + "label": str(detection.get("label", instance_id)), + "score": score, + "bbox_xyxy": detection.get("bbox_xyxy", detection.get("bbox")), + "mask_path": mask_path.relative_to(output_parent).as_posix(), + } + if detection_score is not None: + record["detection_score"] = float(detection_score) + if segmentation_score is not None: + record["segmentation_score"] = float(segmentation_score) + for key in ("prompt_index", "prompt_query"): + if key in detection: + record[key] = detection[key] + return record + + +def safe_name(value: str) -> str: + return "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in value).strip("_") or "instance" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/sam3d_objects_reconstruct.py b/vendor/gs_playground/scripts/real2sim_external/sam3d_objects_reconstruct.py new file mode 100644 index 0000000..315cb1b --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/sam3d_objects_reconstruct.py @@ -0,0 +1,897 @@ +#!/usr/bin/env python3 +"""SAM 3D Objects adapter for one Real2Sim object. + +This follows Meta's public quick-start shape: + from inference import Inference, load_image, load_single_mask + output = inference(image, mask) + output["gs"].save_ply("splat.ply") + +The script intentionally fails with a clear message if the gated checkpoints +or SAM 3D environment are not available. +""" + +from __future__ import annotations + +import argparse +import importlib +import inspect +import json +import os +import struct +import sys +from pathlib import Path + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.contracts import Pose, read_json, write_json +from real2sim.image_io import read_gray, write_gray + + +class PreparedSam3DInputs: + def __init__(self, image_path: Path, mask_path: Path, mask: np.ndarray, image_size: tuple[int, int], metadata: dict): + self.image_path = image_path + self.mask_path = mask_path + self.mask = mask + self.image_size = image_size + self.metadata = metadata + + +class PlyVertexTable: + def __init__( + self, + *, + header_lines: list[str], + body_start: int, + fmt: str, + vertex_count: int, + property_names: list[str], + property_types: list[str], + array: np.ndarray, + ): + self.header_lines = header_lines + self.body_start = body_start + self.fmt = fmt + self.vertex_count = vertex_count + self.property_names = property_names + self.property_types = property_types + self.array = array + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run SAM 3D Objects for one detected object") + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--mask", type=Path, required=True) + parser.add_argument("--instance", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--ply", type=Path, required=True) + parser.add_argument("--mesh", type=Path, required=True) + parser.add_argument("--pose", type=Path, required=True) + parser.add_argument("--cameras", type=Path, default=None, help="AnySplat cameras JSON used to match image preprocessing") + parser.add_argument("--match-camera-preprocess", action="store_true", help="Run SAM-3D on the same aspect-preserving center crop used by AnySplat") + parser.add_argument("--object-centric-preprocess", action="store_true", help="Crop around the instance mask before SAM-3D preprocessing") + parser.add_argument("--object-preprocess-size", type=int, default=0, help="Override SAM-3D object input size with a square image") + parser.add_argument("--object-crop-margin", type=float, default=0.12, help="Relative padding around the instance mask for object-centric preprocessing") + parser.add_argument("--remove-video-overlay", action="store_true", help="Inpaint a central video play overlay before SAM-3D inference") + parser.add_argument("--canonicalize-gaussian-to-mesh", action="store_true", help="Rotate SAM-3D Gaussian PLY into the mesh canonical frame") + parser.add_argument("--texture-baking", action="store_true", help="Use SAM-3D's textured GLB postprocess path for the exported preview mesh") + parser.add_argument("--texture-render-backend", choices=("auto", "inria", "gsplat"), default="gsplat") + parser.add_argument("--texture-baking-engine", choices=("nvdiffrast", "pytorch3d"), default="pytorch3d") + parser.add_argument("--sam3d-root", type=Path, default=os.environ.get("REAL2SIM_SAM3D_ROOT")) + parser.add_argument("--config", type=Path, default=os.environ.get("REAL2SIM_SAM3D_CONFIG")) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--allow-fallback-mesh", action="store_true", help="Write a collision proxy if SAM 3D returns no exportable mesh") + args = parser.parse_args() + + if args.sam3d_root is None: + raise SystemExit("SAM 3D root is required via --sam3d-root or REAL2SIM_SAM3D_ROOT") + if args.config is None: + args.config = args.sam3d_root / "checkpoints" / "hf" / "pipeline.yaml" + if not args.config.exists(): + raise SystemExit(f"SAM 3D config/checkpoints are not available: {args.config}") + + sys.path.insert(0, (args.sam3d_root / "notebook").as_posix()) + try: + from inference import Inference, load_image + except Exception as exc: + raise SystemExit(f"failed to import SAM 3D inference from {args.sam3d_root}: {exc}") from exc + + import torch + + args.output_dir.mkdir(parents=True, exist_ok=True) + image_path = args.image + mask_np = (read_gray(args.mask) > 0.5).astype(np.uint8) + preprocess_metadata = None + overlay_metadata = None + if args.match_camera_preprocess: + if args.cameras is None: + raise SystemExit("--match-camera-preprocess requires --cameras") + target_shape = None + if args.object_preprocess_size > 0: + target_shape = (int(args.object_preprocess_size), int(args.object_preprocess_size)) + prepared = prepare_camera_preprocessed_inputs( + args.image, + args.mask, + args.cameras, + args.output_dir / "camera_preprocess", + object_centric=args.object_centric_preprocess, + target_shape=target_shape, + object_margin_ratio=args.object_crop_margin, + ) + image_path = prepared.image_path + mask_np = prepared.mask.astype(np.uint8) + preprocess_metadata = prepared.metadata + if args.remove_video_overlay: + image_path, overlay_metadata = remove_center_video_overlay(image_path, mask_np > 0, args.output_dir / "video_overlay_removal") + image = load_image(image_path.as_posix()) + inference = Inference(args.config.as_posix(), compile=False) + with torch.inference_mode(): + output = run_sam3d_inference( + inference, + image, + mask_np, + seed=args.seed, + texture_baking=args.texture_baking, + texture_render_backend=args.texture_render_backend, + texture_baking_engine=args.texture_baking_engine, + ) + + save_gaussian(output, args.ply) + + wrote_mesh = export_mesh(output, args.mesh, prefer_textured=args.texture_baking) + if not wrote_mesh and args.allow_fallback_mesh: + write_bbox_mesh_from_mask(args.mesh, mask_np) + wrote_mesh = True + if not wrote_mesh: + raise SystemExit( + "SAM 3D output did not include an exportable mesh. " + "Use --allow-fallback-mesh only for collision-proxy debugging. " + f"Output keys: {sorted(output.keys())}" + ) + canonicalization_metadata = None + if args.canonicalize_gaussian_to_mesh: + canonicalization_metadata = canonicalize_gaussian_to_mesh_frame(args.ply, args.mesh) + + pose = Pose() + layout = output.get("layout") or output.get("pose") or output + if isinstance(layout, dict): + pose = pose_from_layout(layout) + write_json(args.pose, pose.to_json()) + + metadata = { + "backend": "sam3d_objects", + "source_instance": read_json(args.instance), + "sam3d_config": args.config.as_posix(), + "texture_baking": bool(args.texture_baking), + "texture_render_backend": args.texture_render_backend, + "texture_baking_engine": args.texture_baking_engine, + } + if preprocess_metadata is not None: + metadata["camera_preprocess"] = preprocess_metadata + if overlay_metadata is not None: + metadata["video_overlay_removal"] = overlay_metadata + if canonicalization_metadata is not None: + metadata["gaussian_canonicalization"] = canonicalization_metadata + write_json(args.output_dir / "sam3d_metadata.json", metadata) + return 0 + + +def prepare_camera_preprocessed_inputs( + image_path: Path, + mask_path: Path, + cameras_path: Path, + output_dir: Path, + *, + object_centric: bool = False, + target_shape: tuple[int, int] | None = None, + object_margin_ratio: float = 0.12, +) -> PreparedSam3DInputs: + cameras = read_json(cameras_path) + target_h, target_w = target_shape or camera_image_shape(cameras) + from PIL import Image + + image = Image.open(image_path).convert("RGB") + mask = read_gray(mask_path) > 0.5 + if object_centric and mask.any(): + processed_image, processed_mask, crop_meta = object_centric_resize_pad_image_and_mask( + image, + mask, + (target_h, target_w), + margin_ratio=object_margin_ratio, + resample=Image.BICUBIC, + ) + else: + processed_image, crop_meta = cover_resize_center_crop_image(image, (target_h, target_w), resample=Image.BICUBIC) + processed_mask = cover_resize_center_crop_mask(mask, (target_h, target_w), crop_meta) + crop_meta = {"preprocess_mode": "camera_center_crop", **crop_meta} + + output_dir.mkdir(parents=True, exist_ok=True) + processed_image_path = output_dir / "sam3d_camera_image.png" + processed_mask_path = output_dir / "sam3d_camera_mask.pgm" + processed_image.save(processed_image_path) + write_gray(processed_mask_path, processed_mask.astype(np.float32)) + + metadata = { + "source_image": image_path.as_posix(), + "source_mask": mask_path.as_posix(), + "cameras": cameras_path.as_posix(), + "source_size": [int(image.height), int(image.width)], + "target_size": [int(target_h), int(target_w)], + **crop_meta, + "image": processed_image_path.as_posix(), + "mask": processed_mask_path.as_posix(), + "mask_pixels": int(processed_mask.sum()), + } + return PreparedSam3DInputs( + image_path=processed_image_path, + mask_path=processed_mask_path, + mask=processed_mask, + image_size=(target_h, target_w), + metadata=metadata, + ) + + +def camera_image_shape(cameras: dict) -> tuple[int, int]: + shape = cameras.get("image_shape") if isinstance(cameras, dict) else None + if isinstance(shape, (list, tuple)) and len(shape) >= 2: + return (int(shape[0]), int(shape[1])) + intrinsics = cameras.get("intrinsics", {}) if isinstance(cameras, dict) else {} + height = int(intrinsics.get("height", 448)) + width = int(intrinsics.get("width", 448)) + return (height, width) + + +def cover_resize_center_crop_image(image, shape: tuple[int, int], *, resample) -> tuple: + dst_h, dst_w = shape + src_w, src_h = image.size + scale = max(dst_w / float(src_w), dst_h / float(src_h)) + resized_w = max(dst_w, int(src_w * scale)) + resized_h = max(dst_h, int(src_h * scale)) + resized = image.resize((resized_w, resized_h), resample) + left = max(0, (resized_w - dst_w) // 2) + top = max(0, (resized_h - dst_h) // 2) + crop = resized.crop((left, top, left + dst_w, top + dst_h)) + return crop, { + "resized_size": [int(resized_h), int(resized_w)], + "crop_xyxy": [int(left), int(top), int(left + dst_w), int(top + dst_h)], + "scale": float(scale), + } + + +def cover_resize_center_crop_mask(mask: np.ndarray, shape: tuple[int, int], crop_meta: dict) -> np.ndarray: + from PIL import Image + + dst_h, dst_w = shape + resized_h, resized_w = crop_meta["resized_size"] + left, top, right, bottom = crop_meta["crop_xyxy"] + mask_img = Image.fromarray(mask.astype(np.uint8) * 255) + resized = mask_img.resize((int(resized_w), int(resized_h)), Image.NEAREST) + cropped = resized.crop((int(left), int(top), int(right), int(bottom))) + arr = np.asarray(cropped, dtype=np.uint8) > 127 + if arr.shape == (dst_h, dst_w): + return arr + out = np.zeros((dst_h, dst_w), dtype=bool) + out[: arr.shape[0], : arr.shape[1]] = arr + return out + + +def object_centric_resize_pad_image_and_mask(image, mask: np.ndarray, shape: tuple[int, int], *, margin_ratio: float, resample) -> tuple: + from PIL import Image + + dst_h, dst_w = shape + src_w, src_h = image.size + x0, y0, x1, y1 = mask_bbox_xyxy(mask) + box_w = max(1, x1 - x0) + box_h = max(1, y1 - y0) + margin = max(1, int(round(max(box_w, box_h) * max(float(margin_ratio), 0.0)))) + crop_x0 = max(0, x0 - margin) + crop_y0 = max(0, y0 - margin) + crop_x1 = min(src_w, x1 + margin) + crop_y1 = min(src_h, y1 + margin) + if crop_x1 <= crop_x0 or crop_y1 <= crop_y0: + processed_image, crop_meta = cover_resize_center_crop_image(image, shape, resample=resample) + processed_mask = cover_resize_center_crop_mask(mask, shape, crop_meta) + return processed_image, processed_mask, {"preprocess_mode": "camera_center_crop_empty_object", **crop_meta} + + crop = image.crop((crop_x0, crop_y0, crop_x1, crop_y1)) + crop_mask = mask[crop_y0:crop_y1, crop_x0:crop_x1] + crop_w, crop_h = crop.size + scale = min(dst_w / float(crop_w), dst_h / float(crop_h)) + resized_w = max(1, min(dst_w, int(round(crop_w * scale)))) + resized_h = max(1, min(dst_h, int(round(crop_h * scale)))) + offset_x = (dst_w - resized_w) // 2 + offset_y = (dst_h - resized_h) // 2 + + fill = local_background_color(crop, crop_mask) + canvas = Image.new("RGB", (dst_w, dst_h), fill) + resized = crop.resize((resized_w, resized_h), resample) + canvas.paste(resized, (offset_x, offset_y)) + + mask_canvas = np.zeros((dst_h, dst_w), dtype=bool) + resized_mask = Image.fromarray(crop_mask.astype(np.uint8) * 255).resize((resized_w, resized_h), Image.NEAREST) + mask_arr = np.asarray(resized_mask, dtype=np.uint8) > 127 + mask_canvas[offset_y : offset_y + resized_h, offset_x : offset_x + resized_w] = mask_arr + return canvas, mask_canvas, { + "preprocess_mode": "object_centric", + "object_bbox_xyxy": [int(x0), int(y0), int(x1), int(y1)], + "source_crop_xyxy": [int(crop_x0), int(crop_y0), int(crop_x1), int(crop_y1)], + "resized_size": [int(resized_h), int(resized_w)], + "pad_xy": [int(offset_x), int(offset_y)], + "scale": float(scale), + "object_margin_ratio": float(margin_ratio), + } + + +def mask_bbox_xyxy(mask: np.ndarray) -> tuple[int, int, int, int]: + ys, xs = np.where(np.asarray(mask, dtype=bool)) + if len(xs) == 0: + return (0, 0, int(mask.shape[1]), int(mask.shape[0])) + return (int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1) + + +def local_background_color(image, mask: np.ndarray) -> tuple[int, int, int]: + arr = np.asarray(image.convert("RGB"), dtype=np.uint8) + bg = arr[~np.asarray(mask, dtype=bool)] + values = bg if bg.size else arr.reshape(-1, 3) + color = np.median(values.reshape(-1, 3), axis=0) + return tuple(int(np.clip(round(float(channel)), 0, 255)) for channel in color) + + +def remove_center_video_overlay(image_path: Path, object_mask: np.ndarray, output_dir: Path) -> tuple[Path, dict]: + from PIL import Image + + image = Image.open(image_path).convert("RGB") + arr = np.asarray(image, dtype=np.uint8) + mask = np.asarray(object_mask, dtype=bool) + if mask.shape != arr.shape[:2]: + mask = resize_bool_mask(mask, arr.shape[:2]) + + overlay_mask = detect_center_video_overlay(arr, mask) + overlay_pixels = int(overlay_mask.sum()) + metadata = { + "source_image": image_path.as_posix(), + "detected": overlay_pixels > 0, + "overlay_pixels": overlay_pixels, + } + if overlay_pixels == 0: + return image_path, metadata + + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / "sam3d_overlay_removed.png" + try: + import cv2 + + inpaint_mask = (overlay_mask.astype(np.uint8) * 255) + cleaned = cv2.inpaint(arr, inpaint_mask, 5, cv2.INPAINT_TELEA) + except Exception: + cleaned = simple_mask_fill(arr, overlay_mask) + metadata["fallback"] = "simple_mask_fill" + Image.fromarray(cleaned.astype(np.uint8)).save(output_path) + metadata["image"] = output_path.as_posix() + return output_path, metadata + + +def detect_center_video_overlay(image: np.ndarray, object_mask: np.ndarray) -> np.ndarray: + height, width = image.shape[:2] + yy, xx = np.ogrid[:height, :width] + cx = width * 0.5 + cy = height * 0.5 + radius = max(8.0, min(height, width) * 0.11) + disk = (xx - cx) ** 2 + (yy - cy) ** 2 <= radius ** 2 + rgb = image.astype(np.float32) / 255.0 + max_channel = rgb.max(axis=2) + min_channel = rgb.min(axis=2) + saturation = (max_channel - min_channel) / np.maximum(max_channel, 1e-6) + bright_neutral = (max_channel > 0.72) & (saturation < 0.28) + raw = disk & object_mask & bright_neutral + if int(raw.sum()) < max(20, int(radius)): + return np.zeros((height, width), dtype=bool) + return dilate_bool_mask(raw, radius=3) + + +def resize_bool_mask(mask: np.ndarray, shape: tuple[int, int]) -> np.ndarray: + from PIL import Image + + height, width = shape + resized = Image.fromarray(mask.astype(np.uint8) * 255).resize((width, height), Image.NEAREST) + return np.asarray(resized, dtype=np.uint8) > 127 + + +def dilate_bool_mask(mask: np.ndarray, *, radius: int) -> np.ndarray: + try: + import cv2 + + kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius * 2 + 1, radius * 2 + 1)) + return cv2.dilate(mask.astype(np.uint8), kernel, iterations=1) > 0 + except Exception: + padded = np.pad(mask, radius, mode="constant", constant_values=False) + out = np.zeros_like(mask, dtype=bool) + for dy in range(radius * 2 + 1): + for dx in range(radius * 2 + 1): + if (dx - radius) ** 2 + (dy - radius) ** 2 <= radius ** 2: + out |= padded[dy : dy + mask.shape[0], dx : dx + mask.shape[1]] + return out + + +def simple_mask_fill(image: np.ndarray, mask: np.ndarray) -> np.ndarray: + cleaned = image.copy() + if not mask.any(): + return cleaned + yy, xx = np.where(mask) + y0, y1 = max(0, int(yy.min()) - 5), min(image.shape[0], int(yy.max()) + 6) + x0, x1 = max(0, int(xx.min()) - 5), min(image.shape[1], int(xx.max()) + 6) + patch = image[y0:y1, x0:x1] + patch_mask = mask[y0:y1, x0:x1] + fill_values = patch[~patch_mask] + fill = np.median(fill_values, axis=0) if fill_values.size else np.median(image.reshape(-1, 3), axis=0) + cleaned[mask] = fill.astype(np.uint8) + return cleaned + + +PLY_NUMPY_DTYPES = { + "char": "i1", + "int8": "i1", + "uchar": "u1", + "uint8": "u1", + "short": "i2", + "int16": "i2", + "ushort": "u2", + "uint16": "u2", + "int": "i4", + "int32": "i4", + "uint": "u4", + "uint32": "u4", + "float": "f4", + "float32": "f4", + "double": "f8", + "float64": "f8", +} + + +def canonicalize_gaussian_to_mesh_frame(gaussian_ply: Path, mesh_obj: Path) -> dict: + table = load_ply_vertex_table(gaussian_ply) + mesh_vertices = load_obj_vertices(mesh_obj) + if table.array.size == 0 or mesh_vertices.size == 0: + return { + "enabled": True, + "status": "skipped_empty_geometry", + "gaussian": gaussian_ply.as_posix(), + "mesh": mesh_obj.as_posix(), + } + + xyz_indices = indices_for(table.property_names, ("x", "y", "z")) + points = table.array[:, xyz_indices] + gaussian_extent = extent(points) + mesh_extent = extent(mesh_vertices) + rotation, mapping, signs = canonical_axis_rotation(gaussian_extent, mesh_extent) + changed = not np.allclose(rotation, np.eye(3), atol=1e-6) + if changed: + apply_axis_rotation_to_ply_table(table, rotation) + write_ply_vertex_table(gaussian_ply, table) + + return { + "enabled": True, + "status": "ok", + "changed": bool(changed), + "gaussian": gaussian_ply.as_posix(), + "mesh": mesh_obj.as_posix(), + "gaussian_extent_before": vector_json(gaussian_extent), + "mesh_extent": vector_json(mesh_extent), + "axis_mapping": { + "source": "extent_permutation", + "gaussian_axes_to_mesh_axes": [int(value) for value in mapping], + "mesh_axis_signs": [int(value) for value in signs], + }, + "rotation_matrix": [[round(float(value), 7) for value in row] for row in rotation.tolist()], + } + + +def canonical_axis_rotation(gaussian_extent: np.ndarray, mesh_extent: np.ndarray) -> tuple[np.ndarray, list[int], list[int]]: + gaussian_order = list(np.argsort(-np.asarray(gaussian_extent, dtype=np.float32))) + mesh_order = list(np.argsort(-np.asarray(mesh_extent, dtype=np.float32))) + mapping = [0, 1, 2] + for mesh_axis, gaussian_axis in zip(mesh_order, gaussian_order): + mapping[int(mesh_axis)] = int(gaussian_axis) + signs = [1, 1, 1] + rotation = rotation_from_axis_mapping(mapping, signs) + if np.linalg.det(rotation) < 0.0: + signs[int(mesh_order[-1])] = -1 + rotation = rotation_from_axis_mapping(mapping, signs) + return rotation.astype(np.float32), mapping, signs + + +def rotation_from_axis_mapping(mapping: list[int], signs: list[int]) -> np.ndarray: + rotation = np.zeros((3, 3), dtype=np.float32) + for mesh_axis, gaussian_axis in enumerate(mapping): + rotation[mesh_axis, gaussian_axis] = float(signs[mesh_axis]) + return rotation + + +def apply_axis_rotation_to_ply_table(table: PlyVertexTable, rotation: np.ndarray) -> None: + names = table.property_names + xyz_indices = indices_for(names, ("x", "y", "z")) + table.array[:, xyz_indices] = table.array[:, xyz_indices] @ rotation.T + if all(name in names for name in ("nx", "ny", "nz")): + normal_indices = indices_for(names, ("nx", "ny", "nz")) + table.array[:, normal_indices] = table.array[:, normal_indices] @ rotation.T + if all(name in names for name in ("rot_0", "rot_1", "rot_2", "rot_3")): + rot_indices = indices_for(names, ("rot_0", "rot_1", "rot_2", "rot_3")) + transform_quat = quat_wxyz_from_rotation_matrix(rotation) + for idx in range(table.array.shape[0]): + table.array[idx, rot_indices] = quat_multiply(transform_quat, normalize_quat(table.array[idx, rot_indices])) + + +def load_ply_vertex_table(path: Path) -> PlyVertexTable: + data = path.read_bytes() + header_marker = b"end_header" + marker_index = data.find(header_marker) + if marker_index < 0: + raise SystemExit(f"PLY missing end_header: {path}") + body_start = data.find(b"\n", marker_index) + if body_start < 0: + raise SystemExit(f"PLY header missing final newline: {path}") + body_start += 1 + header_lines = data[:body_start].decode("ascii", errors="ignore").splitlines() + fmt = "ascii" + vertex_count = 0 + property_names: list[str] = [] + property_types: list[str] = [] + current_element = "" + for line in header_lines: + parts = line.strip().split() + if len(parts) >= 3 and parts[0] == "format": + fmt = parts[1] + elif len(parts) >= 3 and parts[0] == "element": + current_element = parts[1] + if current_element == "vertex": + vertex_count = int(parts[2]) + elif len(parts) >= 3 and parts[0] == "property" and current_element == "vertex": + if parts[1] == "list": + raise SystemExit(f"PLY vertex list properties are not supported for canonicalization: {path}") + property_types.append(parts[1]) + property_names.append(parts[-1]) + if vertex_count <= 0: + return PlyVertexTable( + header_lines=header_lines, + body_start=body_start, + fmt=fmt, + vertex_count=0, + property_names=property_names, + property_types=property_types, + array=np.zeros((0, len(property_names)), dtype=np.float32), + ) + if not all(name in property_names for name in ("x", "y", "z")): + raise SystemExit(f"PLY must expose x/y/z vertex properties: {path}") + body = data[body_start:] + if fmt == "ascii": + array = load_ascii_ply_vertex_array(body, vertex_count, len(property_names)) + elif fmt in {"binary_little_endian", "binary_big_endian"}: + array = load_binary_ply_vertex_array(body, vertex_count, property_types, little_endian=fmt == "binary_little_endian") + else: + raise SystemExit(f"unsupported PLY format for canonicalization: {fmt}") + return PlyVertexTable( + header_lines=header_lines, + body_start=body_start, + fmt=fmt, + vertex_count=vertex_count, + property_names=property_names, + property_types=property_types, + array=array.astype(np.float32, copy=False), + ) + + +def load_ascii_ply_vertex_array(body: bytes, vertex_count: int, property_count: int) -> np.ndarray: + rows = [] + for line in body.decode("ascii", errors="ignore").splitlines(): + if len(rows) >= vertex_count: + break + stripped = line.strip() + if not stripped: + continue + values = [float(value) for value in stripped.split()[:property_count]] + if len(values) == property_count: + rows.append(values) + return np.asarray(rows, dtype=np.float32) + + +def load_binary_ply_vertex_array(body: bytes, vertex_count: int, property_types: list[str], *, little_endian: bool) -> np.ndarray: + byte_order = "<" if little_endian else ">" + try: + dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(property_types)]) + except KeyError as exc: + raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc + structured = np.frombuffer(body, dtype=dtype, count=vertex_count) + columns = [structured[f"f{idx}"].astype(np.float32) for idx in range(len(property_types))] + return np.stack(columns, axis=1) if columns else np.zeros((vertex_count, 0), dtype=np.float32) + + +def write_ply_vertex_table(path: Path, table: PlyVertexTable) -> None: + header = "\n".join(table.header_lines) + "\n" + if table.fmt == "ascii": + lines = [" ".join(f"{float(value):.9g}" for value in row) for row in table.array] + path.write_text(header + "\n".join(lines) + "\n", encoding="ascii") + return + if table.fmt in {"binary_little_endian", "binary_big_endian"}: + byte_order = "<" if table.fmt == "binary_little_endian" else ">" + try: + dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(table.property_types)]) + except KeyError as exc: + raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc + structured = np.empty(table.array.shape[0], dtype=dtype) + for idx in range(table.array.shape[1]): + structured[f"f{idx}"] = table.array[:, idx].astype(dtype[idx]) + path.write_bytes(header.encode("ascii") + structured.tobytes()) + return + raise SystemExit(f"unsupported PLY format for canonicalization: {table.fmt}") + + +def load_obj_vertices(path: Path) -> np.ndarray: + vertices = [] + with path.open("r", encoding="utf-8", errors="ignore") as f: + for line in f: + if not line.startswith("v "): + continue + parts = line.split() + if len(parts) >= 4: + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + return np.asarray(vertices, dtype=np.float32) + + +def indices_for(names: list[str], required: tuple[str, ...]) -> list[int]: + return [names.index(name) for name in required] + + +def extent(points: np.ndarray) -> np.ndarray: + if points.size == 0: + return np.zeros(3, dtype=np.float32) + return np.asarray(points.max(axis=0) - points.min(axis=0), dtype=np.float32) + + +def normalize_quat(raw: np.ndarray) -> np.ndarray: + quat = np.asarray(raw, dtype=np.float32).reshape(4) + norm = float(np.linalg.norm(quat)) + if norm <= 1e-8: + return np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32) + return quat / norm + + +def quat_multiply(a: np.ndarray, b: np.ndarray) -> np.ndarray: + aw, ax, ay, az = normalize_quat(a) + bw, bx, by, bz = normalize_quat(b) + return normalize_quat( + np.asarray( + [ + aw * bw - ax * bx - ay * by - az * bz, + aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + ], + dtype=np.float32, + ) + ) + + +def quat_wxyz_from_rotation_matrix(rotation: np.ndarray) -> np.ndarray: + matrix = np.asarray(rotation, dtype=np.float64).reshape(3, 3) + trace = float(np.trace(matrix)) + if trace > 0.0: + s = np.sqrt(trace + 1.0) * 2.0 + quat = np.asarray( + [ + 0.25 * s, + (matrix[2, 1] - matrix[1, 2]) / s, + (matrix[0, 2] - matrix[2, 0]) / s, + (matrix[1, 0] - matrix[0, 1]) / s, + ], + dtype=np.float64, + ) + else: + axis = int(np.argmax(np.diag(matrix))) + if axis == 0: + s = np.sqrt(1.0 + matrix[0, 0] - matrix[1, 1] - matrix[2, 2]) * 2.0 + quat = np.asarray( + [ + (matrix[2, 1] - matrix[1, 2]) / s, + 0.25 * s, + (matrix[0, 1] + matrix[1, 0]) / s, + (matrix[0, 2] + matrix[2, 0]) / s, + ], + dtype=np.float64, + ) + elif axis == 1: + s = np.sqrt(1.0 + matrix[1, 1] - matrix[0, 0] - matrix[2, 2]) * 2.0 + quat = np.asarray( + [ + (matrix[0, 2] - matrix[2, 0]) / s, + (matrix[0, 1] + matrix[1, 0]) / s, + 0.25 * s, + (matrix[1, 2] + matrix[2, 1]) / s, + ], + dtype=np.float64, + ) + else: + s = np.sqrt(1.0 + matrix[2, 2] - matrix[0, 0] - matrix[1, 1]) * 2.0 + quat = np.asarray( + [ + (matrix[1, 0] - matrix[0, 1]) / s, + (matrix[0, 2] + matrix[2, 0]) / s, + (matrix[1, 2] + matrix[2, 1]) / s, + 0.25 * s, + ], + dtype=np.float64, + ) + return normalize_quat(quat.astype(np.float32)) + + +def vector_json(values: np.ndarray) -> list[float]: + return [round(float(value), 7) for value in np.asarray(values, dtype=np.float32).reshape(-1).tolist()] + + +def pose_from_layout(layout: dict) -> Pose: + position = layout.get("position", layout.get("translation", [0.0, 0.0, 0.0])) + scale = layout.get("scale", [1.0, 1.0, 1.0]) + quat = layout.get("quat_wxyz", layout.get("quaternion_wxyz", layout.get("rotation", [1.0, 0.0, 0.0, 0.0]))) + if "quaternion_xyzw" in layout and "quat_wxyz" not in layout and "quaternion_wxyz" not in layout and "rotation" not in layout: + qx, qy, qz, qw = vector_from_value(layout["quaternion_xyzw"], 4, "quaternion_xyzw") + quat = (qw, qx, qy, qz) + return Pose( + position=vector_from_value(position, 3, "position"), + quat_wxyz=vector_from_value(quat, 4, "quat_wxyz"), + scale=vector_from_value(scale, 3, "scale", broadcast_scalar=True), + ) + + +def run_sam3d_inference( + inference, + image: np.ndarray, + mask_np: np.ndarray, + *, + seed: int, + texture_baking: bool, + texture_render_backend: str = "gsplat", + texture_baking_engine: str = "pytorch3d", +) -> dict: + if not texture_baking: + return inference(image, mask_np, seed=seed) + + patch_texture_baking_render_backend(texture_render_backend) + pipeline = getattr(inference, "_pipeline", None) + merge_mask_to_rgba = getattr(inference, "merge_mask_to_rgba", None) + if pipeline is None or not hasattr(pipeline, "run") or merge_mask_to_rgba is None: + raise SystemExit("--texture-baking requires SAM-3D notebook Inference._pipeline.run support") + if hasattr(pipeline, "rendering_engine"): + pipeline.rendering_engine = texture_baking_engine + + rgba = merge_mask_to_rgba(image, mask_np) + options = { + "stage1_only": False, + "with_mesh_postprocess": False, + "with_texture_baking": True, + "with_layout_postprocess": False, + "use_vertex_color": False, + "stage1_inference_steps": None, + "pointmap": None, + } + return pipeline.run(rgba, None, seed, **supported_kwargs(pipeline.run, options)) + + +def patch_texture_baking_render_backend(backend: str) -> bool: + if backend in {"", "auto", "inria"}: + return False + if backend != "gsplat": + raise SystemExit(f"unsupported SAM-3D texture render backend: {backend}") + + render_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.render_utils") + postprocessing_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.postprocessing_utils") + + def render_multiview_with_backend(sample, resolution=512, nviews=30): + radius = 2 + fov = 40 + cameras = [render_utils.sphere_hammersley_sequence(index, nviews) for index in range(nviews)] + yaws = [camera[0] for camera in cameras] + pitches = [camera[1] for camera in cameras] + extrinsics, intrinsics = render_utils.yaw_pitch_r_fov_to_extrinsics_intrinsics(yaws, pitches, radius, fov) + result = render_utils.render_frames( + sample, + extrinsics, + intrinsics, + {"resolution": resolution, "bg_color": (0, 0, 0), "backend": "gsplat"}, + ) + return result["color"], extrinsics, intrinsics + + postprocessing_utils.render_multiview = render_multiview_with_backend + return True + + +def supported_kwargs(callable_obj, options: dict[str, object]) -> dict[str, object]: + signature = inspect.signature(callable_obj) + if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values()): + return options + return {key: value for key, value in options.items() if key in signature.parameters} + + +def save_gaussian(output: dict, path: Path) -> None: + for key in ("gs", "gaussian", "gaussians"): + if key not in output: + continue + value = first_value(output[key]) + if hasattr(value, "save_ply"): + path.parent.mkdir(parents=True, exist_ok=True) + value.save_ply(path.as_posix()) + return + raise SystemExit(f"SAM 3D output does not expose a Gaussian save_ply method; keys={sorted(output.keys())}") + + +def export_mesh(output: dict, path: Path, *, prefer_textured: bool = False) -> bool: + keys = ("glb", "mesh", "trimesh", "obj") if prefer_textured else ("mesh", "trimesh", "glb", "obj") + for key in keys: + if key not in output: + continue + value = first_value(output[key]) + if hasattr(value, "export"): + path.parent.mkdir(parents=True, exist_ok=True) + value.export(path.as_posix()) + return True + if isinstance(value, (str, os.PathLike)): + source = Path(value) + if source.is_file(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(source.read_bytes()) + return True + return False + + +def first_value(value): + if isinstance(value, (list, tuple)): + return value[0] if value else value + return value + + +def vector_from_value(value, size: int, field_name: str, *, broadcast_scalar: bool = False) -> tuple[float, ...]: + if hasattr(value, "detach"): + value = value.detach().cpu().numpy() + arr = np.asarray(value, dtype=np.float64).squeeze() + if arr.ndim == 0: + if broadcast_scalar: + return tuple(round(float(arr), 7) for _ in range(size)) + if size == 1: + return (round(float(arr), 7),) + flat = arr.reshape(-1) + if flat.size != size: + raise SystemExit(f"{field_name} must have {size} values, got shape {arr.shape}") + return tuple(round(float(x), 7) for x in flat) + + +def write_bbox_mesh_from_mask(path: Path, mask) -> None: + import numpy as np + + ys, xs = np.where(mask) + if ys.size == 0: + extent = np.array([0.1, 0.1, 0.1], dtype=np.float32) + else: + extent = np.array([max((xs.max() - xs.min()) / 512.0, 0.05), 0.08, max((ys.max() - ys.min()) / 512.0, 0.05)], dtype=np.float32) + half = extent / 2.0 + vertices = [ + (-half[0], -half[1], -half[2]), + (half[0], -half[1], -half[2]), + (half[0], half[1], -half[2]), + (-half[0], half[1], -half[2]), + (-half[0], -half[1], half[2]), + (half[0], -half[1], half[2]), + (half[0], half[1], half[2]), + (-half[0], half[1], half[2]), + ] + faces = [(1, 2, 3), (1, 3, 4), (5, 8, 7), (5, 7, 6), (1, 5, 6), (1, 6, 2), (2, 6, 7), (2, 7, 3), (3, 7, 8), (3, 8, 4), (4, 8, 5), (4, 5, 1)] + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="ascii") as f: + f.write("o sam3d_collision_fallback\n") + for v in vertices: + f.write(f"v {v[0]:.7g} {v[1]:.7g} {v[2]:.7g}\n") + for face in faces: + f.write(f"f {face[0]} {face[1]} {face[2]}\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/speedy_splat_prune.py b/vendor/gs_playground/scripts/real2sim_external/speedy_splat_prune.py new file mode 100644 index 0000000..205cbb7 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/speedy_splat_prune.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Speedy-Splat pruning adapter for Real2Sim 3DGS assets. + +This adapter bridges Real2Sim's exported PLY assets into the official +Speedy-Splat pruning implementation. It loads the input PLY with +Speedy-Splat's GaussianModel, builds a camera from the AnySplat camera JSON, +calls the official train.prune function, and writes the pruned PLY. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import traceback +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np + + +def main() -> int: + parser = argparse.ArgumentParser(description="Prune a 3DGS PLY with official Speedy-Splat scoring") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--image", type=Path, required=True, help="Source image used to build the pruning camera") + parser.add_argument("--cameras", type=Path, required=True, help="AnySplat cameras.json artifact") + parser.add_argument("--keep-ratio", type=float, default=float(os.environ.get("REAL2SIM_SPEEDY_KEEP_RATIO", "0.85"))) + parser.add_argument("--keep-count", type=int, default=None) + parser.add_argument("--speedy-root", type=Path, default=os.environ.get("REAL2SIM_SPEEDY_SPLAT_ROOT")) + parser.add_argument("--background", choices=("black", "white"), default=os.environ.get("REAL2SIM_SPEEDY_BACKGROUND", "black")) + parser.add_argument("--metadata", type=Path, default=None) + args = parser.parse_args() + + if args.speedy_root is None: + raise SystemExit("Speedy-Splat root is required via --speedy-root or REAL2SIM_SPEEDY_SPLAT_ROOT") + if not args.input.is_file(): + raise SystemExit(f"input PLY not found: {args.input}") + if not args.image.is_file(): + raise SystemExit(f"source image not found: {args.image}") + if not args.cameras.is_file(): + raise SystemExit(f"camera JSON not found: {args.cameras}") + + sys.path.insert(0, args.speedy_root.as_posix()) + + import torch + from PIL import Image + from plyfile import PlyData + from scene.cameras import Camera + from scene.gaussian_model import GaussianModel + from train import prune as speedy_prune + + props = [prop.name for prop in PlyData.read(args.input)["vertex"].properties] + sh_degree = infer_sh_degree(props) + gaussians = GaussianModel(sh_degree) + gaussians.load_ply(args.input.as_posix()) + gaussians.spatial_lr_scale = 1.0 + gaussians.training_setup(speedy_training_args()) + gaussians.max_radii2D = torch.zeros((gaussians.get_xyz.shape[0],), device="cuda") + prune_policy = install_exact_count_pruning(torch, gaussians) + + camera_payload = load_json(args.cameras) + intrinsics = require_intrinsics(camera_payload) + width = int(intrinsics["width"]) + height = int(intrinsics["height"]) + image_tensor = load_image_tensor(Image, torch, args.image, width, height) + view_index = image_view_index(camera_payload, args.image) + extrinsic = first_matrix(camera_payload.get("pred_context_pose", {}).get("extrinsic"), rows=4, cols=4, view_index=view_index) + rotation, translation = speedy_camera_rt(extrinsic) + camera = Camera( + colmap_id=0, + R=rotation, + T=translation, + FoVx=focal_to_fov(float(intrinsics["fx"]), width), + FoVy=focal_to_fov(float(intrinsics["fy"]), height), + image=image_tensor, + gt_alpha_mask=None, + image_name=args.image.stem, + uid=0, + ) + + prune_ratio = prune_ratio_from_keep(args.keep_ratio, args.keep_count, int(gaussians.get_xyz.shape[0])) + background = torch.tensor([1.0, 1.0, 1.0] if args.background == "white" else [0.0, 0.0, 0.0], dtype=torch.float32, device="cuda") + scene = SingleCameraScene([camera]) + pipe = SimpleNamespace(debug=False, convert_SHs_python=False, compute_cov3D_python=False) + prune_metrics = run_speedy_prune(torch, speedy_prune, scene, gaussians, pipe, background, prune_ratio) + + args.output.parent.mkdir(parents=True, exist_ok=True) + gaussians.save_ply(args.output.as_posix()) + metadata_path = args.metadata or args.output.with_suffix(".speedy_splat.json") + write_json( + metadata_path, + { + "backend": "speedy_splat", + "speedy_root": args.speedy_root.as_posix(), + "input": args.input.as_posix(), + "output": args.output.as_posix(), + "image": args.image.as_posix(), + "cameras": args.cameras.as_posix(), + "sh_degree": sh_degree, + "keep_ratio": args.keep_ratio, + "keep_count": args.keep_count, + "camera_view_index": view_index, + "prune_ratio": prune_ratio, + "input_gaussians": int(len(PlyData.read(args.input)["vertex"])), + "output_gaussians": int(gaussians.get_xyz.shape[0]), + "prune_metrics": prune_metrics, + "prune_policy": prune_policy, + }, + ) + return 0 + + +class SingleCameraScene: + def __init__(self, cameras: list[Any]) -> None: + self._cameras = cameras + + def getTrainCameras(self) -> list[Any]: + return self._cameras + + +def run_speedy_prune( + torch_module: Any, + speedy_prune: Any, + scene: Any, + gaussians: Any, + pipe: Any, + background: Any, + prune_ratio: float, +) -> dict[str, Any]: + try: + metrics = speedy_prune(scene, gaussians, pipe, background, prune_ratio) + synchronize_cuda(torch_module) + return dict(metrics) if isinstance(metrics, dict) else {"status": "ok"} + except RuntimeError as exc: + trace = traceback.format_exc() + if not is_cuda_event_timing_failure(exc, trace): + raise + synchronize_cuda(torch_module) + return { + "status": "ok_with_cuda_timing_warning", + "timing_warning": first_line(str(exc)), + } + + +def install_exact_count_pruning(torch_module: Any, gaussians: Any) -> str: + def prune_gaussians(percent: float, import_score: Any) -> None: + prune_gaussians_exact_count(torch_module, gaussians, percent, import_score) + + gaussians.prune_gaussians = prune_gaussians + return "speedy_splat_exact_count_tie_safe" + + +def prune_gaussians_exact_count(torch_module: Any, gaussians: Any, percent: float, import_score: Any) -> None: + scores = import_score.reshape(-1) + total = int(scores.shape[0]) + prune_count = exact_prune_count(total, percent) + prune_mask = torch_module.zeros((total,), dtype=torch_module.bool, device=scores.device) + if prune_count > 0: + try: + order = torch_module.argsort(scores, stable=True) + except TypeError: + order = torch_module.argsort(scores) + prune_mask[order[:prune_count]] = True + gaussians.prune_points(prune_mask) + + +def exact_prune_count(total: int, percent: float) -> int: + if total <= 1: + return 0 + if percent <= 0.0: + return 0 + return min(max(int(round(total * percent)), 0), total - 1) + + +def is_cuda_event_timing_failure(exc: RuntimeError, trace: str) -> bool: + message = str(exc).lower() + return "device not ready" in message and "elapsed_time" in trace + + +def synchronize_cuda(torch_module: Any) -> None: + cuda = getattr(torch_module, "cuda", None) + synchronize = getattr(cuda, "synchronize", None) + if callable(synchronize): + synchronize() + + +def first_line(value: str) -> str: + return value.splitlines()[0] if value.splitlines() else value + + +def speedy_training_args() -> SimpleNamespace: + return SimpleNamespace( + percent_dense=0.01, + position_lr_init=0.00016, + position_lr_final=0.0000016, + position_lr_delay_mult=0.01, + position_lr_max_steps=30_000, + feature_lr=0.0025, + opacity_lr=0.05, + scaling_lr=0.005, + rotation_lr=0.001, + ) + + +def infer_sh_degree(property_names: list[str]) -> int: + f_rest = [name for name in property_names if name.startswith("f_rest_")] + coeffs = (len(f_rest) + 3) / 3 + degree = math.sqrt(coeffs) - 1 + rounded = int(round(degree)) + if rounded < 0 or not math.isclose(degree, rounded, rel_tol=0.0, abs_tol=1e-6): + raise SystemExit(f"cannot infer spherical harmonics degree from {len(f_rest)} f_rest properties") + return rounded + + +def prune_ratio_from_keep(keep_ratio: float, keep_count: int | None, total: int) -> float: + if total <= 0: + raise SystemExit("input PLY has no gaussians") + if keep_count is not None: + if keep_count <= 0: + raise SystemExit("--keep-count must be positive") + keep_ratio = min(1.0, max(0.0, keep_count / total)) + if not 0.0 < keep_ratio <= 1.0: + raise SystemExit("--keep-ratio must be in (0, 1]") + return max(0.0, 1.0 - keep_ratio) + + +def require_intrinsics(payload: dict[str, Any]) -> dict[str, float]: + intrinsics = payload.get("intrinsics") + required = ("width", "height", "fx", "fy") + if not isinstance(intrinsics, dict) or any(key not in intrinsics for key in required): + raise SystemExit(f"camera JSON must contain intrinsics with {', '.join(required)}") + return intrinsics + + +def focal_to_fov(focal: float, pixels: int) -> float: + if focal <= 0: + raise SystemExit("camera focal length must be positive") + return 2.0 * math.atan(float(pixels) / (2.0 * focal)) + + +def image_view_index(payload: dict[str, Any], image_path: Path) -> int: + image_paths = payload.get("image_paths") + if not isinstance(image_paths, list): + return 0 + target = image_path.expanduser().resolve() + for index, raw_path in enumerate(image_paths): + candidate = Path(str(raw_path)).expanduser().resolve() + if candidate == target: + return index + return 0 + + +def first_matrix(value: Any, *, rows: int, cols: int, view_index: int = 0) -> np.ndarray: + arr = np.asarray(value, dtype=np.float32) + while arr.ndim > 2 and arr.shape[0] == 1: + arr = arr[0] + if arr.ndim == 3 and arr.shape[1:] == (rows, cols): + index = min(max(int(view_index), 0), arr.shape[0] - 1) + arr = arr[index] + if arr.shape != (rows, cols): + raise SystemExit(f"expected matrix shape {(rows, cols)}, got {arr.shape}") + return arr + + +def speedy_camera_rt(extrinsic: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + # Speedy-Splat's Camera builds world_view_transform with R.T and T. + return np.asarray(extrinsic[:3, :3], dtype=np.float32).T, np.asarray(extrinsic[:3, 3], dtype=np.float32) + + +def load_image_tensor(image_module: Any, torch_module: Any, path: Path, width: int, height: int): + image = image_module.open(path).convert("RGB").resize((width, height)) + arr = np.asarray(image, dtype=np.float32) / 255.0 + return torch_module.from_numpy(arr).permute(2, 0, 1).contiguous() + + +def load_json(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + raise SystemExit(f"expected JSON object: {path}") + return data + + +def write_json(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + f.write("\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_external/synthetic_multiview_expand.py b/vendor/gs_playground/scripts/real2sim_external/synthetic_multiview_expand.py new file mode 100644 index 0000000..c9a3208 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_external/synthetic_multiview_expand.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Expand one RGB reference image into synthetic multi-view frames. + +This is an optional Real2Sim adapter. It calls an OpenAI-style image edit API +with one reference image, saves the returned contact sheet, and slices it into +ordered view images that AnySplat can consume through ``--frames-dir``. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +from pathlib import Path +from typing import Any + + +VIEW_NAMES = [ + "01_front_main", + "02_front_left_45", + "03_left_90", + "04_rear_left_135", + "05_rear_back", + "06_rear_right_225", + "07_right_270", + "08_top_oblique", +] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate synthetic multi-view frames from one image") + parser.add_argument("--image", type=Path, required=True, help="Reference RGB image") + parser.add_argument("--output-dir", type=Path, required=True, help="Directory for generated metadata and contact sheet") + parser.add_argument("--frames-dir", type=Path, required=True, help="Directory where sliced views are written") + parser.add_argument("--prompt", default="", help="Image edit prompt. A real2sim prompt is built when omitted.") + parser.add_argument("--mode", choices=["object", "scene"], default="object", help="Default prompt mode when --prompt is omitted") + parser.add_argument("--api-url", default=os.environ.get("REAL2SIM_IMAGE_EDIT_API_URL", "https://chatapi.applyma.com/v1/images/edits")) + parser.add_argument("--api-key-env", default="REAL2SIM_IMAGE_EDIT_API_KEY", help="Environment variable that contains the API key") + parser.add_argument("--model", default=os.environ.get("REAL2SIM_IMAGE_EDIT_MODEL", "gpt-image-2")) + parser.add_argument("--size", default=os.environ.get("REAL2SIM_IMAGE_EDIT_SIZE", "1536x1024")) + parser.add_argument("--quality", default=os.environ.get("REAL2SIM_IMAGE_EDIT_QUALITY", "auto")) + parser.add_argument("--response-format", default="b64_json") + parser.add_argument("--cols", type=int, default=4) + parser.add_argument("--rows", type=int, default=2) + parser.add_argument( + "--cell-crop-ratio", + type=float, + default=float(os.environ.get("REAL2SIM_IMAGE_EDIT_CELL_CROP_RATIO", "0.015")), + help="Small per-cell edge crop used to remove generated contact-sheet gutters", + ) + parser.add_argument( + "--keep-label-bars", + action="store_true", + help="Keep generated panel label bars instead of automatically cropping dark bottom captions", + ) + parser.add_argument("--timeout", type=float, default=float(os.environ.get("REAL2SIM_IMAGE_EDIT_TIMEOUT", "300"))) + args = parser.parse_args() + + if not args.image.is_file(): + raise SystemExit(f"reference image not found: {args.image}") + api_key = os.environ.get(args.api_key_env) + if not api_key: + raise SystemExit(f"missing API key: set {args.api_key_env}") + + args.output_dir.mkdir(parents=True, exist_ok=True) + prompt = args.prompt.strip() or build_real2sim_multiview_prompt("the reference object", mode=args.mode) + (args.output_dir / "prompt.txt").write_text(prompt, encoding="utf-8") + payload = request_image_edit( + api_url=args.api_url, + api_key=api_key, + image=args.image, + prompt=prompt, + model=args.model, + size=args.size, + quality=args.quality, + response_format=args.response_format, + timeout=args.timeout, + ) + write_response_metadata(payload, args.output_dir / "response_meta.json") + contact_sheet = args.output_dir / "contact_sheet.png" + save_first_image_payload(payload, contact_sheet, timeout=args.timeout) + views = split_contact_sheet( + contact_sheet, + args.frames_dir, + cols=args.cols, + rows=args.rows, + cell_crop_ratio=args.cell_crop_ratio, + remove_label_bars=not args.keep_label_bars, + ) + manifest = { + "backend": "synthetic_multiview_expand", + "reference_image": args.image.as_posix(), + "contact_sheet": contact_sheet.as_posix(), + "frames_dir": args.frames_dir.as_posix(), + "views": [path.as_posix() for path in views], + "grid": {"cols": args.cols, "rows": args.rows}, + "cell_crop_ratio": args.cell_crop_ratio, + "model": args.model, + "size": args.size, + "quality": args.quality, + } + (args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return 0 + + +def build_real2sim_multiview_prompt(subject: str, *, mode: str = "object") -> str: + if mode == "scene": + return ( + "Create a single photorealistic real2sim scene-completion multiview contact sheet from the reference image. " + f"Scene identity to preserve: {subject}. Treat the reference as the canonical source view and preserve the exact visible layout, " + "floor plane, walls, doors, windows, static furniture, fixtures, material colors, lighting direction, and object identities. " + "Output exactly 8 equal portrait-photo panels in a clean 4 columns x 2 rows grid, filling one 1536x1024 image. " + "Use thin white gutters between panels. No panel titles, labels, captions, numbers, UI icons, or watermarks. " + "Views, left to right, top row then bottom row: source/front view, front-left 45 degree view, left side 90 degree view, " + "rear-left 135 degree view, rear/back view, rear-right 225 degree view, right side 270 degree view, and top-oblique " + "30 to 45 degree view. Infer unseen regions conservatively as physically plausible continuations of the visible scene, " + "keeping camera height, focal length, scale, horizon, navigable ground continuity, occlusion relationships, and shadows consistent. " + "Do not turn the scene into an isolated object turntable, do not invent a different room, and do not add people." + ) + return ( + "Create a single photorealistic real2sim multiview capture contact sheet from the reference image. " + f"Subject identity to preserve: {subject}. Keep the same object geometry, material, colors, labels, markings, and scale. " + "Output exactly 8 equal portrait-photo panels in a clean 4 columns x 2 rows grid, filling one 1536x1024 image. " + "Use thin white gutters between panels. No panel titles, labels, captions, numbers, UI icons, or watermarks. " + "Views, left to right, top row then bottom row: front main view, front-left 45 degree view, left side 90 degree view, " + "rear-left 135 degree view, rear/back view, rear-right 225 degree view, right side 270 degree view, and top-oblique " + "30 to 45 degree view. Keep the object stationary on the same tabletop, consistent focal length, consistent soft lighting, " + "sharp focus, no motion blur, no hands, no people blocking the object, no play button overlays. Keep the object fully inside " + "every panel with 10 to 15 percent silhouette margin and preserve reconstruction-useful texture and contour details." + ) + + +def request_image_edit( + *, + api_url: str, + api_key: str, + image: Path, + prompt: str, + model: str, + size: str, + quality: str, + response_format: str, + timeout: float, +) -> dict[str, Any]: + try: + import requests + except Exception as exc: + raise SystemExit(f"synthetic multiview expansion requires requests: {exc}") from exc + + with image.open("rb") as image_file: + response = requests.post( + api_url, + headers={"Authorization": f"Bearer {api_key}"}, + data={ + "model": model, + "prompt": prompt, + "size": size, + "quality": quality, + "response_format": response_format, + "n": "1", + }, + files=[("image", (image.name, image_file, mime_type_for_image(image)))], + timeout=timeout, + ) + if response.status_code >= 400: + raise SystemExit(f"image edit API failed with status {response.status_code}: {response.text[:1000]}") + try: + payload = response.json() + except Exception as exc: + raise SystemExit(f"image edit API returned non-JSON response: {exc}") from exc + if not isinstance(payload, dict): + raise SystemExit("image edit API response must be a JSON object") + return payload + + +def write_response_metadata(payload: dict[str, Any], path: Path) -> None: + redacted = {key: value for key, value in payload.items() if key != "data"} + if isinstance(payload.get("data"), list): + redacted["data"] = [ + {key: value for key, value in item.items() if key != "b64_json"} + for item in payload["data"] + if isinstance(item, dict) + ] + path.write_text(json.dumps(redacted, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def save_first_image_payload(payload: dict[str, Any], output: Path, *, timeout: float) -> None: + data = payload.get("data") + if not isinstance(data, list) or not data or not isinstance(data[0], dict): + raise SystemExit("image edit API response missing data[0]") + item = data[0] + if item.get("b64_json"): + output.write_bytes(base64.b64decode(str(item["b64_json"]))) + return + if item.get("url"): + try: + import requests + except Exception as exc: + raise SystemExit(f"downloading URL response requires requests: {exc}") from exc + response = requests.get(str(item["url"]), timeout=timeout) + if response.status_code >= 400: + raise SystemExit(f"failed to download generated image: HTTP {response.status_code}") + output.write_bytes(response.content) + return + raise SystemExit("image edit API response data[0] must contain b64_json or url") + + +def split_contact_sheet( + contact_sheet: Path, + frames_dir: Path, + *, + cols: int = 4, + rows: int = 2, + cell_crop_ratio: float = 0.015, + remove_label_bars: bool = True, +) -> list[Path]: + from PIL import Image + + if cols <= 0 or rows <= 0: + raise ValueError("contact sheet grid must have positive cols and rows") + if not 0 <= cell_crop_ratio < 0.25: + raise ValueError("cell_crop_ratio must be in [0, 0.25)") + image = Image.open(contact_sheet).convert("RGB") + width, height = image.size + cell_w = width // cols + cell_h = height // rows + if cell_w <= 0 or cell_h <= 0: + raise ValueError(f"contact sheet is too small for {cols}x{rows}: {width}x{height}") + frames_dir.mkdir(parents=True, exist_ok=True) + views: list[Path] = [] + for index in range(cols * rows): + col = index % cols + row = index // cols + left = col * cell_w + top = row * cell_h + right = width if col == cols - 1 else (col + 1) * cell_w + bottom = height if row == rows - 1 else (row + 1) * cell_h + crop_x = int((right - left) * cell_crop_ratio) + crop_y = int((bottom - top) * cell_crop_ratio) + if right - left > (2 * crop_x + 8) and bottom - top > (2 * crop_y + 8): + left += crop_x + right -= crop_x + top += crop_y + bottom -= crop_y + name = VIEW_NAMES[index] if index < len(VIEW_NAMES) else f"{index + 1:02d}_view" + path = frames_dir / f"{name}.jpg" + panel = image.crop((left, top, right, bottom)) + if remove_label_bars: + panel = crop_dark_bottom_label_bar(panel) + panel.save(path, quality=95) + views.append(path) + return views + + +def crop_dark_bottom_label_bar(image: "Image.Image") -> "Image.Image": + import numpy as np + + gray = np.asarray(image.convert("L"), dtype=np.float32) / 255.0 + height, width = gray.shape + if height < 80 or width < 80: + return image + + dark_fraction = np.mean(gray < 0.10, axis=1) + row_mean = np.mean(gray, axis=1) + start = height + scan_limit = int(height * 0.55) + for row in range(height - 1, scan_limit, -1): + if dark_fraction[row] > 0.55 and row_mean[row] < 0.22: + start = row + continue + if start < height: + break + + if start == height: + return image + + while start > scan_limit and dark_fraction[start - 1] > 0.35 and row_mean[start - 1] < 0.35: + start -= 1 + + bar_height = height - start + if bar_height < max(12, int(height * 0.035)): + return image + if start < int(height * 0.60): + return image + return image.crop((0, 0, width, start)) + + +def mime_type_for_image(path: Path) -> str: + suffix = path.suffix.lower() + if suffix in {".jpg", ".jpeg"}: + return "image/jpeg" + if suffix == ".webp": + return "image/webp" + return "image/png" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_movable_tasks_to_objects.py b/vendor/gs_playground/scripts/real2sim_movable_tasks_to_objects.py new file mode 100644 index 0000000..9efd48d --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_movable_tasks_to_objects.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Materialize movable extraction tasks into provisional Real2Sim object assets.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from real2sim.builtin import write_3dgs_ply, write_box_obj +from real2sim.contracts import write_json +from real2sim.image_io import read_gray, read_rgb + + +def main() -> int: + parser = argparse.ArgumentParser(description="Convert movable extraction task crops/masks into provisional object assets") + parser.add_argument("--package", type=Path, required=True, help="Real2Sim package root") + parser.add_argument("--tasks", type=Path, default=None, help="movable_extraction_tasks.json; defaults under package") + parser.add_argument("--output-objects-json", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--max-objects", type=int, default=3) + args = parser.parse_args() + + result = materialize_movable_tasks( + package_root=args.package, + tasks_path=args.tasks or args.package / "artifacts" / "assets" / "movable_extraction_tasks.json", + output_objects_json=args.output_objects_json, + output_dir=args.output_dir, + max_objects=max(1, int(args.max_objects)), + ) + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 + + +def materialize_movable_tasks( + *, + package_root: Path, + tasks_path: Path, + output_objects_json: Path, + output_dir: Path, + max_objects: int, +) -> dict[str, Any]: + tasks_data = json.loads(tasks_path.read_text(encoding="utf-8")) + tasks = tasks_data.get("tasks", []) if isinstance(tasks_data, dict) else [] + output_dir.mkdir(parents=True, exist_ok=True) + objects: list[dict[str, Any]] = [] + for index, task in enumerate(tasks[:max_objects]): + if not isinstance(task, dict): + continue + asset = materialize_task_asset(package_root=package_root, output_dir=output_dir, task=task, index=index) + if asset is not None: + objects.append(asset) + payload = { + "format": "gs-playground-real2sim-provisional-objects-v1", + "objects": objects, + "quality": { + "object_count": len(objects), + "source_task_count": len(tasks), + "provisional": True, + }, + "commercial_gap_notes": [ + "These assets are deterministic provisional billboards/boxes from extraction crops.", + "Replace them with SAM/SAM3D object reconstructions for commercial visual fidelity.", + ], + } + write_json(output_objects_json, payload) + return {"objects_json": output_objects_json.as_posix(), **payload["quality"]} + + +def materialize_task_asset(*, package_root: Path, output_dir: Path, task: dict[str, Any], index: int) -> dict[str, Any] | None: + crop_value = task.get("crop") + mask_value = task.get("mask") + if not crop_value or not mask_value: + return None + crop_path = package_root / str(crop_value) + mask_path = package_root / str(mask_value) + rgb = read_rgb(crop_path) + mask = read_gray(mask_path) + local_mask = local_crop_mask(mask, task.get("bbox_xyxy"), rgb.shape[:2]) + if int(local_mask.sum()) < 4: + local_mask = np.ones(rgb.shape[:2], dtype=bool) + points, colors = crop_billboard_points(rgb, local_mask, max_points=6000) + if points.size == 0: + return None + label = best_prompt_label(task) + name = f"{index:02d}_{safe_token(label)}" + asset_dir = output_dir / name + gaussian = asset_dir / "gaussian.ply" + mesh = asset_dir / "mesh.obj" + extent = object_extent_from_mask(local_mask, rgb.shape[:2], label=label) + write_3dgs_ply(gaussian, points, colors, point_scale=0.01, opacity_logit=4.0) + write_box_obj(mesh, extent) + pose = pose_from_task_bbox(task.get("bbox_xyxy")) + return { + "name": name, + "label": label, + "gaussian_ply": gaussian.as_posix(), + "mesh_obj": mesh.as_posix(), + "pose": pose, + "dynamic": True, + "source_instance_id": str(task.get("id") or name), + "provisional": True, + } + + +def local_crop_mask(global_mask: np.ndarray, bbox: Any, crop_shape: tuple[int, int]) -> np.ndarray: + if not isinstance(bbox, list) or len(bbox) != 4: + return np.ones(crop_shape, dtype=bool) + x0, y0, x1, y1 = [int(value) for value in bbox] + cropped = global_mask[max(0, y0) : max(0, y1), max(0, x0) : max(0, x1)] > 0.5 + if cropped.shape != crop_shape: + return np.ones(crop_shape, dtype=bool) + return cropped + + +def crop_billboard_points(rgb: np.ndarray, mask: np.ndarray, *, max_points: int) -> tuple[np.ndarray, np.ndarray]: + ys, xs = np.where(mask) + if ys.size == 0: + return np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.float32) + if ys.size > max_points: + indices = np.linspace(0, ys.size - 1, max_points, dtype=np.int64) + ys = ys[indices] + xs = xs[indices] + h, w = rgb.shape[:2] + x = ((xs.astype(np.float32) / max(w - 1, 1)) - 0.5) * 0.38 + z = (0.5 - (ys.astype(np.float32) / max(h - 1, 1))) * 0.38 + y = np.zeros_like(x) + points = np.stack([x, y, z], axis=1).astype(np.float32) + colors = rgb[ys, xs, :3].astype(np.float32) + return points, colors + + +def object_extent_from_mask(mask: np.ndarray, shape: tuple[int, int], *, label: str) -> np.ndarray: + ys, xs = np.where(mask) + if ys.size == 0: + return np.array([0.2, 0.08, 0.2], dtype=np.float32) + h, w = shape + width = max((float(xs.max() - xs.min() + 1) / max(w, 1)) * 0.38, 0.06) + height = max((float(ys.max() - ys.min() + 1) / max(h, 1)) * 0.38, 0.06) + depth = 0.08 + if "bottle" in label or "cup" in label: + depth = max(width * 0.55, 0.06) + return np.array([width, depth, height], dtype=np.float32) + + +def pose_from_task_bbox(bbox: Any) -> dict[str, list[float]]: + if not isinstance(bbox, list) or len(bbox) != 4: + return {"position": [0.0, -0.25, 0.65], "quat_wxyz": [1.0, 0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]} + x0, y0, x1, y1 = [float(value) for value in bbox] + cx = (x0 + x1) * 0.5 + cy = (y0 + y1) * 0.5 + position = [round((cx - 224.0) / 224.0 * 0.9, 6), -0.35, round((448.0 - cy) / 448.0 * 0.65 + 0.25, 6)] + return {"position": position, "quat_wxyz": [1.0, 0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]} + + +def best_prompt_label(task: dict[str, Any]) -> str: + prompts = task.get("prompt_candidates") + if isinstance(prompts, list) and prompts: + return str(prompts[0]) + return "movable_object" + + +def safe_token(value: str) -> str: + return "".join(ch if ch.isalnum() else "_" for ch in value.lower()).strip("_") or "object" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_obj_mesh_cleanup.py b/vendor/gs_playground/scripts/real2sim_obj_mesh_cleanup.py new file mode 100644 index 0000000..6348c43 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_obj_mesh_cleanup.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python3 +"""Filter OBJ mesh fragments before Real2Sim texture baking.""" + +from __future__ import annotations + +import argparse +import json +from collections import defaultdict, deque +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + + +@dataclass(frozen=True) +class ObjFace: + vertices: tuple[int, int, int] + texcoords: tuple[int, int, int] + + +@dataclass(frozen=True) +class ObjMesh: + vertices: np.ndarray + colors: np.ndarray + texcoords: np.ndarray + faces: tuple[ObjFace, ...] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Filter small disconnected OBJ mesh fragments") + parser.add_argument("--input-obj", type=Path, required=True) + parser.add_argument("--output-obj", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--keep-largest", type=int, default=1, help="Keep this many largest connected components") + parser.add_argument("--min-faces", type=int, default=0, help="Also keep components with at least this many faces") + parser.add_argument("--min-area-ratio", type=float, default=0.0, help="Also keep components with area >= ratio * total area") + parser.add_argument("--max-face-area-percentile", type=float, default=100.0, help="Drop triangles above this area percentile") + parser.add_argument("--max-edge-length-percentile", type=float, default=100.0, help="Drop triangles whose longest edge is above this percentile") + parser.add_argument("--min-centroid-x-percentile", type=float, default=0.0, help="Drop triangles whose centroid x is below this percentile") + parser.add_argument("--max-centroid-x-percentile", type=float, default=100.0, help="Drop triangles whose centroid x is above this percentile") + parser.add_argument("--min-centroid-y-percentile", type=float, default=0.0, help="Drop triangles whose centroid y is below this percentile") + parser.add_argument("--max-centroid-y-percentile", type=float, default=100.0, help="Drop triangles whose centroid y is above this percentile") + parser.add_argument("--min-centroid-z-percentile", type=float, default=0.0, help="Drop triangles whose centroid z is below this percentile") + parser.add_argument("--max-centroid-z-percentile", type=float, default=100.0, help="Drop triangles whose centroid z is above this percentile") + args = parser.parse_args() + + report = cleanup_obj_mesh( + input_obj=args.input_obj, + output_obj=args.output_obj, + keep_largest=max(int(args.keep_largest), 0), + min_faces=max(int(args.min_faces), 0), + min_area_ratio=max(float(args.min_area_ratio), 0.0), + max_face_area_percentile=float(args.max_face_area_percentile), + max_edge_length_percentile=float(args.max_edge_length_percentile), + centroid_percentile_bounds={ + "x": (float(args.min_centroid_x_percentile), float(args.max_centroid_x_percentile)), + "y": (float(args.min_centroid_y_percentile), float(args.max_centroid_y_percentile)), + "z": (float(args.min_centroid_z_percentile), float(args.max_centroid_z_percentile)), + }, + ) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + + +def cleanup_obj_mesh( + *, + input_obj: Path, + output_obj: Path, + keep_largest: int = 1, + min_faces: int = 0, + min_area_ratio: float = 0.0, + max_face_area_percentile: float = 100.0, + max_edge_length_percentile: float = 100.0, + centroid_percentile_bounds: dict[str, tuple[float, float]] | None = None, +) -> dict[str, Any]: + mesh = read_obj_mesh(input_obj) + components = connected_face_components(mesh) + face_areas = triangle_areas(mesh) + face_edges = triangle_max_edges(mesh) + centroids = triangle_centroids(mesh) + total_area = float(np.sum(face_areas)) + area_threshold = percentile_threshold(face_areas, max_face_area_percentile) + edge_threshold = percentile_threshold(face_edges, max_edge_length_percentile) + centroid_filters = centroid_axis_filters(centroids, centroid_percentile_bounds) + ranked = sorted( + ( + { + "component": idx, + "face_count": len(face_indices), + "area": float(np.sum(face_areas[np.asarray(face_indices, dtype=np.int64)])), + "faces": face_indices, + } + for idx, face_indices in enumerate(components) + ), + key=lambda item: (item["face_count"], item["area"]), + reverse=True, + ) + keep_component_ids: set[int] = set() + for item in ranked[:keep_largest]: + keep_component_ids.add(int(item["component"])) + for item in ranked: + area_ratio = (float(item["area"]) / total_area) if total_area > 0 else 0.0 + if (min_faces > 0 and int(item["face_count"]) >= min_faces) or ( + min_area_ratio > 0.0 and area_ratio >= min_area_ratio + ): + keep_component_ids.add(int(item["component"])) + kept_faces_before_quality = [ + face_index + for component_id, face_indices in enumerate(components) + if component_id in keep_component_ids + for face_index in face_indices + ] + kept_faces = [ + face_index + for face_index in kept_faces_before_quality + if (area_threshold is None or face_areas[int(face_index)] <= area_threshold) + and (edge_threshold is None or face_edges[int(face_index)] <= edge_threshold) + and centroid_face_allowed(centroids[int(face_index)], centroid_filters) + ] + cleaned = remap_mesh(mesh, sorted(kept_faces)) + output_obj.parent.mkdir(parents=True, exist_ok=True) + write_obj_mesh(cleaned, output_obj) + kept_area = float(np.sum(face_areas[np.asarray(kept_faces, dtype=np.int64)])) if kept_faces else 0.0 + report = { + "format": "gs-playground-real2sim-obj-mesh-cleanup-report-v1", + "input_obj": input_obj.resolve().as_posix(), + "output_obj": output_obj.resolve().as_posix(), + "input_vertex_count": int(mesh.vertices.shape[0]), + "input_texcoord_count": int(mesh.texcoords.shape[0]), + "input_face_count": int(len(mesh.faces)), + "component_count": int(len(components)), + "kept_component_count": int(len(keep_component_ids)), + "component_kept_face_count": int(len(kept_faces_before_quality)), + "kept_face_count": int(len(kept_faces)), + "removed_face_count": int(len(mesh.faces) - len(kept_faces)), + "quality_removed_face_count": int(len(kept_faces_before_quality) - len(kept_faces)), + "centroid_filter_removed_face_count": int( + sum( + 1 + for face_index in kept_faces_before_quality + if not centroid_face_allowed(centroids[int(face_index)], centroid_filters) + ) + ), + "kept_area_ratio": round((kept_area / total_area) if total_area > 0 else 0.0, 6), + "keep_largest": int(keep_largest), + "min_faces": int(min_faces), + "min_area_ratio": float(min_area_ratio), + "max_face_area_percentile": float(max_face_area_percentile), + "max_edge_length_percentile": float(max_edge_length_percentile), + "face_area_threshold": float(area_threshold) if area_threshold is not None else None, + "edge_length_threshold": float(edge_threshold) if edge_threshold is not None else None, + "centroid_filters": centroid_filters, + "largest_components": [ + { + "component": int(item["component"]), + "face_count": int(item["face_count"]), + "area_ratio": round((float(item["area"]) / total_area) if total_area > 0 else 0.0, 6), + } + for item in ranked[:10] + ], + } + return report + + +def connected_face_components(mesh: ObjMesh) -> list[list[int]]: + vertex_to_faces: dict[int, list[int]] = defaultdict(list) + for face_index, face in enumerate(mesh.faces): + for vertex in face.vertices: + vertex_to_faces[int(vertex)].append(face_index) + visited = np.zeros(len(mesh.faces), dtype=bool) + components: list[list[int]] = [] + for start in range(len(mesh.faces)): + if visited[start]: + continue + queue: deque[int] = deque([start]) + visited[start] = True + component: list[int] = [] + while queue: + face_index = queue.popleft() + component.append(face_index) + for vertex in mesh.faces[face_index].vertices: + for neighbor in vertex_to_faces[int(vertex)]: + if not visited[neighbor]: + visited[neighbor] = True + queue.append(neighbor) + components.append(component) + return components + + +def triangle_areas(mesh: ObjMesh) -> np.ndarray: + if not mesh.faces: + return np.zeros((0,), dtype=np.float64) + indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64) + points = mesh.vertices[indices].astype(np.float64) + areas = np.linalg.norm(np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]), axis=1) * 0.5 + return areas + + +def triangle_max_edges(mesh: ObjMesh) -> np.ndarray: + if not mesh.faces: + return np.zeros((0,), dtype=np.float64) + indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64) + points = mesh.vertices[indices].astype(np.float64) + e01 = np.linalg.norm(points[:, 0] - points[:, 1], axis=1) + e12 = np.linalg.norm(points[:, 1] - points[:, 2], axis=1) + e20 = np.linalg.norm(points[:, 2] - points[:, 0], axis=1) + return np.maximum(np.maximum(e01, e12), e20) + + +def triangle_centroids(mesh: ObjMesh) -> np.ndarray: + if not mesh.faces: + return np.zeros((0, 3), dtype=np.float64) + indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64) + points = mesh.vertices[indices].astype(np.float64) + return points.mean(axis=1) + + +def percentile_threshold(values: np.ndarray, percentile: float) -> float | None: + if values.size == 0 or percentile >= 100.0: + return None + clamped = min(max(float(percentile), 0.0), 100.0) + return float(np.percentile(values, clamped)) + + +def centroid_axis_filters( + centroids: np.ndarray, + percentile_bounds: dict[str, tuple[float, float]] | None, +) -> dict[str, dict[str, float | None]]: + axes = {"x": 0, "y": 1, "z": 2} + filters: dict[str, dict[str, float | None]] = {} + if centroids.size == 0: + return { + axis: {"min_percentile": None, "max_percentile": None, "min_value": None, "max_value": None} + for axis in axes + } + percentile_bounds = percentile_bounds or {} + for axis, index in axes.items(): + lower, upper = percentile_bounds.get(axis, (0.0, 100.0)) + lower = min(max(float(lower), 0.0), 100.0) + upper = min(max(float(upper), 0.0), 100.0) + if lower > upper: + lower, upper = upper, lower + values = centroids[:, index] + min_value = float(np.percentile(values, lower)) if lower > 0.0 else None + max_value = float(np.percentile(values, upper)) if upper < 100.0 else None + filters[axis] = { + "min_percentile": lower if min_value is not None else None, + "max_percentile": upper if max_value is not None else None, + "min_value": min_value, + "max_value": max_value, + } + return filters + + +def centroid_face_allowed(centroid: np.ndarray, filters: dict[str, dict[str, float | None]]) -> bool: + for axis, index in {"x": 0, "y": 1, "z": 2}.items(): + bounds = filters.get(axis, {}) + min_value = bounds.get("min_value") + max_value = bounds.get("max_value") + value = float(centroid[index]) + if min_value is not None and value < float(min_value): + return False + if max_value is not None and value > float(max_value): + return False + return True + + +def remap_mesh(mesh: ObjMesh, face_indices: list[int]) -> ObjMesh: + vertex_map: dict[int, int] = {} + texcoord_map: dict[int, int] = {} + vertices: list[np.ndarray] = [] + colors: list[np.ndarray] = [] + texcoords: list[np.ndarray] = [] + faces: list[ObjFace] = [] + for face_index in face_indices: + face = mesh.faces[int(face_index)] + new_vertices: list[int] = [] + new_texcoords: list[int] = [] + for vertex in face.vertices: + if vertex not in vertex_map: + vertex_map[vertex] = len(vertices) + vertices.append(mesh.vertices[vertex]) + colors.append(mesh.colors[vertex]) + new_vertices.append(vertex_map[vertex]) + for texcoord in face.texcoords: + if texcoord not in texcoord_map: + texcoord_map[texcoord] = len(texcoords) + texcoords.append(mesh.texcoords[texcoord]) + new_texcoords.append(texcoord_map[texcoord]) + faces.append(ObjFace(vertices=tuple(new_vertices), texcoords=tuple(new_texcoords))) # type: ignore[arg-type] + return ObjMesh( + vertices=np.asarray(vertices, dtype=np.float32), + colors=np.asarray(colors, dtype=np.float32), + texcoords=np.asarray(texcoords, dtype=np.float32), + faces=tuple(faces), + ) + + +def read_obj_mesh(path: Path) -> ObjMesh: + vertices: list[tuple[float, float, float]] = [] + colors: list[tuple[float, float, float]] = [] + texcoords: list[tuple[float, float]] = [] + faces: list[ObjFace] = [] + for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + if raw.startswith("v "): + parts = raw.split() + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + if len(parts) >= 7: + colors.append((float(parts[4]), float(parts[5]), float(parts[6]))) + else: + colors.append((0.5, 0.5, 0.5)) + elif raw.startswith("vt "): + parts = raw.split() + if len(parts) >= 3: + texcoords.append((float(parts[1]), float(parts[2]))) + elif raw.startswith("f "): + refs = [parse_face_ref(token) for token in raw.split()[1:]] + if len(refs) < 3: + continue + for idx in range(1, len(refs) - 1): + tri = (refs[0], refs[idx], refs[idx + 1]) + faces.append( + ObjFace( + vertices=tuple(item[0] for item in tri), + texcoords=tuple(item[1] if item[1] is not None else item[0] for item in tri), + ) + ) + if not texcoords: + texcoords = [(0.0, 0.0) for _ in vertices] + return ObjMesh( + vertices=np.asarray(vertices, dtype=np.float32), + colors=np.clip(np.asarray(colors, dtype=np.float32), 0.0, 1.0), + texcoords=np.asarray(texcoords, dtype=np.float32), + faces=tuple(faces), + ) + + +def parse_face_ref(token: str) -> tuple[int, int | None]: + parts = token.split("/") + vertex = int(parts[0]) - 1 + texcoord = int(parts[1]) - 1 if len(parts) >= 2 and parts[1] else None + return vertex, texcoord + + +def write_obj_mesh(mesh: ObjMesh, output_obj: Path) -> None: + material_name = output_obj.stem + "_mat" + material_file = output_obj.with_suffix(".mtl") + with material_file.open("w", encoding="utf-8") as dst: + dst.write("# Generated Real2Sim cleaned OBJ material\n") + dst.write(f"newmtl {material_name}\n") + dst.write("Ka 1.000000 1.000000 1.000000\n") + dst.write("Kd 1.000000 1.000000 1.000000\n") + dst.write("Ks 0.030000 0.030000 0.030000\n") + dst.write("Ns 8.000000\n") + dst.write("d 1.000000\n") + dst.write("illum 2\n") + with output_obj.open("w", encoding="utf-8") as dst: + dst.write("# Real2Sim cleaned scene mesh\n") + dst.write(f"mtllib {material_file.name}\n") + dst.write("o real2sim_cleaned_scene\n") + for vertex, color in zip(mesh.vertices, mesh.colors): + dst.write( + f"v {vertex[0]:.9g} {vertex[1]:.9g} {vertex[2]:.9g} " + f"{color[0]:.6f} {color[1]:.6f} {color[2]:.6f}\n" + ) + for texcoord in mesh.texcoords: + dst.write(f"vt {texcoord[0]:.7g} {texcoord[1]:.7g}\n") + dst.write(f"usemtl {material_name}\n") + for face in mesh.faces: + dst.write( + "f " + f"{face.vertices[0] + 1}/{face.texcoords[0] + 1} " + f"{face.vertices[1] + 1}/{face.texcoords[1] + 1} " + f"{face.vertices[2] + 1}/{face.texcoords[2] + 1}\n" + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_obj_scene_completion.py b/vendor/gs_playground/scripts/real2sim_obj_scene_completion.py new file mode 100644 index 0000000..2eed14f --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_obj_scene_completion.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Add coarse room/tabletop completion planes to a reconstructed OBJ scene. + +The COLMAP/Delaunay mesh path often produces visually useful fragments but +leaves large holes in walls, tabletops, or floors. This utility keeps the +original OBJ intact and appends a conservative, quantile-bounded planar shell +so MotrixSim has a continuous scene scaffold for inspection and collision. +""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +@dataclass(frozen=True) +class Bounds: + low: tuple[float, float, float] + high: tuple[float, float, float] + + +@dataclass(frozen=True) +class Plane: + name: str + axis: int + value: float + u_axis: int + v_axis: int + u_min: float + u_max: float + v_min: float + v_max: float + material: str + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Append quantile-bounded scene completion planes to an OBJ") + parser.add_argument("--input-obj", type=Path, required=True) + parser.add_argument("--output-obj", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--low-quantile", type=float, default=0.05) + parser.add_argument("--high-quantile", type=float, default=0.95) + parser.add_argument("--margin-ratio", type=float, default=0.03) + parser.add_argument("--up-axis", choices=("x", "y", "z"), default="z") + parser.add_argument("--grid", type=int, default=24, help="Subdivisions per plane axis") + parser.add_argument( + "--planes", + default="floor,ceiling,left,right,back,front", + help="Comma-separated planes to append: floor,ceiling,left,right,back,front", + ) + args = parser.parse_args(argv) + + vertices, texcoord_count = read_obj_counts(args.input_obj) + if len(vertices) < 3: + raise SystemExit(f"OBJ has too few vertices for scene completion: {args.input_obj}") + low_q = clamp(args.low_quantile, 0.0, 0.49) + high_q = clamp(args.high_quantile, 0.51, 1.0) + if low_q >= high_q: + raise SystemExit("--low-quantile must be below --high-quantile") + bounds = quantile_bounds(vertices, low_q, high_q, margin_ratio=max(args.margin_ratio, 0.0)) + up_axis = "xyz".index(args.up_axis) + plane_names = tuple(name.strip().lower() for name in args.planes.split(",") if name.strip()) + planes = build_planes(bounds, up_axis=up_axis, names=plane_names) + if not planes: + raise SystemExit("No completion planes selected") + + args.output_obj.parent.mkdir(parents=True, exist_ok=True) + mtl_path = args.output_obj.with_name(f"{args.output_obj.stem}_completion.mtl") + append_completion( + args.input_obj, + args.output_obj, + mtl_path, + vertices_start=len(vertices), + texcoords_start=texcoord_count, + planes=planes, + grid=max(args.grid, 1), + ) + report = { + "format": "gs-playground-real2sim-obj-scene-completion-report-v1", + "input_obj": args.input_obj.as_posix(), + "output_obj": args.output_obj.as_posix(), + "material_library": mtl_path.name, + "input_vertex_count": len(vertices), + "input_texcoord_count": texcoord_count, + "bounds_low": list(bounds.low), + "bounds_high": list(bounds.high), + "low_quantile": low_q, + "high_quantile": high_q, + "margin_ratio": max(args.margin_ratio, 0.0), + "up_axis": args.up_axis, + "grid": max(args.grid, 1), + "plane_count": len(planes), + "planes": [plane.name for plane in planes], + "added_vertex_count": len(planes) * (max(args.grid, 1) + 1) ** 2, + "added_texcoord_count": len(planes) * (max(args.grid, 1) + 1) ** 2, + "added_face_count": len(planes) * max(args.grid, 1) * max(args.grid, 1) * 2, + } + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + + +def read_obj_counts(path: Path) -> tuple[list[tuple[float, float, float]], int]: + vertices: list[tuple[float, float, float]] = [] + texcoord_count = 0 + with path.open("r", encoding="utf-8", errors="ignore") as handle: + for line in handle: + if line.startswith("vt "): + texcoord_count += 1 + continue + if not line.startswith("v "): + continue + parts = line.split() + if len(parts) < 4: + continue + try: + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + except ValueError: + continue + return vertices, texcoord_count + + +def quantile_bounds( + vertices: Iterable[tuple[float, float, float]], + low_quantile: float, + high_quantile: float, + *, + margin_ratio: float, +) -> Bounds: + columns = list(zip(*vertices)) + lows: list[float] = [] + highs: list[float] = [] + for values in columns: + ordered = sorted(values) + low = percentile(ordered, low_quantile) + high = percentile(ordered, high_quantile) + span = max(high - low, 1e-6) + lows.append(low - span * margin_ratio) + highs.append(high + span * margin_ratio) + return Bounds(tuple(lows), tuple(highs)) + + +def percentile(values: list[float], q: float) -> float: + if not values: + raise ValueError("percentile requires values") + if len(values) == 1: + return values[0] + pos = q * (len(values) - 1) + lo = int(pos) + hi = min(lo + 1, len(values) - 1) + frac = pos - lo + return values[lo] * (1.0 - frac) + values[hi] * frac + + +def build_planes(bounds: Bounds, *, up_axis: int, names: tuple[str, ...]) -> list[Plane]: + axes = {0, 1, 2} + horizontal = [axis for axis in axes if axis != up_axis] + a0, a1 = horizontal + low, high = bounds.low, bounds.high + specs = { + "floor": Plane("floor", up_axis, low[up_axis], a0, a1, low[a0], high[a0], low[a1], high[a1], "real2sim_completion_floor"), + "ceiling": Plane("ceiling", up_axis, high[up_axis], a0, a1, low[a0], high[a0], low[a1], high[a1], "real2sim_completion_ceiling"), + "left": Plane("left", a0, low[a0], a1, up_axis, low[a1], high[a1], low[up_axis], high[up_axis], "real2sim_completion_wall"), + "right": Plane("right", a0, high[a0], a1, up_axis, low[a1], high[a1], low[up_axis], high[up_axis], "real2sim_completion_wall"), + "back": Plane("back", a1, low[a1], a0, up_axis, low[a0], high[a0], low[up_axis], high[up_axis], "real2sim_completion_wall"), + "front": Plane("front", a1, high[a1], a0, up_axis, low[a0], high[a0], low[up_axis], high[up_axis], "real2sim_completion_wall"), + } + return [specs[name] for name in names if name in specs] + + +def append_completion( + input_obj: Path, + output_obj: Path, + mtl_path: Path, + *, + vertices_start: int, + texcoords_start: int, + planes: list[Plane], + grid: int, +) -> None: + source = input_obj.read_text(encoding="utf-8", errors="ignore") + output = [source] + if not source.endswith("\n"): + output.append("\n") + output.append(f"mtllib {mtl_path.name}\n") + output.append("o real2sim_scene_completion\n") + vertex_index = vertices_start + for plane in planes: + output.append(f"g real2sim_completion_{plane.name}\n") + output.append(f"usemtl {plane.material}\n") + vertex_start = vertex_index + 1 + texcoord_start = texcoords_start + 1 + for iv in range(grid + 1): + v_fraction = iv / grid + v = lerp(plane.v_min, plane.v_max, v_fraction) + for iu in range(grid + 1): + u_fraction = iu / grid + u = lerp(plane.u_min, plane.u_max, u_fraction) + coords = [0.0, 0.0, 0.0] + coords[plane.axis] = plane.value + coords[plane.u_axis] = u + coords[plane.v_axis] = v + output.append(f"v {coords[0]:.8f} {coords[1]:.8f} {coords[2]:.8f}\n") + output.append(f"vt {u_fraction:.8f} {v_fraction:.8f}\n") + for iv in range(grid): + for iu in range(grid): + a = vertex_start + iv * (grid + 1) + iu + b = a + 1 + c = a + (grid + 1) + d = c + 1 + ta = texcoord_start + iv * (grid + 1) + iu + tb = ta + 1 + tc = ta + (grid + 1) + td = tc + 1 + output.append(f"f {a}/{ta} {b}/{tb} {d}/{td}\n") + output.append(f"f {a}/{ta} {d}/{td} {c}/{tc}\n") + vertex_index += (grid + 1) * (grid + 1) + texcoords_start += (grid + 1) * (grid + 1) + output_obj.write_text("".join(output), encoding="utf-8") + write_completion_mtl(mtl_path) + + +def write_completion_mtl(path: Path) -> None: + path.write_text( + "\n".join( + [ + "# Generated Real2Sim scene completion materials", + "newmtl real2sim_completion_floor", + "Kd 0.420000 0.410000 0.390000", + "Ka 0.120000 0.120000 0.120000", + "Ks 0.020000 0.020000 0.020000", + "Ns 8.000000", + "", + "newmtl real2sim_completion_wall", + "Kd 0.680000 0.650000 0.590000", + "Ka 0.150000 0.145000 0.135000", + "Ks 0.015000 0.015000 0.015000", + "Ns 6.000000", + "", + "newmtl real2sim_completion_ceiling", + "Kd 0.720000 0.700000 0.660000", + "Ka 0.160000 0.155000 0.145000", + "Ks 0.010000 0.010000 0.010000", + "Ns 4.000000", + "", + ] + ), + encoding="utf-8", + ) + + +def lerp(a: float, b: float, t: float) -> float: + return a * (1.0 - t) + b * t + + +def clamp(value: float, low: float, high: float) -> float: + return min(max(value, low), high) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_obj_structure_report.py b/vendor/gs_playground/scripts/real2sim_obj_structure_report.py new file mode 100644 index 0000000..edebf45 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_obj_structure_report.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +"""Extract coarse room/scene structure cues from an OBJ mesh.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import numpy as np + + +def main() -> int: + parser = argparse.ArgumentParser(description="Analyze axis-aligned scene structure candidates in an OBJ mesh") + parser.add_argument("--input-obj", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--up-axis", choices=("x", "y", "z"), default="z") + parser.add_argument("--normal-threshold", type=float, default=0.78) + parser.add_argument("--support-normal-threshold", type=float, default=0.82) + args = parser.parse_args() + + report = analyze_obj_structure( + input_obj=args.input_obj, + up_axis=args.up_axis, + normal_threshold=max(min(float(args.normal_threshold), 1.0), 0.0), + support_normal_threshold=max(min(float(args.support_normal_threshold), 1.0), 0.0), + ) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + + +def analyze_obj_structure( + *, + input_obj: Path, + up_axis: str = "z", + normal_threshold: float = 0.78, + support_normal_threshold: float = 0.82, +) -> dict[str, Any]: + vertices, faces = read_obj_triangles(input_obj) + if len(vertices) < 3 or len(faces) < 1: + raise SystemExit(f"OBJ has no usable triangle mesh: {input_obj}") + points = vertices[faces] + normals = triangle_normals(points) + areas = triangle_areas(points) + centroids = points.mean(axis=1) + total_area = float(np.sum(areas)) + up_index = "xyz".index(up_axis) + axis_reports = [ + axis_structure_report( + axis=index, + name=name, + normals=normals, + centroids=centroids, + areas=areas, + total_area=total_area, + threshold=normal_threshold, + ) + for index, name in enumerate("xyz") + ] + support = support_plane_candidates( + up_axis=up_index, + normals=normals, + centroids=centroids, + areas=areas, + total_area=total_area, + threshold=support_normal_threshold, + ) + bounds = { + "min": [float(value) for value in vertices.min(axis=0)], + "max": [float(value) for value in vertices.max(axis=0)], + "p05": [float(np.percentile(vertices[:, idx], 5)) for idx in range(3)], + "p95": [float(np.percentile(vertices[:, idx], 95)) for idx in range(3)], + } + semantic_planes = semantic_plane_candidates( + up_axis=up_index, + normals=normals, + centroids=centroids, + areas=areas, + bounds=bounds, + total_area=total_area, + threshold=normal_threshold, + ) + semantic_planes = add_recovered_floor_candidate( + semantic_planes, + up_axis=up_index, + normals=normals, + centroids=centroids, + areas=areas, + bounds=bounds, + total_area=total_area, + threshold=support_normal_threshold, + ) + structure_score = min( + 1.0, + sum(float(axis["area_ratio"]) for axis in axis_reports) / 3.0 + + min(float(support["support_area_ratio"]) * 0.5, 0.25), + ) + semantic_score = semantic_structure_score(semantic_planes) + return { + "format": "gs-playground-real2sim-obj-structure-report-v1", + "input_obj": input_obj.resolve().as_posix(), + "vertex_count": int(vertices.shape[0]), + "face_count": int(faces.shape[0]), + "total_area": total_area, + "up_axis": up_axis, + "bounds": bounds, + "axis_aligned_planes": axis_reports, + "support_planes": support, + "semantic_planes": semantic_planes, + "structure_score": round(float(structure_score), 6), + "commercial_readiness": { + "structure_score": round(float(structure_score), 6), + "semantic_structure_score": round(float(semantic_score), 6), + "has_floor": any(plane["semantic_label"] == "floor" for plane in semantic_planes), + "has_wall": any(plane["semantic_label"].startswith("wall_") for plane in semantic_planes), + "has_support_surface": any(plane["semantic_label"] == "support_surface" for plane in semantic_planes), + "remaining_blockers": [ + "semantic instance segmentation for movable assets", + "watertight hole filling and room-boundary regularization", + "collision/nav mesh generation from semantic planes", + "physically meaningful material parameters", + ], + }, + "commercial_gap_notes": [ + "This report now adds coarse semantic structure candidates, but it is not full room understanding.", + "Commercial-grade Real2Sim still needs instance semantics, watertight reconstruction, collision/nav mesh derivation, and physical material inference.", + ], + } + + +def axis_structure_report( + *, + axis: int, + name: str, + normals: np.ndarray, + centroids: np.ndarray, + areas: np.ndarray, + total_area: float, + threshold: float, +) -> dict[str, Any]: + aligned = np.abs(normals[:, axis]) >= threshold + area = float(np.sum(areas[aligned])) if np.any(aligned) else 0.0 + values = centroids[aligned, axis] if np.any(aligned) else np.asarray([], dtype=np.float64) + return { + "axis": name, + "aligned_face_count": int(np.count_nonzero(aligned)), + "area": area, + "area_ratio": round((area / total_area) if total_area > 0 else 0.0, 6), + "low_plane": plane_summary(values, areas[aligned], quantile=5), + "high_plane": plane_summary(values, areas[aligned], quantile=95), + } + + +def support_plane_candidates( + *, + up_axis: int, + normals: np.ndarray, + centroids: np.ndarray, + areas: np.ndarray, + total_area: float, + threshold: float, +) -> dict[str, Any]: + upward = np.abs(normals[:, up_axis]) >= threshold + values = centroids[upward, up_axis] if np.any(upward) else np.asarray([], dtype=np.float64) + support_areas = areas[upward] if np.any(upward) else np.asarray([], dtype=np.float64) + low = plane_summary(values, support_areas, quantile=10) + high = plane_summary(values, support_areas, quantile=75) + support_area = float(np.sum(support_areas)) + return { + "up_axis": "xyz"[up_axis], + "support_face_count": int(np.count_nonzero(upward)), + "support_area": support_area, + "support_area_ratio": round((support_area / total_area) if total_area > 0 else 0.0, 6), + "floor_like": low, + "tabletop_or_shelf_like": high, + } + + +def semantic_plane_candidates( + *, + up_axis: int, + normals: np.ndarray, + centroids: np.ndarray, + areas: np.ndarray, + bounds: dict[str, list[float]], + total_area: float, + threshold: float, +) -> list[dict[str, Any]]: + planes: list[dict[str, Any]] = [] + for axis, name in enumerate("xyz"): + axis_span = max(float(bounds["p95"][axis] - bounds["p05"][axis]), 1e-6) + clusters = clustered_axis_planes( + axis=axis, + normals=normals, + centroids=centroids, + areas=areas, + threshold=threshold, + cluster_tolerance=max(axis_span * 0.035, 0.04), + ) + for cluster in clusters: + label = semantic_label_for_plane(axis=axis, up_axis=up_axis, coordinate=cluster["coordinate"], bounds=bounds) + if label == "vertical_partition" and cluster["area_ratio"] >= 0.05: + label = "wall_visible" + if label == "support_surface" and cluster["area_ratio"] < 0.01: + continue + if label.startswith("wall_") and cluster["area_ratio"] < 0.015: + continue + planes.append( + { + "semantic_label": label, + "axis": name, + "coordinate": round(float(cluster["coordinate"]), 6), + "area": cluster["area"], + "area_ratio": round(float(cluster["area_ratio"]), 6), + "face_count": cluster["face_count"], + "extent": cluster["extent"], + "collision_proxy_hint": collision_proxy_hint( + label=label, + axis=axis, + coordinate=float(cluster["coordinate"]), + extent=cluster["extent"], + bounds=bounds, + ), + } + ) + planes.sort(key=lambda item: (semantic_priority(item["semantic_label"]), -float(item["area_ratio"]))) + return planes[:16] + + +def add_recovered_floor_candidate( + planes: list[dict[str, Any]], + *, + up_axis: int, + normals: np.ndarray, + centroids: np.ndarray, + areas: np.ndarray, + bounds: dict[str, list[float]], + total_area: float, + threshold: float, +) -> list[dict[str, Any]]: + if any(plane["semantic_label"] == "floor" for plane in planes): + return planes + candidate = recovered_floor_candidate( + up_axis=up_axis, + normals=normals, + centroids=centroids, + areas=areas, + bounds=bounds, + total_area=total_area, + threshold=threshold, + ) + if candidate is None: + return planes + updated = [candidate, *planes] + updated.sort(key=lambda item: (semantic_priority(item["semantic_label"]), -float(item["area_ratio"]))) + return updated[:16] + + +def recovered_floor_candidate( + *, + up_axis: int, + normals: np.ndarray, + centroids: np.ndarray, + areas: np.ndarray, + bounds: dict[str, list[float]], + total_area: float, + threshold: float, +) -> dict[str, Any] | None: + upward = np.abs(normals[:, up_axis]) >= threshold + if not np.any(upward): + return None + up_values = centroids[upward, up_axis] + up_areas = areas[upward] + low = float(bounds["p05"][up_axis]) + high = float(bounds["p95"][up_axis]) + span = max(high - low, 1e-6) + coordinate = float(np.percentile(up_values, 10)) + if coordinate > low + span * 0.18: + return None + near = np.abs(up_values - coordinate) <= max(span * 0.025, 0.035) + if not np.any(near): + return None + near_points = centroids[upward][near] + near_area = float(np.sum(up_areas[near])) + area_ratio = near_area / total_area if total_area > 0 else 0.0 + if area_ratio < 0.012 or near_points.shape[0] < 80: + return None + extent = { + name: [float(np.percentile(near_points[:, idx], 5)), float(np.percentile(near_points[:, idx], 95))] + for idx, name in enumerate("xyz") + } + return { + "semantic_label": "floor", + "axis": "xyz"[up_axis], + "coordinate": round(coordinate, 6), + "area": near_area, + "area_ratio": round(float(area_ratio), 6), + "face_count": int(near_points.shape[0]), + "extent": extent, + "recovery_method": "low_horizontal_support_quantile", + "recovery_confidence": round(min(1.0, area_ratio / 0.05), 6), + "collision_proxy_hint": collision_proxy_hint( + label="floor", + axis=up_axis, + coordinate=coordinate, + extent=extent, + bounds=bounds, + ), + } + + +def clustered_axis_planes( + *, + axis: int, + normals: np.ndarray, + centroids: np.ndarray, + areas: np.ndarray, + threshold: float, + cluster_tolerance: float, +) -> list[dict[str, Any]]: + aligned = np.abs(normals[:, axis]) >= threshold + if not np.any(aligned): + return [] + aligned_centroids = centroids[aligned] + aligned_areas = areas[aligned] + order = np.argsort(aligned_centroids[:, axis]) + sorted_centroids = aligned_centroids[order] + sorted_areas = aligned_areas[order] + clusters: list[dict[str, Any]] = [] + start = 0 + for index in range(1, sorted_centroids.shape[0] + 1): + boundary = index == sorted_centroids.shape[0] + if not boundary: + boundary = float(sorted_centroids[index, axis] - sorted_centroids[index - 1, axis]) > cluster_tolerance + if boundary: + points = sorted_centroids[start:index] + weights = sorted_areas[start:index] + area = float(np.sum(weights)) + if points.size > 0 and area > 0: + clusters.append( + { + "coordinate": float(np.average(points[:, axis], weights=weights)), + "area": area, + "area_ratio": area / float(np.sum(areas)) if np.sum(areas) > 0 else 0.0, + "face_count": int(points.shape[0]), + "extent": { + name: [float(np.percentile(points[:, idx], 5)), float(np.percentile(points[:, idx], 95))] + for idx, name in enumerate("xyz") + }, + } + ) + start = index + clusters.sort(key=lambda item: item["area"], reverse=True) + return clusters[:8] + + +def semantic_label_for_plane(*, axis: int, up_axis: int, coordinate: float, bounds: dict[str, list[float]]) -> str: + low = float(bounds["p05"][axis]) + high = float(bounds["p95"][axis]) + span = max(high - low, 1e-6) + if axis == up_axis: + if coordinate <= low + span * 0.12: + return "floor" + if coordinate >= high - span * 0.12: + return "ceiling" + return "support_surface" + if coordinate <= low + span * 0.18: + return f"wall_{'xyz'[axis]}_low" + if coordinate >= high - span * 0.18: + return f"wall_{'xyz'[axis]}_high" + return "vertical_partition" + + +def collision_proxy_hint( + *, + label: str, + axis: int, + coordinate: float, + extent: dict[str, list[float]], + bounds: dict[str, list[float]], +) -> dict[str, Any]: + center = [(float(extent[name][0]) + float(extent[name][1])) * 0.5 for name in "xyz"] + size = [max(float(extent[name][1]) - float(extent[name][0]), 0.02) for name in "xyz"] + center[axis] = coordinate + size[axis] = max(float(bounds["p95"][axis] - bounds["p05"][axis]) * 0.01, 0.02) + return { + "type": "box", + "label": label, + "center": [round(value, 6) for value in center], + "size": [round(value, 6) for value in size], + } + + +def semantic_priority(label: str) -> int: + if label == "floor": + return 0 + if label.startswith("wall_"): + return 1 + if label == "support_surface": + return 2 + if label == "ceiling": + return 3 + return 4 + + +def semantic_structure_score(planes: list[dict[str, Any]]) -> float: + has_floor = any(plane["semantic_label"] == "floor" for plane in planes) + wall_count = sum(1 for plane in planes if plane["semantic_label"].startswith("wall_")) + support_count = sum(1 for plane in planes if plane["semantic_label"] == "support_surface") + return min(1.0, (0.35 if has_floor else 0.0) + min(wall_count, 4) * 0.12 + min(support_count, 3) * 0.08) + + +def plane_summary(values: np.ndarray, areas: np.ndarray, *, quantile: float) -> dict[str, Any] | None: + if values.size == 0: + return None + value = float(np.percentile(values, quantile)) + span = max(float(np.percentile(values, 95) - np.percentile(values, 5)), 1e-6) + near = np.abs(values - value) <= max(span * 0.04, 0.02) + near_area = float(np.sum(areas[near])) if np.any(near) else 0.0 + return { + "coordinate": value, + "quantile": float(quantile), + "near_face_count": int(np.count_nonzero(near)), + "near_area": near_area, + } + + +def read_obj_triangles(path: Path) -> tuple[np.ndarray, np.ndarray]: + vertices: list[tuple[float, float, float]] = [] + faces: list[tuple[int, int, int]] = [] + for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + if raw.startswith("v "): + parts = raw.split() + if len(parts) >= 4: + vertices.append((float(parts[1]), float(parts[2]), float(parts[3]))) + elif raw.startswith("f "): + refs = [parse_vertex_ref(token) for token in raw.split()[1:]] + for idx in range(1, len(refs) - 1): + faces.append((refs[0], refs[idx], refs[idx + 1])) + return np.asarray(vertices, dtype=np.float64), np.asarray(faces, dtype=np.int64) + + +def parse_vertex_ref(token: str) -> int: + return int(token.split("/")[0]) - 1 + + +def triangle_normals(points: np.ndarray) -> np.ndarray: + normals = np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]) + norms = np.linalg.norm(normals, axis=1, keepdims=True) + return normals / np.maximum(norms, 1e-9) + + +def triangle_areas(points: np.ndarray) -> np.ndarray: + return np.linalg.norm(np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]), axis=1) * 0.5 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_one_image_motrix_scene.py b/vendor/gs_playground/scripts/real2sim_one_image_motrix_scene.py new file mode 100644 index 0000000..d02cc92 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_one_image_motrix_scene.py @@ -0,0 +1,910 @@ +#!/usr/bin/env python3 +"""One-image Real2Sim scene reconstruction workflow for MotrixSim. + +This script is intentionally orchestration-only: it does not store API keys and +does not replace the project CLI. It chains the existing image-edit multiview +adapter, scene generation CLI, package verification, preview projection, and +archive creation into one reproducible command. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tarfile +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image, ImageDraw + +PIL_BILINEAR = getattr(getattr(Image, "Resampling", Image), "BILINEAR") + +REPO_ROOT = Path(__file__).resolve().parents[1] +if REPO_ROOT.as_posix() not in sys.path: + sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.camera import load_anysplat_source_camera + + +DEFAULT_PROMPT = ( + "Generate a reconstruction-grade, geometry-consistent 8-view image-to-image contact sheet for Real2Sim and MotrixSim import. " + "Use the input photo as exact source truth. Preserve all visible object identities, relative spacing, material colors, reflections, " + "shadows, the supporting surface, static background, camera scale, and lighting. Output exactly one 4 columns x 2 rows contact sheet, " + "8 equal portrait panels, thin white gutters, no labels, no captions, no extra text, no watermarks. Views in order: original front, " + "front-left 45 degrees, left 90 degrees, rear-left 135 degrees, rear/back 180 degrees, rear-right 225 degrees, right 270 degrees, " + "top oblique 35 degrees. Do not add people, hands, robots, extra objects, UI, a different room, or a product render background. " + "Prefer sharp contours and reconstruction-useful texture over artistic beautification." +) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate and verify a MotrixSim-importable scene package from one image") + parser.add_argument("--image", type=Path, required=True, help="Source RGB image") + parser.add_argument("--output", type=Path, required=True, help="Output Real2Sim scene package directory") + parser.add_argument("--work-dir", type=Path, required=True, help="Intermediate workflow directory") + parser.add_argument("--prompt", default=DEFAULT_PROMPT, help="Image-to-image multiview prompt") + parser.add_argument("--scene-prompt", default="", help="Scene reconstruction prompt; defaults to --prompt") + parser.add_argument("--config", type=Path, default=REPO_ROOT / "configs/real2sim.external.example.json") + parser.add_argument("--backend", default="builtin", choices=["builtin", "external", "hybrid", "auto", "anysplat-direct"]) + parser.add_argument("--api-key-env", default="REAL2SIM_IMAGE_EDIT_API_KEY") + parser.add_argument("--api-url", default=os.environ.get("REAL2SIM_IMAGE_EDIT_API_URL", "https://chatapi.applyma.com/v1/images/edits")) + parser.add_argument("--model", default=os.environ.get("REAL2SIM_IMAGE_EDIT_MODEL", "gpt-image-2")) + parser.add_argument("--size", default=os.environ.get("REAL2SIM_IMAGE_EDIT_SIZE", "1536x1024")) + parser.add_argument("--quality", default=os.environ.get("REAL2SIM_IMAGE_EDIT_QUALITY", "auto")) + parser.add_argument( + "--candidates", + type=int, + default=int(os.environ.get("REAL2SIM_IMAGE_EDIT_CANDIDATES", "1")), + help="Generate and score multiple image-edit multiview candidates, then reconstruct the best one", + ) + parser.add_argument( + "--min-quality-score", + type=float, + default=float(os.environ.get("REAL2SIM_MIN_MULTIVIEW_QUALITY", "0.42")), + help="Fail before reconstruction when synthetic multiview quality is below this score", + ) + parser.add_argument( + "--allow-low-quality", + action="store_true", + help="Continue even when the best synthetic multiview candidate is below --min-quality-score", + ) + parser.add_argument("--frames-dir", type=Path, default=None, help="Reuse existing multiview frames instead of calling the image API") + parser.add_argument("--no-image-edit", action="store_true", help="Require --frames-dir and skip the image-edit API call") + parser.add_argument("--no-prune", action="store_true", help="Pass --no-prune to scene generation") + parser.add_argument("--anysplat-python", default=os.environ.get("REAL2SIM_ANY_SPLAT_PYTHON", ""), help="Python executable for AnySplat direct mode") + parser.add_argument("--anysplat-root", type=Path, default=os.environ.get("REAL2SIM_ANYSPLAT_ROOT"), help="AnySplat repository root for direct mode") + parser.add_argument("--anysplat-max-views", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MAX_VIEWS", "8"))) + parser.add_argument("--anysplat-mesh-grid", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MESH_GRID", "48"))) + parser.add_argument("--source-gs-preview", type=Path, default=None, help="Optional source-view 3DGS preview PNG to include in the Motrix package") + parser.add_argument( + "--min-source-preview-psnr", + type=float, + default=float(os.environ.get("REAL2SIM_MIN_SOURCE_PREVIEW_PSNR", "18.0")), + help="Fail AnySplat direct mode when source-view render PSNR is below this threshold", + ) + parser.add_argument( + "--max-source-preview-mae", + type=float, + default=float(os.environ.get("REAL2SIM_MAX_SOURCE_PREVIEW_MAE", "0.10")), + help="Fail AnySplat direct mode when source-view render MAE is above this threshold", + ) + parser.add_argument( + "--min-scene-vertices", + type=int, + default=int(os.environ.get("REAL2SIM_MIN_SCENE_VERTICES", "100000")), + help="Fail final package verification when scene.ply has fewer vertices", + ) + parser.add_argument( + "--skip-final-package-verify", + action="store_true", + help="Skip final Motrix package archive verification", + ) + parser.add_argument("--archive", type=Path, default=None, help="Optional .tar.gz path; defaults to .tar.gz") + args = parser.parse_args() + + if not args.image.is_file(): + raise SystemExit(f"source image not found: {args.image}") + args.work_dir.mkdir(parents=True, exist_ok=True) + + frames_dir = args.frames_dir + multiview_dir = args.work_dir / "synthetic_multiview" + quality_report = args.work_dir / "synthetic_multiview_quality.json" + if args.no_image_edit: + if frames_dir is None: + raise SystemExit("--no-image-edit requires --frames-dir") + elif frames_dir is None: + frames_dir = run_best_image_edit_candidate(args, multiview_dir, quality_report) + + if frames_dir is None or not frames_dir.is_dir(): + raise SystemExit(f"frames directory not found: {frames_dir}") + if not quality_report.exists(): + write_multiview_quality_report(args.image, frames_dir, quality_report) + + anysplat_metadata = None + if args.backend == "anysplat-direct": + anysplat_metadata = run_anysplat_direct_generation(args, frames_dir) + else: + run_scene_generation(args, frames_dir) + report = verify_motrix_package(args.output) + preview = write_projection_preview(args.output / "3dgs" / "scene.ply", args.output / "pointcloud_projection_preview.png") + if args.backend == "anysplat-direct" and (args.output / "motrix_import.json").is_file(): + import_files = { + "manifest": args.output / "motrix_import.json", + "readme": args.output / "MOTRIX_IMPORT.md", + } + copy_quality_report(args.output, quality_report) + if anysplat_metadata is not None: + attach_anysplat_metadata_to_package(args.output, anysplat_metadata) + else: + import_files = write_motrix_import_files(args.output, report, preview, quality_report) + archive = create_archive(args.output, args.archive or args.output.with_suffix(".tar.gz")) + final_verification = None + if not args.skip_final_package_verify and args.backend == "anysplat-direct": + final_verification = run_final_package_verification(args, archive) + archive = create_archive(args.output, archive) + + print( + json.dumps( + { + "package": report, + "preview": preview.as_posix(), + "motrix_import": import_files["manifest"].as_posix(), + "motrix_readme": import_files["readme"].as_posix(), + "archive": archive.as_posix(), + "anysplat_metadata": (args.output / "artifacts" / "anysplat_metadata.json").as_posix() + if (args.output / "artifacts" / "anysplat_metadata.json").is_file() + else None, + "final_package_verification": final_verification.as_posix() if final_verification is not None else None, + }, + indent=2, + ensure_ascii=False, + ) + ) + return 0 + + +def run_final_package_verification(args: argparse.Namespace, archive: Path) -> Path: + report_path = args.output / "motrix_package_verification.json" + command = [ + sys.executable, + "scripts/real2sim_verify_motrix_package.py", + "--package", + archive.as_posix(), + "--min-scene-vertices", + str(args.min_scene_vertices), + "--min-source-preview-psnr", + str(args.min_source_preview_psnr), + "--max-source-preview-mae", + str(args.max_source_preview_mae), + ] + completed = subprocess.run(command, cwd=REPO_ROOT, text=True, capture_output=True) + if completed.returncode != 0: + if completed.stdout: + print(completed.stdout, file=sys.stderr) + if completed.stderr: + print(completed.stderr, file=sys.stderr) + raise SystemExit(completed.returncode) + report_path.write_text(completed.stdout, encoding="utf-8") + return report_path + + +def run_best_image_edit_candidate(args: argparse.Namespace, multiview_dir: Path, quality_report: Path) -> Path: + if args.candidates <= 0: + raise SystemExit("--candidates must be positive") + candidate_reports: list[dict[str, Any]] = [] + best_report: dict[str, Any] | None = None + best_frames: Path | None = None + for index in range(args.candidates): + suffix = f"candidate_{index + 1:02d}" if args.candidates > 1 else "candidate_01" + candidate_dir = multiview_dir / suffix + frames_dir = candidate_dir / "frames" + run_image_edit(args, candidate_dir, frames_dir) + report = score_multiview_frames(args.image, frames_dir) + report["candidate"] = index + 1 + report["frames_dir"] = frames_dir.as_posix() + report["contact_sheet"] = (candidate_dir / "contact_sheet.png").as_posix() + candidate_reports.append(report) + if best_report is None or float(report["score"]) > float(best_report["score"]): + best_report = report + best_frames = frames_dir + + if best_report is None or best_frames is None: + raise SystemExit("image-edit API produced no synthetic multiview candidates") + quality = { + "selected_candidate": best_report["candidate"], + "selected_frames_dir": best_frames.as_posix(), + "min_quality_score": args.min_quality_score, + "passed": float(best_report["score"]) >= args.min_quality_score, + "candidates": candidate_reports, + } + quality_report.parent.mkdir(parents=True, exist_ok=True) + quality_report.write_text(json.dumps(quality, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + if not quality["passed"] and not args.allow_low_quality: + raise SystemExit( + "synthetic multiview quality gate failed: " + f"best score {float(best_report['score']):.3f} < {args.min_quality_score:.3f}; " + "inspect synthetic_multiview_quality.json or rerun with --allow-low-quality" + ) + return best_frames + + +def run_image_edit(args: argparse.Namespace, multiview_dir: Path, frames_dir: Path) -> None: + api_key = os.environ.get(args.api_key_env) + if not api_key: + raise SystemExit(f"missing image-edit API key: set {args.api_key_env}") + command = [ + sys.executable, + "scripts/real2sim_external/synthetic_multiview_expand.py", + "--image", + args.image.as_posix(), + "--output-dir", + multiview_dir.as_posix(), + "--frames-dir", + frames_dir.as_posix(), + "--mode", + "scene", + "--api-url", + args.api_url, + "--model", + args.model, + "--size", + args.size, + "--quality", + args.quality, + "--prompt", + args.prompt, + ] + env = os.environ.copy() + env["REAL2SIM_IMAGE_EDIT_API_KEY"] = api_key + run(command, env=env) + + +def write_multiview_quality_report(source_image: Path, frames_dir: Path, output: Path) -> None: + report = score_multiview_frames(source_image, frames_dir) + quality = { + "selected_candidate": None, + "selected_frames_dir": frames_dir.as_posix(), + "min_quality_score": None, + "passed": True, + "candidates": [report], + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(quality, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def score_multiview_frames(source_image: Path, frames_dir: Path) -> dict[str, Any]: + frame_paths = sorted( + path + for path in frames_dir.iterdir() + if path.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".bmp"} + ) + frames = [Image.open(path).convert("RGB") for path in frame_paths] + source = Image.open(source_image).convert("RGB") + + frame_count_score = min(len(frames), 8) / 8.0 + resolution_score = 0.0 + exposure_score = 0.0 + sharpness_score = 0.0 + if frames: + resolution_score = float(np.mean([min((img.width * img.height) / (360 * 360), 1.0) for img in frames])) + exposure_score = float(np.mean([exposure_quality(img) for img in frames])) + sharpness_score = float(np.mean([sharpness_quality(img) for img in frames])) + source_similarity = first_view_similarity(source, frames[0]) if frames else 0.0 + diversity_score = view_diversity(frames) + + score = ( + 0.20 * frame_count_score + + 0.12 * resolution_score + + 0.18 * source_similarity + + 0.22 * diversity_score + + 0.18 * sharpness_score + + 0.10 * exposure_score + ) + return { + "score": round(float(score), 4), + "frame_count": len(frames), + "frame_count_score": round(float(frame_count_score), 4), + "resolution_score": round(float(resolution_score), 4), + "source_similarity": round(float(source_similarity), 4), + "view_diversity": round(float(diversity_score), 4), + "sharpness_score": round(float(sharpness_score), 4), + "exposure_score": round(float(exposure_score), 4), + "frames": [path.as_posix() for path in frame_paths], + "warnings": multiview_quality_warnings( + frame_count=len(frames), + source_similarity=source_similarity, + diversity_score=diversity_score, + sharpness_score=sharpness_score, + exposure_score=exposure_score, + ), + } + + +def image_array(image: Image.Image, size: tuple[int, int] = (160, 120)) -> np.ndarray: + return np.asarray(image.resize(size, PIL_BILINEAR), dtype=np.float32) / 255.0 + + +def first_view_similarity(source: Image.Image, first_view: Image.Image) -> float: + src = image_array(source) + view = image_array(first_view) + mean_abs = float(np.mean(np.abs(src - view))) + color_delta = float(np.mean(np.abs(src.mean(axis=(0, 1)) - view.mean(axis=(0, 1))))) + return max(0.0, min(1.0, 1.0 - (1.7 * mean_abs + 0.8 * color_delta))) + + +def view_diversity(frames: list[Image.Image]) -> float: + if len(frames) < 2: + return 0.0 + arrays = [image_array(frame, size=(96, 72)) for frame in frames] + diffs = [float(np.mean(np.abs(arrays[index] - arrays[index - 1]))) for index in range(1, len(arrays))] + mean_diff = float(np.mean(diffs)) + if mean_diff < 0.035: + return mean_diff / 0.035 * 0.45 + if mean_diff > 0.42: + return max(0.0, 1.0 - (mean_diff - 0.42) / 0.30) + return min(1.0, 0.45 + (mean_diff - 0.035) / (0.42 - 0.035) * 0.55) + + +def sharpness_quality(image: Image.Image) -> float: + gray = np.asarray(image.convert("L").resize((192, 144), PIL_BILINEAR), dtype=np.float32) / 255.0 + gx = np.diff(gray, axis=1) + gy = np.diff(gray, axis=0) + energy = float(np.mean(np.abs(gx)) + np.mean(np.abs(gy))) + return max(0.0, min(1.0, energy / 0.11)) + + +def exposure_quality(image: Image.Image) -> float: + gray = np.asarray(image.convert("L").resize((96, 72), PIL_BILINEAR), dtype=np.float32) / 255.0 + clipped = float(np.mean((gray < 0.025) | (gray > 0.975))) + contrast = float(np.std(gray)) + return max(0.0, min(1.0, (1.0 - clipped * 2.0) * min(contrast / 0.22, 1.0))) + + +def multiview_quality_warnings( + *, + frame_count: int, + source_similarity: float, + diversity_score: float, + sharpness_score: float, + exposure_score: float, +) -> list[str]: + warnings: list[str] = [] + if frame_count < 8: + warnings.append("expected 8 synthetic views") + if source_similarity < 0.35: + warnings.append("first synthetic view may not preserve the source image") + if diversity_score < 0.35: + warnings.append("synthetic views may be too similar for scene reconstruction") + if sharpness_score < 0.35: + warnings.append("synthetic views look blurry or low-detail") + if exposure_score < 0.35: + warnings.append("synthetic views have weak contrast or clipped exposure") + return warnings + + +def run_scene_generation(args: argparse.Namespace, frames_dir: Path) -> None: + command = [ + sys.executable, + "-m", + "real2sim", + "generate-scene", + "--input", + frames_dir.as_posix(), + "--output", + args.output.as_posix(), + "--work-dir", + (args.work_dir / "scene_work").as_posix(), + "--backend", + args.backend, + "--config", + args.config.as_posix(), + "--prompt", + args.scene_prompt or args.prompt, + ] + if args.no_prune: + command.append("--no-prune") + run(command) + + +def run_anysplat_direct_generation(args: argparse.Namespace, frames_dir: Path) -> Path | None: + scene_dir = args.work_dir / "anysplat_scene" + scene_dir.mkdir(parents=True, exist_ok=True) + scene_ply = scene_dir / "scene.ply" + scene_mesh = scene_dir / "mesh.obj" + cameras_json = scene_dir / "cameras.json" + depth_map = scene_dir / "depth.pgm" + source_preview = args.source_gs_preview or (scene_dir / "source_gs_preview.png") + anysplat_python = args.anysplat_python or sys.executable + + command = [ + anysplat_python, + "scripts/real2sim_external/anysplat_scene.py", + "--image", + args.image.as_posix(), + "--frames-dir", + frames_dir.as_posix(), + "--output-dir", + scene_dir.as_posix(), + "--ply", + scene_ply.as_posix(), + "--mesh", + scene_mesh.as_posix(), + "--cameras", + cameras_json.as_posix(), + "--depth", + depth_map.as_posix(), + "--source-preview", + source_preview.as_posix(), + "--max-views", + str(args.anysplat_max_views), + "--mesh-grid", + str(max(8, int(getattr(args, "anysplat_mesh_grid", 48)))), + ] + if args.anysplat_root is not None: + command.extend(["--anysplat-root", args.anysplat_root.as_posix()]) + run(command) + metadata = scene_dir / "anysplat_metadata.json" + if metadata.exists(): + enforce_anysplat_preview_quality( + metadata, + min_psnr=args.min_source_preview_psnr, + max_mae=args.max_source_preview_mae, + allow_low_quality=args.allow_low_quality, + ) + + package_command = [ + sys.executable, + "-m", + "real2sim", + "package-existing", + "--source", + args.image.as_posix(), + "--scene-ply", + scene_ply.as_posix(), + "--scene-mesh", + scene_mesh.as_posix(), + "--cameras-json", + cameras_json.as_posix(), + "--depth-map", + depth_map.as_posix(), + "--background-image", + args.image.as_posix(), + "--output", + args.output.as_posix(), + "--prompt", + args.scene_prompt or args.prompt, + ] + package_command.extend(["--preview-image", source_preview.as_posix()]) + run(package_command) + return metadata if metadata.exists() else None + + +def copy_quality_report(package_dir: Path, quality_report: Path) -> None: + if quality_report.exists(): + quality_dest = package_dir / "synthetic_multiview_quality.json" + quality_dest.write_text(quality_report.read_text(encoding="utf-8"), encoding="utf-8") + + +def enforce_anysplat_preview_quality( + metadata_path: Path, + *, + min_psnr: float, + max_mae: float, + allow_low_quality: bool, +) -> dict[str, Any]: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + metrics = metadata.get("source_preview_metrics") + if not isinstance(metrics, dict): + raise SystemExit(f"AnySplat metadata missing source_preview_metrics: {metadata_path}") + psnr = float(metrics.get("psnr", 0.0)) + mae = float(metrics.get("mae", 1.0)) + passed = psnr >= min_psnr and mae <= max_mae + gate = { + "min_source_preview_psnr": min_psnr, + "max_source_preview_mae": max_mae, + "passed": passed, + } + metadata["source_preview_quality_gate"] = gate + metadata_path.write_text(json.dumps(metadata, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + if not passed and not allow_low_quality: + raise SystemExit( + "AnySplat source-view quality gate failed: " + f"psnr={psnr:.3f} < {min_psnr:.3f} or mae={mae:.4f} > {max_mae:.4f}; " + "rerun with --allow-low-quality to keep the package" + ) + return metadata + + +def attach_anysplat_metadata_to_package(package_dir: Path, metadata_path: Path) -> Path: + artifact_dir = package_dir / "artifacts" + artifact_dir.mkdir(parents=True, exist_ok=True) + dest = artifact_dir / "anysplat_metadata.json" + dest.write_text(metadata_path.read_text(encoding="utf-8"), encoding="utf-8") + + import_path = package_dir / "motrix_import.json" + if import_path.is_file(): + import_manifest = json.loads(import_path.read_text(encoding="utf-8")) + import_manifest["reconstruction_metadata"] = dest.relative_to(package_dir).as_posix() + metadata = json.loads(dest.read_text(encoding="utf-8")) + metrics = metadata.get("source_preview_metrics") + gate = metadata.get("source_preview_quality_gate") + if isinstance(metrics, dict): + import_manifest.setdefault("verification", {})["source_preview_metrics"] = metrics + if isinstance(gate, dict): + import_manifest.setdefault("verification", {})["source_preview_quality_gate"] = gate + import_path.write_text(json.dumps(import_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return dest + + +def verify_motrix_package(package_dir: Path) -> dict[str, Any]: + scene_xml = package_dir / "mjcf" / "scene.xml" + scene_ply = package_dir / "3dgs" / "scene.ply" + config_json = package_dir / "configs" / "go2_real2sim.json" + manifest_json = package_dir / "package_manifest.json" + cameras_json = package_dir / "artifacts" / "cameras_json.json" + for path in (scene_xml, scene_ply, config_json, manifest_json, cameras_json): + if not path.is_file() or path.stat().st_size <= 0: + raise SystemExit(f"package verification failed; missing or empty file: {path}") + + ET.parse(scene_xml) + config = json.loads(config_json.read_text(encoding="utf-8")) + manifest = json.loads(manifest_json.read_text(encoding="utf-8")) + cameras = json.loads(cameras_json.read_text(encoding="utf-8")) + load_anysplat_source_camera(cameras_json) + + scene_gaussians = config.get("scene_gaussians") + if not isinstance(scene_gaussians, dict) or "scene" not in scene_gaussians: + raise SystemExit("package verification failed; config missing scene_gaussians.scene") + gaussian_path = resolve_package_config_path(config_json, scene_gaussians["scene"]) + if not gaussian_path.is_file(): + raise SystemExit(f"package verification failed; scene Gaussian PLY does not exist: {gaussian_path}") + + manifest_scene = manifest.get("scene", {}) + manifest_ply = package_dir / str(manifest_scene.get("gaussian_ply", "")) + if not manifest_ply.is_file(): + raise SystemExit(f"package verification failed; manifest scene gaussian_ply does not exist: {manifest_ply}") + + return { + "root": package_dir.as_posix(), + "scene_xml": scene_xml.as_posix(), + "config_json": config_json.as_posix(), + "scene_ply": scene_ply.as_posix(), + "scene_ply_vertices": count_ply_vertices(scene_ply), + "capture_kind": manifest.get("metadata", {}).get("capture_kind"), + "scene_fusion": manifest.get("metadata", {}).get("builtin_scene_fusion"), + "scene_view_count": manifest.get("metadata", {}).get("builtin_scene_view_count"), + "camera_view_count": len(cameras.get("views", [])) if isinstance(cameras.get("views"), list) else 0, + } + + +def resolve_package_config_path(config_json: Path, value: Any) -> Path: + path = Path(str(value)) + if path.is_absolute(): + return path + return (config_json.parent / path).resolve() + + +def count_ply_vertices(path: Path) -> int: + with path.open("r", encoding="ascii", errors="ignore") as handle: + for line in handle: + if line.startswith("element vertex "): + return int(line.split()[-1]) + if line.strip() == "end_header": + break + raise SystemExit(f"PLY vertex count not found: {path}") + + +def write_projection_preview(ply: Path, output: Path) -> Path: + points, colors = load_ply_points_for_preview(ply, max_points=90000) + width, height = 900, 520 + canvas = Image.new("RGB", (width * 2 + 20, height), (245, 245, 245)) + draw_projection(canvas, points, colors, (0, 2), 0, width, height, "front projection: x/z") + draw_projection(canvas, points, colors, (0, 1), width + 20, width, height, "top projection: x/y") + output.parent.mkdir(parents=True, exist_ok=True) + canvas.save(output) + return output + + +def write_motrix_import_files( + package_dir: Path, + report: dict[str, Any], + preview: Path, + quality_report: Path | None = None, +) -> dict[str, Path]: + """Write explicit MotrixSim import entrypoints into the generated package.""" + manifest_path = package_dir / "motrix_import.json" + readme_path = package_dir / "MOTRIX_IMPORT.md" + package_name = package_dir.name + recommended_camera = "source_3dgs_preview" + viewer_args = [ + "--package", + f"/path/to/{package_name}", + "--camera", + "closeup", + "--gs-width", + "900", + "--gs-height", + "700", + "--static-gs-preview", + "--with-gs", + "--show-source-gs", + "--source-gs-only", + ] + import_manifest = { + "format": "gs-playground-real2sim-motrix-import-v1", + "import_type": "mjcf_scene_with_3dgs", + "scene_xml": "mjcf/scene.xml", + "config": "configs/go2_real2sim.json", + "scene_gaussians": {"scene": "3dgs/scene.ply"}, + "camera_artifacts": {"cameras_json": "artifacts/cameras_json.json"}, + "preview": preview.relative_to(package_dir).as_posix(), + "recommended_camera": recommended_camera, + "viewer_args": viewer_args, + "wuying_archive_launch": ( + "scripts/real2sim_wuying_launch_package.sh " + f"--archive /path/to/{package_name}.tar.gz " + f"--dest /tmp/{package_name}_import --with-gs" + ), + "verification": { + "scene_ply_vertices": report.get("scene_ply_vertices"), + "scene_fusion": report.get("scene_fusion"), + "scene_view_count": report.get("scene_view_count"), + "camera_view_count": report.get("camera_view_count"), + "recommended_preview": recommended_camera, + }, + } + if quality_report is not None and quality_report.exists(): + quality_dest = package_dir / "synthetic_multiview_quality.json" + quality_dest.write_text(quality_report.read_text(encoding="utf-8"), encoding="utf-8") + import_manifest["synthetic_multiview_quality"] = quality_dest.relative_to(package_dir).as_posix() + manifest_path.write_text(json.dumps(import_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + readme_path.write_text( + "\n".join( + [ + "# MotrixSim Import", + "", + "This package is a scene-level Real2Sim reconstruction from one source image plus synthetic multiview completion.", + "", + "Required entrypoints:", + "", + "- MJCF scene: `mjcf/scene.xml`", + "- Scene Gaussian asset: `3dgs/scene.ply`", + "- Viewer config: `configs/go2_real2sim.json`", + "- Source cameras: `artifacts/cameras_json.json`", + "- Projection preview: `pointcloud_projection_preview.png`", + "", + "Wuying launch command:", + "", + "```bash", + import_manifest["wuying_archive_launch"], + "```", + "", + "Direct viewer arguments:", + "", + "```bash", + " ".join(import_manifest["viewer_args"]), + "```", + "", + "Verification:", + "", + f"- scene vertices: {report.get('scene_ply_vertices')}", + f"- scene fusion: {report.get('scene_fusion')}", + f"- scene views: {report.get('scene_view_count')}", + f"- camera views: {report.get('camera_view_count')}", + "", + ] + ), + encoding="utf-8", + ) + return {"manifest": manifest_path, "readme": readme_path} + + +def load_ply_points_for_preview(path: Path, *, max_points: int) -> tuple[np.ndarray, np.ndarray]: + ply_format = read_ply_format(path) + if ply_format != "ascii": + return load_binary_ply_points_for_preview(path, max_points=max_points) + + with path.open("r", encoding="ascii", errors="ignore") as handle: + vertex_count = 0 + for line in handle: + if line.startswith("element vertex "): + vertex_count = int(line.split()[-1]) + if line.strip() == "end_header": + break + stride = max(1, vertex_count // max_points) + points = [] + colors = [] + for index, line in enumerate(handle): + if index >= vertex_count: + break + if index % stride: + continue + values = line.split() + if len(values) < 9: + continue + points.append(tuple(map(float, values[:3]))) + rgb = np.asarray(list(map(float, values[6:9])), dtype=np.float32) + rgb = 1.0 / (1.0 + np.exp(-rgb)) + colors.append((rgb * 255.0).clip(0.0, 255.0).astype(np.uint8)) + if not points: + raise SystemExit(f"no previewable points found in {path}") + return np.asarray(points, dtype=np.float32), np.asarray(colors, dtype=np.uint8) + + +def read_ply_format(path: Path) -> str: + with path.open("rb") as handle: + for raw in handle: + line = raw.decode("ascii", errors="ignore").strip() + if line.startswith("format "): + parts = line.split() + return parts[1] if len(parts) > 1 else "ascii" + if line == "end_header": + break + return "ascii" + + +def load_binary_ply_points_for_preview(path: Path, *, max_points: int) -> tuple[np.ndarray, np.ndarray]: + try: + from plyfile import PlyData + except Exception as exc: + return load_binary_ply_points_without_plyfile(path, max_points=max_points, import_error=exc) + + ply = PlyData.read(path) + vertex = ply["vertex"].data + names = set(vertex.dtype.names or ()) + required = {"x", "y", "z"} + if not required.issubset(names): + raise SystemExit(f"binary PLY missing required xyz fields: {path}") + + vertex_count = len(vertex) + stride = max(1, vertex_count // max_points) + sampled = vertex[::stride] + points = np.column_stack([sampled["x"], sampled["y"], sampled["z"]]).astype(np.float32, copy=False) + if {"red", "green", "blue"}.issubset(names): + colors = np.column_stack([sampled["red"], sampled["green"], sampled["blue"]]).astype(np.uint8, copy=False) + elif {"f_dc_0", "f_dc_1", "f_dc_2"}.issubset(names): + rgb = np.column_stack([sampled["f_dc_0"], sampled["f_dc_1"], sampled["f_dc_2"]]).astype(np.float32, copy=False) + colors = (1.0 / (1.0 + np.exp(-rgb)) * 255.0).clip(0.0, 255.0).astype(np.uint8) + else: + colors = np.full((len(points), 3), 220, dtype=np.uint8) + if len(points) == 0: + raise SystemExit(f"no previewable points found in {path}") + return points, colors + + +def load_binary_ply_points_without_plyfile( + path: Path, + *, + max_points: int, + import_error: Exception | None = None, +) -> tuple[np.ndarray, np.ndarray]: + data = path.read_bytes() + header_end = data.find(b"end_header\n") + newline_len = 1 + if header_end < 0: + header_end = data.find(b"end_header\r\n") + newline_len = 2 + if header_end < 0: + raise SystemExit(f"binary PLY missing end_header: {path}") from import_error + + header_text = data[: header_end + len("end_header")].decode("ascii", errors="ignore") + header_lines = [line.strip() for line in header_text.splitlines()] + ply_format = "binary_little_endian" + vertex_count = 0 + vertex_properties: list[tuple[str, str]] = [] + in_vertex = False + for line in header_lines: + if line.startswith("format "): + parts = line.split() + if len(parts) > 1: + ply_format = parts[1] + elif line.startswith("element "): + parts = line.split() + in_vertex = len(parts) >= 3 and parts[1] == "vertex" + if in_vertex: + vertex_count = int(parts[2]) + elif in_vertex and line.startswith("property "): + parts = line.split() + if len(parts) == 3: + vertex_properties.append((parts[1], parts[2])) + elif len(parts) > 3 and parts[1] == "list": + raise SystemExit(f"binary PLY vertex list properties are unsupported for preview: {path}") from import_error + + if vertex_count <= 0 or not vertex_properties: + raise SystemExit(f"binary PLY has no previewable vertex table: {path}") from import_error + endian = "<" if ply_format == "binary_little_endian" else ">" + dtype = np.dtype([(name, numpy_dtype_for_ply_type(kind, endian)) for kind, name in vertex_properties]) + offset = header_end + len("end_header") + newline_len + vertex = np.frombuffer(data, dtype=dtype, count=vertex_count, offset=offset) + names = set(vertex.dtype.names or ()) + if not {"x", "y", "z"}.issubset(names): + raise SystemExit(f"binary PLY missing required xyz fields: {path}") from import_error + + stride = max(1, vertex_count // max_points) + sampled = vertex[::stride] + points = np.column_stack([sampled["x"], sampled["y"], sampled["z"]]).astype(np.float32, copy=False) + if {"red", "green", "blue"}.issubset(names): + colors = np.column_stack([sampled["red"], sampled["green"], sampled["blue"]]).astype(np.uint8, copy=False) + elif {"f_dc_0", "f_dc_1", "f_dc_2"}.issubset(names): + rgb = np.column_stack([sampled["f_dc_0"], sampled["f_dc_1"], sampled["f_dc_2"]]).astype(np.float32, copy=False) + colors = (1.0 / (1.0 + np.exp(-rgb)) * 255.0).clip(0.0, 255.0).astype(np.uint8) + else: + colors = np.full((len(points), 3), 220, dtype=np.uint8) + return points, colors + + +def numpy_dtype_for_ply_type(kind: str, endian: str) -> str: + mapping = { + "char": "i1", + "int8": "i1", + "uchar": "u1", + "uint8": "u1", + "short": "i2", + "int16": "i2", + "ushort": "u2", + "uint16": "u2", + "int": "i4", + "int32": "i4", + "uint": "u4", + "uint32": "u4", + "float": "f4", + "float32": "f4", + "double": "f8", + "float64": "f8", + } + if kind not in mapping: + raise SystemExit(f"unsupported binary PLY property type: {kind}") + code = mapping[kind] + return code if code.endswith("1") else endian + code + + +def draw_projection( + canvas: Image.Image, + points: np.ndarray, + colors: np.ndarray, + axes: tuple[int, int], + offset: int, + width: int, + height: int, + title: str, +) -> None: + sub = Image.new("RGB", (width, height), (250, 250, 250)) + draw = ImageDraw.Draw(sub) + xy = points[:, list(axes)] + low = xy.min(axis=0) + high = xy.max(axis=0) + span = np.maximum(high - low, 1e-6) + uv = (xy - low) / span + px = (uv[:, 0] * (width - 60) + 30).astype(int) + py = ((1.0 - uv[:, 1]) * (height - 60) + 30).astype(int) + order = np.argsort(points[:, 1] if axes != (0, 1) else points[:, 2]) + for index in order: + x = int(px[index]) + y = int(py[index]) + if 0 <= x < width and 0 <= y < height: + sub.putpixel((x, y), tuple(int(v) for v in colors[index])) + draw.rectangle((0, 0, width - 1, height - 1), outline=(190, 190, 190)) + draw.text((14, 12), title, fill=(30, 30, 30)) + canvas.paste(sub, (offset, 0)) + + +def create_archive(package_dir: Path, archive: Path) -> Path: + archive.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive, "w:gz") as tar: + tar.add(package_dir, arcname=package_dir.name) + return archive + + +def run(command: list[str], *, env: dict[str, str] | None = None) -> None: + result = subprocess.run(command, cwd=REPO_ROOT, env=env, text=True) + if result.returncode != 0: + raise SystemExit(result.returncode) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_ply_mesh_to_obj.py b/vendor/gs_playground/scripts/real2sim_ply_mesh_to_obj.py new file mode 100644 index 0000000..20f266a --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_ply_mesh_to_obj.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +"""Convert a colored PLY mesh to a Motrix-friendly OBJ. + +The converter keeps vertex colors for tools that understand them and also emits +an OBJ material palette so viewers that ignore OBJ vertex colors still show the +scene with explicit diffuse materials. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import struct +from dataclasses import dataclass +from pathlib import Path +from typing import Any, BinaryIO + + +SCALAR_TYPES: dict[str, tuple[str, int]] = { + "char": ("b", 1), + "int8": ("b", 1), + "uchar": ("B", 1), + "uint8": ("B", 1), + "short": ("h", 2), + "int16": ("h", 2), + "ushort": ("H", 2), + "uint16": ("H", 2), + "int": ("i", 4), + "int32": ("i", 4), + "uint": ("I", 4), + "uint32": ("I", 4), + "float": ("f", 4), + "float32": ("f", 4), + "double": ("d", 8), + "float64": ("d", 8), +} + +COLOR_NAMES = ( + ("red", "green", "blue"), + ("diffuse_red", "diffuse_green", "diffuse_blue"), + ("r", "g", "b"), +) + + +@dataclass(frozen=True) +class PlyHeader: + ply_format: str + vertex_count: int + face_count: int + vertex_properties: tuple[tuple[str, str], ...] + face_count_type: str + face_index_type: str + data_offset: int + + +@dataclass(frozen=True) +class ObjVertex: + x: float + y: float + z: float + r: float + g: float + b: float + + +@dataclass(frozen=True) +class ObjTriangle: + a: int + b: int + c: int + + +def main() -> int: + parser = argparse.ArgumentParser(description="Convert a colored PLY mesh to a Motrix-compatible OBJ") + parser.add_argument("--input-ply", type=Path, required=True) + parser.add_argument("--output-obj", type=Path, required=True) + parser.add_argument("--report", type=Path, default=None) + parser.add_argument("--no-material-palette", action="store_true", help="Write vertex-color-only OBJ without a sidecar MTL") + parser.add_argument("--material-palette-levels", type=int, default=18, help="RGB quantization levels for generated diffuse materials") + args = parser.parse_args() + + report = convert_ply_mesh_to_obj( + args.input_ply, + args.output_obj, + material_palette=not args.no_material_palette, + material_palette_levels=max(int(args.material_palette_levels), 2), + ) + if args.report is not None: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + else: + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + + +def convert_ply_mesh_to_obj( + input_ply: Path, + output_obj: Path, + *, + material_palette: bool = True, + material_palette_levels: int = 18, +) -> dict[str, Any]: + header = read_ply_header(input_ply) + output_obj.parent.mkdir(parents=True, exist_ok=True) + with input_ply.open("rb") as src: + src.seek(header.data_offset) + if header.ply_format == "ascii": + vertices, triangles = read_ascii_mesh(src, header) + elif header.ply_format == "binary_little_endian": + vertices, triangles = read_binary_mesh(src, header, endian="<") + elif header.ply_format == "binary_big_endian": + vertices, triangles = read_binary_mesh(src, header, endian=">") + else: + raise SystemExit(f"unsupported PLY format: {header.ply_format}") + material_count = write_obj_mesh( + input_ply=input_ply, + output_obj=output_obj, + vertices=vertices, + triangles=triangles, + material_palette=material_palette, + material_palette_levels=max(int(material_palette_levels), 2), + ) + return { + "format": "gs-playground-real2sim-ply-mesh-to-obj-report-v1", + "input_ply": input_ply.resolve().as_posix(), + "output_obj": output_obj.resolve().as_posix(), + "vertex_count": int(header.vertex_count), + "face_count": int(len(triangles)), + "texcoord_count": int(len(vertices)), + "material_palette": bool(material_palette), + "material_palette_levels": int(max(int(material_palette_levels), 2)), + "material_count": int(material_count), + "material_library": output_obj.with_suffix(".mtl").resolve().as_posix() if material_palette else None, + } + + +def read_ply_header(path: Path) -> PlyHeader: + ply_format = "ascii" + vertex_count = 0 + face_count = 0 + vertex_properties: list[tuple[str, str]] = [] + face_count_type = "uchar" + face_index_type = "int" + active_element: str | None = None + with path.open("rb") as handle: + first = handle.readline().decode("ascii", errors="ignore").strip() + if first != "ply": + raise SystemExit(f"not a PLY file: {path}") + while True: + raw = handle.readline() + if not raw: + raise SystemExit(f"PLY header missing end_header: {path}") + line = raw.decode("ascii", errors="ignore").strip() + if line == "end_header": + data_offset = handle.tell() + break + parts = line.split() + if not parts: + continue + if parts[0] == "format" and len(parts) >= 2: + ply_format = parts[1] + elif parts[0] == "element" and len(parts) >= 3: + active_element = parts[1] + if active_element == "vertex": + vertex_count = int(parts[2]) + elif active_element == "face": + face_count = int(parts[2]) + elif parts[0] == "property" and active_element == "vertex" and len(parts) == 3: + vertex_properties.append((parts[2], parts[1])) + elif parts[0] == "property" and active_element == "face" and len(parts) >= 5 and parts[1] == "list": + face_count_type = parts[2] + face_index_type = parts[3] + if vertex_count <= 0: + raise SystemExit(f"PLY has no vertices: {path}") + return PlyHeader( + ply_format=ply_format, + vertex_count=vertex_count, + face_count=face_count, + vertex_properties=tuple(vertex_properties), + face_count_type=face_count_type, + face_index_type=face_index_type, + data_offset=data_offset, + ) + + +def read_ascii_mesh(src: BinaryIO, header: PlyHeader) -> tuple[list[ObjVertex], list[ObjTriangle]]: + property_names = [name for name, _kind in header.vertex_properties] + index = {name: idx for idx, name in enumerate(property_names)} + color_indices = find_color_indices(index) + vertices: list[ObjVertex] = [] + for _ in range(header.vertex_count): + parts = src.readline().decode("utf-8", errors="ignore").split() + if len(parts) < len(property_names): + raise SystemExit("truncated ASCII PLY vertex table") + vertices.append(make_obj_vertex(parts, index, color_indices)) + triangles: list[ObjTriangle] = [] + for _ in range(header.face_count): + parts = src.readline().decode("utf-8", errors="ignore").split() + if not parts: + continue + count = int(parts[0]) + indices = [int(value) + 1 for value in parts[1 : 1 + count]] + triangles.extend(triangulate(indices)) + return vertices, triangles + + +def read_binary_mesh(src: BinaryIO, header: PlyHeader, *, endian: str) -> tuple[list[ObjVertex], list[ObjTriangle]]: + property_names = [name for name, _kind in header.vertex_properties] + index = {name: idx for idx, name in enumerate(property_names)} + color_indices = find_color_indices(index) + vertex_struct = struct.Struct(endian + "".join(scalar_format(kind) for _name, kind in header.vertex_properties)) + vertices: list[ObjVertex] = [] + for _ in range(header.vertex_count): + data = src.read(vertex_struct.size) + if len(data) != vertex_struct.size: + raise SystemExit("truncated binary PLY vertex table") + values = vertex_struct.unpack(data) + vertices.append(make_obj_vertex(values, index, color_indices)) + count_struct = struct.Struct(endian + scalar_format(header.face_count_type)) + index_struct = struct.Struct(endian + scalar_format(header.face_index_type)) + triangles: list[ObjTriangle] = [] + for _ in range(header.face_count): + raw_count = src.read(count_struct.size) + if len(raw_count) != count_struct.size: + raise SystemExit("truncated binary PLY face table") + count = int(count_struct.unpack(raw_count)[0]) + raw_indices = src.read(index_struct.size * count) + if len(raw_indices) != index_struct.size * count: + raise SystemExit("truncated binary PLY face indices") + indices = [ + int(index_struct.unpack_from(raw_indices, offset)[0]) + 1 + for offset in range(0, len(raw_indices), index_struct.size) + ] + triangles.extend(triangulate(indices)) + return vertices, triangles + + +def scalar_format(kind: str) -> str: + if kind not in SCALAR_TYPES: + raise SystemExit(f"unsupported PLY scalar type: {kind}") + return SCALAR_TYPES[kind][0] + + +def find_color_indices(index: dict[str, int]) -> tuple[int, int, int] | None: + for names in COLOR_NAMES: + if all(name in index for name in names): + return tuple(index[name] for name in names) + return None + + +def make_obj_vertex(values: Any, index: dict[str, int], color_indices: tuple[int, int, int] | None) -> ObjVertex: + x = float(values[index["x"]]) + y = float(values[index["y"]]) + z = float(values[index["z"]]) + if color_indices is None: + r, g, b = 0.72, 0.72, 0.72 + else: + channels = [float(values[idx]) for idx in color_indices] + if max(channels) > 1.0: + channels = [value / 255.0 for value in channels] + r, g, b = [min(max(value, 0.0), 1.0) for value in channels] + return ObjVertex(x=x, y=y, z=z, r=r, g=g, b=b) + + +def triangulate(indices: list[int]) -> list[ObjTriangle]: + if len(indices) < 3: + return [] + triangles: list[ObjTriangle] = [] + for idx in range(1, len(indices) - 1): + triangles.append(ObjTriangle(indices[0], indices[idx], indices[idx + 1])) + return triangles + + +def write_obj_mesh( + *, + input_ply: Path, + output_obj: Path, + vertices: list[ObjVertex], + triangles: list[ObjTriangle], + material_palette: bool, + material_palette_levels: int, +) -> int: + material_path = output_obj.with_suffix(".mtl") + uv_axes = choose_uv_axes(vertices) + uv_bounds = axis_bounds(vertices, uv_axes) + materials: dict[str, tuple[float, float, float]] = {} + with output_obj.open("w", encoding="utf-8") as dst: + dst.write(f"# Converted from {input_ply.name} for Real2Sim Motrix scene mesh\n") + if material_palette: + dst.write(f"mtllib {material_path.name}\n") + dst.write("o real2sim_colmap_dense_scene\n") + for vertex in vertices: + dst.write( + f"v {vertex.x:.9g} {vertex.y:.9g} {vertex.z:.9g} " + f"{vertex.r:.6f} {vertex.g:.6f} {vertex.b:.6f}\n" + ) + for vertex in vertices: + u, v = vertex_uv(vertex, uv_axes, uv_bounds) + dst.write(f"vt {u:.7g} {v:.7g}\n") + current_material = None + for triangle in triangles: + if material_palette: + material_name, color = triangle_material(vertices, triangle, output_obj.stem, material_palette_levels) + materials.setdefault(material_name, color) + if material_name != current_material: + dst.write(f"usemtl {material_name}\n") + current_material = material_name + dst.write(f"f {triangle.a}/{triangle.a} {triangle.b}/{triangle.b} {triangle.c}/{triangle.c}\n") + if material_palette: + write_material_library(material_path, materials) + elif material_path.exists(): + material_path.unlink() + return len(materials) + + +def choose_uv_axes(vertices: list[ObjVertex]) -> tuple[str, str]: + if not vertices: + return ("x", "y") + extents = {} + for axis in ("x", "y", "z"): + values = [getattr(vertex, axis) for vertex in vertices] + extents[axis] = max(values) - min(values) + ranked = sorted(extents, key=lambda axis: extents[axis], reverse=True) + return ranked[0], ranked[1] + + +def axis_bounds(vertices: list[ObjVertex], axes: tuple[str, str]) -> tuple[tuple[float, float], tuple[float, float]]: + bounds: list[tuple[float, float]] = [] + for axis in axes: + values = [getattr(vertex, axis) for vertex in vertices] + low = min(values) if values else 0.0 + high = max(values) if values else 1.0 + if math.isclose(high, low): + high = low + 1.0 + bounds.append((low, high)) + return bounds[0], bounds[1] + + +def vertex_uv(vertex: ObjVertex, axes: tuple[str, str], bounds: tuple[tuple[float, float], tuple[float, float]]) -> tuple[float, float]: + first, second = axes + (u_low, u_high), (v_low, v_high) = bounds + u = (getattr(vertex, first) - u_low) / max(u_high - u_low, 1e-9) + v = (getattr(vertex, second) - v_low) / max(v_high - v_low, 1e-9) + return min(max(u, 0.0), 1.0), min(max(v, 0.0), 1.0) + + +def triangle_material( + vertices: list[ObjVertex], + triangle: ObjTriangle, + stem: str, + levels: int, +) -> tuple[str, tuple[float, float, float]]: + tri_vertices = [vertices[index - 1] for index in (triangle.a, triangle.b, triangle.c)] + color = tuple(sum(getattr(vertex, channel) for vertex in tri_vertices) / 3.0 for channel in ("r", "g", "b")) + quantized = tuple(round(channel * (levels - 1)) / (levels - 1) for channel in color) + suffix = "_".join(f"{int(round(channel * 255)):02x}" for channel in quantized) + return f"{safe_token(stem)}_mat_{suffix}", quantized + + +def safe_token(value: str) -> str: + token = re.sub(r"[^A-Za-z0-9_]+", "_", value.strip()) + return token.strip("_") or "scene" + + +def write_material_library(path: Path, materials: dict[str, tuple[float, float, float]]) -> None: + with path.open("w", encoding="utf-8") as dst: + dst.write("# Generated Real2Sim dense scene material palette\n") + for name, (r, g, b) in sorted(materials.items()): + dst.write(f"newmtl {name}\n") + dst.write(f"Ka {r:.6f} {g:.6f} {b:.6f}\n") + dst.write(f"Kd {r:.6f} {g:.6f} {b:.6f}\n") + dst.write("Ks 0.030000 0.030000 0.030000\n") + dst.write("Ns 8.000000\n") + dst.write("d 1.000000\n") + dst.write("illum 2\n\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_pose_init_gsplat_train.py b/vendor/gs_playground/scripts/real2sim_pose_init_gsplat_train.py new file mode 100755 index 0000000..a1a1c8a --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_pose_init_gsplat_train.py @@ -0,0 +1,1052 @@ +#!/usr/bin/env python3 +"""Train a small 3DGS scene from Real2Sim pose-init artifacts. + +The input is the dataset exported by real2sim_export_pose_init_dataset.py: + +- transforms.json with registered COLMAP camera-to-world matrices +- images/ with the registered frames +- colmap_sparse_points.ply with sparse SfM points and RGB colors + +This is intentionally dependency-light: it only needs torch, gsplat, numpy and +Pillow at runtime. It is used as a real end-to-end training backend when the +full upstream post-optimization stack is not installed on the target machine. +""" + +from __future__ import annotations + +import argparse +import json +import math +import random +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +from PIL import Image + +PIL_BILINEAR = getattr(getattr(Image, "Resampling", Image), "BILINEAR") +SH_C0 = 0.28209479177387814 + + +@dataclass +class FrameSample: + image_path: Path + pixels: np.ndarray + c2w: np.ndarray + K: np.ndarray + width: int + height: int + + +def main() -> int: + parser = argparse.ArgumentParser(description="Train pose-init 3DGS from Real2Sim COLMAP artifacts") + parser.add_argument("--pose-init-dir", type=Path, required=True, help="Directory containing transforms.json") + parser.add_argument("--output-dir", type=Path, required=True, help="Directory for trained outputs") + parser.add_argument("--output-ply", type=Path, default=None, help="Output trained 3DGS PLY") + parser.add_argument("--preview-output", type=Path, default=None, help="Output side-by-side preview PNG") + parser.add_argument("--multi-preview-output", type=Path, default=None, help="Output multi-frame source/render contact sheet") + parser.add_argument("--metrics-output", type=Path, default=None, help="Output metrics JSON") + parser.add_argument("--camera-matrix-key", default="colmap_transform_matrix") + parser.add_argument("--max-steps", type=int, default=200) + parser.add_argument("--max-frames", type=int, default=16) + parser.add_argument("--image-max-size", type=int, default=448) + parser.add_argument("--max-points", type=int, default=80_000) + parser.add_argument("--target-gaussians", type=int, default=50_000) + parser.add_argument("--dense-image-points-per-frame", type=int, default=1600) + parser.add_argument("--dense-depth-neighbors", type=int, default=4) + parser.add_argument("--dense-depth-max-distance", type=float, default=48.0) + parser.add_argument("--adaptive-densify-every", type=int, default=0, help="Add residual-driven Gaussians every N steps; 0 disables") + parser.add_argument("--adaptive-densify-start", type=int, default=50) + parser.add_argument("--adaptive-densify-stop", type=int, default=0, help="Last step for adaptive densify; 0 means max-steps") + parser.add_argument("--adaptive-densify-points", type=int, default=2500) + parser.add_argument("--adaptive-densify-error-percentile", type=float, default=82.0) + parser.add_argument("--adaptive-densify-depth-neighbors", type=int, default=4) + parser.add_argument("--adaptive-densify-max-distance", type=float, default=64.0) + parser.add_argument("--adaptive-densify-max-projection-points", type=int, default=25_000) + parser.add_argument("--preview-frames", type=int, default=4) + parser.add_argument("--trim-outlier-percentile", type=float, default=99.5) + parser.add_argument("--initial-scale", type=float, default=0.0, help="World-space Gaussian scale; <=0 auto-estimates") + parser.add_argument("--means-lr", type=float, default=0.0015) + parser.add_argument("--scale-lr", type=float, default=0.004) + parser.add_argument("--opacity-lr", type=float, default=0.02) + parser.add_argument("--color-lr", type=float, default=0.03) + parser.add_argument("--scale-reg", type=float, default=0.0005) + parser.add_argument("--opacity-reg", type=float, default=0.0002) + parser.add_argument("--device", default="cuda") + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + if args.max_steps <= 0: + raise SystemExit("--max-steps must be positive") + if args.max_frames <= 0: + raise SystemExit("--max-frames must be positive") + if args.image_max_size <= 0: + raise SystemExit("--image-max-size must be positive") + + args.output_dir.mkdir(parents=True, exist_ok=True) + output_ply = args.output_ply or (args.output_dir / "pose_init_trained_scene.ply") + preview_output = args.preview_output or (args.output_dir / "pose_init_training_preview.png") + multi_preview_output = args.multi_preview_output or (args.output_dir / "pose_init_training_contact_sheet.png") + metrics_output = args.metrics_output or (args.output_dir / "pose_init_training_metrics.json") + + set_seed(args.seed) + start = time.time() + result = train_pose_init_scene( + args.pose_init_dir, + output_ply=output_ply, + preview_output=preview_output, + multi_preview_output=multi_preview_output, + metrics_output=metrics_output, + camera_matrix_key=args.camera_matrix_key, + max_steps=args.max_steps, + max_frames=args.max_frames, + image_max_size=args.image_max_size, + max_points=args.max_points, + target_gaussians=args.target_gaussians, + dense_image_points_per_frame=args.dense_image_points_per_frame, + dense_depth_neighbors=args.dense_depth_neighbors, + dense_depth_max_distance=args.dense_depth_max_distance, + adaptive_densify_every=args.adaptive_densify_every, + adaptive_densify_start=args.adaptive_densify_start, + adaptive_densify_stop=args.adaptive_densify_stop, + adaptive_densify_points=args.adaptive_densify_points, + adaptive_densify_error_percentile=args.adaptive_densify_error_percentile, + adaptive_densify_depth_neighbors=args.adaptive_densify_depth_neighbors, + adaptive_densify_max_distance=args.adaptive_densify_max_distance, + adaptive_densify_max_projection_points=args.adaptive_densify_max_projection_points, + preview_frames=args.preview_frames, + trim_outlier_percentile=args.trim_outlier_percentile, + initial_scale=args.initial_scale, + means_lr=args.means_lr, + scale_lr=args.scale_lr, + opacity_lr=args.opacity_lr, + color_lr=args.color_lr, + scale_reg=args.scale_reg, + opacity_reg=args.opacity_reg, + device=args.device, + seed=args.seed, + ) + result["elapsed_seconds"] = round(time.time() - start, 3) + metrics_output.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 + + +def set_seed(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + + +def train_pose_init_scene( + pose_init_dir: Path, + *, + output_ply: Path, + preview_output: Path, + multi_preview_output: Path, + metrics_output: Path, + camera_matrix_key: str, + max_steps: int, + max_frames: int, + image_max_size: int, + max_points: int, + target_gaussians: int, + dense_image_points_per_frame: int, + dense_depth_neighbors: int, + dense_depth_max_distance: float, + adaptive_densify_every: int, + adaptive_densify_start: int, + adaptive_densify_stop: int, + adaptive_densify_points: int, + adaptive_densify_error_percentile: float, + adaptive_densify_depth_neighbors: int, + adaptive_densify_max_distance: float, + adaptive_densify_max_projection_points: int, + preview_frames: int, + trim_outlier_percentile: float, + initial_scale: float, + means_lr: float, + scale_lr: float, + opacity_lr: float, + color_lr: float, + scale_reg: float, + opacity_reg: float, + device: str, + seed: int, +) -> dict[str, Any]: + import torch + from gsplat import rasterization + + if device == "cuda" and not torch.cuda.is_available(): + device = "cpu" + if device == "cpu": + raise SystemExit("pose-init gsplat training requires CUDA for practical end-to-end validation") + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + frames = load_frames( + pose_init_dir, + camera_matrix_key=camera_matrix_key, + max_frames=max_frames, + image_max_size=image_max_size, + ) + points, colors = load_sparse_points(pose_init_dir / "colmap_sparse_points.ply") + points, colors, trim_report = trim_sparse_outliers(points, colors, percentile=trim_outlier_percentile) + dense_points, dense_colors, dense_report = augment_points_from_registered_images( + points, + colors, + frames, + points_per_frame=dense_image_points_per_frame, + neighbors=dense_depth_neighbors, + max_distance_px=dense_depth_max_distance, + seed=seed, + ) + if len(dense_points): + points = np.concatenate([points, dense_points], axis=0).astype(np.float32, copy=False) + colors = np.concatenate([colors, dense_colors], axis=0).astype(np.float32, copy=False) + points, colors, densify_report = densify_sparse_points( + points, + colors, + target_gaussians=max(target_gaussians, len(points)), + max_points=max_points, + seed=seed, + ) + if len(points) == 0: + raise SystemExit("pose-init sparse point cloud is empty after filtering") + + scale_value = float(initial_scale) if initial_scale > 0 else estimate_initial_scale(points) + means = torch.tensor(points, dtype=torch.float32, device=device).requires_grad_(True) + log_scales = torch.full((len(points), 3), math.log(scale_value), dtype=torch.float32, device=device, requires_grad=True) + opacity_logits = torch.full((len(points),), 0.0, dtype=torch.float32, device=device, requires_grad=True) + color_logits = torch.logit(torch.tensor(colors, dtype=torch.float32, device=device).clamp(1e-4, 1.0 - 1e-4)) + color_logits.requires_grad_(True) + quats = torch.zeros((len(points), 4), dtype=torch.float32, device=device) + quats[:, 0] = 1.0 + + frame_tensors = [ + { + "image_path": frame.image_path, + "pixels": torch.tensor(frame.pixels, dtype=torch.float32, device=device).unsqueeze(0), + "c2w": torch.tensor(frame.c2w, dtype=torch.float32, device=device).unsqueeze(0), + "K": torch.tensor(frame.K, dtype=torch.float32, device=device).unsqueeze(0), + "width": frame.width, + "height": frame.height, + } + for frame in frames + ] + + optimizer = build_optimizer( + torch, + means=means, + log_scales=log_scales, + opacity_logits=opacity_logits, + color_logits=color_logits, + means_lr=means_lr, + scale_lr=scale_lr, + opacity_lr=opacity_lr, + color_lr=color_lr, + ) + losses: list[float] = [] + psnrs: list[float] = [] + maes: list[float] = [] + adaptive_events: list[dict[str, Any]] = [] + densify_stop = max_steps if adaptive_densify_stop <= 0 else min(max_steps, adaptive_densify_stop) + adaptive_rng = np.random.default_rng(seed + 17) + for step in range(max_steps): + frame_index = step % len(frame_tensors) + frame = frame_tensors[frame_index] + optimizer.zero_grad(set_to_none=True) + render, _alpha, _info = rasterization( + means=means, + quats=quats, + scales=torch.exp(log_scales).clamp(1e-5, 1.0), + opacities=torch.sigmoid(opacity_logits), + colors=torch.sigmoid(color_logits), + viewmats=torch.linalg.inv(frame["c2w"]), + Ks=frame["K"], + width=int(frame["width"]), + height=int(frame["height"]), + packed=False, + backgrounds=torch.zeros((1, 3), dtype=torch.float32, device=device), + render_mode="RGB", + rasterize_mode="antialiased", + camera_model="pinhole", + ) + target = frame["pixels"] + l1 = torch.mean(torch.abs(render - target)) + scale_loss = torch.mean(torch.exp(log_scales)) + opacity_loss = torch.mean(torch.sigmoid(opacity_logits)) + loss = l1 + scale_reg * scale_loss + opacity_reg * opacity_loss + loss.backward() + optimizer.step() + + with torch.no_grad(): + metrics = image_metrics(render.detach(), target) + losses.append(float(loss.detach().cpu())) + psnrs.append(metrics["psnr"]) + maes.append(metrics["mae"]) + if should_adaptive_densify( + step=step + 1, + every=adaptive_densify_every, + start=adaptive_densify_start, + stop=densify_stop, + current_count=int(means.shape[0]), + max_points=max_points, + ): + event, updated = adaptive_densify_from_residual( + torch, + frame_sample=frames[frame_index], + render=render.detach(), + target=target.detach(), + means=means, + log_scales=log_scales, + opacity_logits=opacity_logits, + color_logits=color_logits, + quats=quats, + scale_value=scale_value, + max_points=max_points, + add_points=adaptive_densify_points, + error_percentile=adaptive_densify_error_percentile, + depth_neighbors=adaptive_densify_depth_neighbors, + max_distance_px=adaptive_densify_max_distance, + max_projection_points=adaptive_densify_max_projection_points, + rng=adaptive_rng, + device=device, + ) + adaptive_events.append(event) + if updated is not None: + means, log_scales, opacity_logits, color_logits, quats = updated + optimizer = build_optimizer( + torch, + means=means, + log_scales=log_scales, + opacity_logits=opacity_logits, + color_logits=color_logits, + means_lr=means_lr, + scale_lr=scale_lr, + opacity_lr=opacity_lr, + color_lr=color_lr, + ) + if step % 25 == 24 or step == max_steps - 1: + print( + f"step={step + 1}/{max_steps} loss={losses[-1]:.5f} " + f"psnr={psnrs[-1]:.3f} mae={maes[-1]:.4f} gaussians={int(means.shape[0])}", + flush=True, + ) + + with torch.no_grad(): + preview_frame = frame_tensors[0] + preview_render, _alpha, _info = rasterization( + means=means, + quats=quats, + scales=torch.exp(log_scales).clamp(1e-5, 1.0), + opacities=torch.sigmoid(opacity_logits), + colors=torch.sigmoid(color_logits), + viewmats=torch.linalg.inv(preview_frame["c2w"]), + Ks=preview_frame["K"], + width=int(preview_frame["width"]), + height=int(preview_frame["height"]), + packed=False, + backgrounds=torch.zeros((1, 3), dtype=torch.float32, device=device), + render_mode="RGB", + rasterize_mode="antialiased", + camera_model="pinhole", + ) + final_metrics = image_metrics(preview_render, preview_frame["pixels"]) + render_np = tensor_image_to_numpy(preview_render[0]) + source_np = tensor_image_to_numpy(preview_frame["pixels"][0]) + write_side_by_side_preview(source_np, render_np, preview_output) + multi_preview_metrics = write_multi_frame_preview( + rasterization, + frame_tensors, + means=means, + quats=quats, + log_scales=log_scales, + opacity_logits=opacity_logits, + color_logits=color_logits, + output=multi_preview_output, + preview_frames=preview_frames, + device=device, + ) + + write_3dgs_ply( + output_ply, + means.detach().cpu().numpy(), + torch.sigmoid(color_logits).detach().cpu().numpy(), + log_scales.detach().cpu().numpy(), + opacity_logits.detach().cpu().numpy(), + ) + + return { + "format": "real2sim_pose_init_gsplat_training_v1", + "pose_init_dir": pose_init_dir.as_posix(), + "output_ply": output_ply.as_posix(), + "preview_output": preview_output.as_posix(), + "multi_preview_output": multi_preview_output.as_posix(), + "metrics_output": metrics_output.as_posix(), + "device": device, + "max_steps": max_steps, + "frame_count": len(frames), + "source_frame_size": [frames[0].width, frames[0].height], + "input_sparse_point_count": int(trim_report["input_point_count"]), + "trimmed_sparse_point_count": int(trim_report["kept_point_count"]), + "dense_image_point_count": int(dense_report["output_count"]), + "trained_gaussian_count": int(means.shape[0]), + "initial_scale": scale_value, + "final_loss": losses[-1], + "mean_loss_last_10": float(np.mean(losses[-10:])), + "final_preview_psnr": final_metrics["psnr"], + "final_preview_mae": final_metrics["mae"], + "mean_step_psnr_last_10": float(np.mean(psnrs[-10:])), + "mean_step_mae_last_10": float(np.mean(maes[-10:])), + "multi_preview_mean_psnr": float(np.mean([item["psnr"] for item in multi_preview_metrics])), + "multi_preview_mean_mae": float(np.mean([item["mae"] for item in multi_preview_metrics])), + "multi_preview_metrics": multi_preview_metrics, + "adaptive_densify": { + "enabled": adaptive_densify_every > 0, + "every": adaptive_densify_every, + "start": adaptive_densify_start, + "stop": densify_stop, + "points_per_event": adaptive_densify_points, + "event_count": len(adaptive_events), + "added_point_count": int(sum(event.get("added_count", 0) for event in adaptive_events)), + "events": adaptive_events, + }, + "trim_report": trim_report, + "dense_image_report": dense_report, + "densify_report": densify_report, + } + + +def build_optimizer( + torch: Any, + *, + means: Any, + log_scales: Any, + opacity_logits: Any, + color_logits: Any, + means_lr: float, + scale_lr: float, + opacity_lr: float, + color_lr: float, +) -> Any: + return torch.optim.Adam( + [ + {"params": [means], "lr": means_lr}, + {"params": [log_scales], "lr": scale_lr}, + {"params": [opacity_logits], "lr": opacity_lr}, + {"params": [color_logits], "lr": color_lr}, + ] + ) + + +def should_adaptive_densify( + *, + step: int, + every: int, + start: int, + stop: int, + current_count: int, + max_points: int, +) -> bool: + return every > 0 and step >= start and step <= stop and step % every == 0 and current_count < max_points + + +def adaptive_densify_from_residual( + torch: Any, + *, + frame_sample: FrameSample, + render: Any, + target: Any, + means: Any, + log_scales: Any, + opacity_logits: Any, + color_logits: Any, + quats: Any, + scale_value: float, + max_points: int, + add_points: int, + error_percentile: float, + depth_neighbors: int, + max_distance_px: float, + max_projection_points: int, + rng: np.random.Generator, + device: str, +) -> tuple[dict[str, Any], tuple[Any, Any, Any, Any, Any] | None]: + remaining = max(0, int(max_points) - int(means.shape[0])) + requested = min(max(int(add_points), 0), remaining) + event: dict[str, Any] = { + "frame": frame_sample.image_path.name, + "requested_count": int(add_points), + "current_count": int(means.shape[0]), + "max_points": int(max_points), + "added_count": 0, + } + if requested <= 0: + event["reason"] = "max_points_reached" + return event, None + + error = torch.mean(torch.abs(torch.clamp(render[0], 0.0, 1.0) - torch.clamp(target[0], 0.0, 1.0)), dim=-1) + error_np = error.detach().cpu().numpy() + pixels = sample_high_error_pixels( + error_np, + count=requested, + percentile=error_percentile, + rng=rng, + ) + if len(pixels) == 0: + event["reason"] = "no_high_error_pixels" + return event, None + + projection_points = means.detach().cpu().numpy().astype(np.float32, copy=False) + if len(projection_points) > max_projection_points > 0: + selected = rng.choice(len(projection_points), size=max_projection_points, replace=False) + projection_points = projection_points[np.sort(selected)] + projected = project_world_points(projection_points, frame_sample) + if len(projected["uv"]) < max(32, depth_neighbors): + event["reason"] = "too_few_projected_points" + event["projected_count"] = int(len(projected["uv"])) + return event, None + + depths, nearest_distances = estimate_depths_from_sparse_projection( + pixels, + projected["uv"], + projected["depth"], + neighbors=max(1, depth_neighbors), + ) + valid = np.isfinite(depths) & (depths > 0.0) & (nearest_distances <= max_distance_px) + if int(valid.sum()) < max(32, int(requested * 0.1)): + valid = np.isfinite(depths) & (depths > 0.0) + pixels = pixels[valid] + depths = depths[valid] + nearest_distances = nearest_distances[valid] + if len(pixels) == 0: + event["reason"] = "no_valid_depths" + event["projected_count"] = int(len(projected["uv"])) + return event, None + + if len(pixels) > requested: + chosen = rng.choice(len(pixels), size=requested, replace=False) + pixels = pixels[np.sort(chosen)] + depths = depths[np.sort(chosen)] + nearest_distances = nearest_distances[np.sort(chosen)] + + world = unproject_pixels_to_world(pixels, depths, frame_sample).astype(np.float32, copy=False) + rgb = frame_sample.pixels[ + np.clip(np.rint(pixels[:, 1]).astype(np.int32), 0, frame_sample.height - 1), + np.clip(np.rint(pixels[:, 0]).astype(np.int32), 0, frame_sample.width - 1), + ].astype(np.float32, copy=False) + + new_means = torch.tensor(world, dtype=torch.float32, device=device) + new_log_scales = torch.full((len(world), 3), math.log(scale_value), dtype=torch.float32, device=device) + new_opacity_logits = torch.full((len(world),), 0.0, dtype=torch.float32, device=device) + new_color_logits = torch.logit(torch.tensor(rgb, dtype=torch.float32, device=device).clamp(1e-4, 1.0 - 1e-4)) + new_quats = torch.zeros((len(world), 4), dtype=torch.float32, device=device) + new_quats[:, 0] = 1.0 + + updated_means = torch.cat([means.detach(), new_means], dim=0).requires_grad_(True) + updated_scales = torch.cat([log_scales.detach(), new_log_scales], dim=0).requires_grad_(True) + updated_opacity = torch.cat([opacity_logits.detach(), new_opacity_logits], dim=0).requires_grad_(True) + updated_colors = torch.cat([color_logits.detach(), new_color_logits], dim=0).requires_grad_(True) + updated_quats = torch.cat([quats.detach(), new_quats], dim=0) + event.update( + { + "added_count": int(len(world)), + "projected_count": int(len(projected["uv"])), + "new_count": int(updated_means.shape[0]), + "error_percentile": float(error_percentile), + "mean_selected_error": float(np.mean(error_np[np.clip(np.rint(pixels[:, 1]).astype(np.int32), 0, frame_sample.height - 1), np.clip(np.rint(pixels[:, 0]).astype(np.int32), 0, frame_sample.width - 1)])), + "nearest_distance_mean": float(np.mean(nearest_distances)), + "depth_median": float(np.median(depths)), + } + ) + return event, (updated_means, updated_scales, updated_opacity, updated_colors, updated_quats) + + +def sample_high_error_pixels( + error: np.ndarray, + *, + count: int, + percentile: float, + rng: np.random.Generator, +) -> np.ndarray: + threshold = float(np.percentile(error, max(0.0, min(100.0, percentile)))) + ys, xs = np.nonzero(error >= threshold) + if len(xs) == 0: + return np.zeros((0, 2), dtype=np.float32) + if len(xs) > count: + chosen = rng.choice(len(xs), size=count, replace=False) + xs = xs[chosen] + ys = ys[chosen] + return np.stack([xs.astype(np.float32), ys.astype(np.float32)], axis=1) + + +def write_multi_frame_preview( + rasterization: Any, + frame_tensors: list[dict[str, Any]], + *, + means: Any, + quats: Any, + log_scales: Any, + opacity_logits: Any, + color_logits: Any, + output: Path, + preview_frames: int, + device: str, +) -> list[dict[str, float]]: + if preview_frames <= 0: + return [] + indices = np.linspace(0, len(frame_tensors) - 1, min(preview_frames, len(frame_tensors)), dtype=np.int64) + pairs: list[tuple[np.ndarray, np.ndarray]] = [] + metrics: list[dict[str, float]] = [] + for index in indices: + frame = frame_tensors[int(index)] + render, _alpha, _info = rasterization( + means=means, + quats=quats, + scales=torch_exp_clamped(log_scales), + opacities=torch_sigmoid(opacity_logits), + colors=torch_sigmoid(color_logits), + viewmats=frame["c2w"].inverse(), + Ks=frame["K"], + width=int(frame["width"]), + height=int(frame["height"]), + packed=False, + backgrounds=zeros_background_like(means, device=device), + render_mode="RGB", + rasterize_mode="antialiased", + camera_model="pinhole", + ) + image_metric = image_metrics(render, frame["pixels"]) + metrics.append( + { + "frame_index": int(index), + "psnr": image_metric["psnr"], + "mae": image_metric["mae"], + } + ) + pairs.append((tensor_image_to_numpy(frame["pixels"][0]), tensor_image_to_numpy(render[0]))) + write_preview_contact_sheet(pairs, output) + return metrics + + +def torch_exp_clamped(value: Any) -> Any: + return value.exp().clamp(1e-5, 1.0) + + +def torch_sigmoid(value: Any) -> Any: + return value.sigmoid() + + +def zeros_background_like(value: Any, *, device: str) -> Any: + import torch + + return torch.zeros((1, 3), dtype=torch.float32, device=device) + + +def write_preview_contact_sheet(pairs: list[tuple[np.ndarray, np.ndarray]], output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + if not pairs: + Image.new("RGB", (1, 1), "black").save(output) + return + tile_w = max(source.shape[1] + render.shape[1] for source, render in pairs) + tile_h = max(max(source.shape[0], render.shape[0]) for source, render in pairs) + canvas = np.zeros((tile_h * len(pairs), tile_w, 3), dtype=np.uint8) + for row, (source, render) in enumerate(pairs): + y = row * tile_h + canvas[y : y + source.shape[0], : source.shape[1]] = source + canvas[y : y + render.shape[0], source.shape[1] : source.shape[1] + render.shape[1]] = render + Image.fromarray(canvas).save(output) + + +def load_frames( + pose_init_dir: Path, + *, + camera_matrix_key: str, + max_frames: int, + image_max_size: int, +) -> list[FrameSample]: + transforms_path = pose_init_dir / "transforms.json" + if not transforms_path.is_file(): + raise SystemExit(f"pose-init transforms.json not found: {transforms_path}") + transforms = json.loads(transforms_path.read_text(encoding="utf-8")) + raw_frames = transforms.get("frames") or [] + if not raw_frames: + raise SystemExit(f"pose-init transforms.json has no frames: {transforms_path}") + + selected_frames = select_evenly_spaced(raw_frames, max_frames) + samples: list[FrameSample] = [] + for frame in selected_frames: + image_path = pose_init_dir / str(frame["file_path"]) + if not image_path.is_file(): + raise SystemExit(f"pose-init image not found: {image_path}") + image = Image.open(image_path).convert("RGB") + original_width, original_height = image.size + scale = min(1.0, float(image_max_size) / max(float(original_width), float(original_height))) + width = max(1, int(round(original_width * scale))) + height = max(1, int(round(original_height * scale))) + if (width, height) != image.size: + image = image.resize((width, height), PIL_BILINEAR) + pixels = np.asarray(image, dtype=np.float32) / 255.0 + + matrix = frame.get(camera_matrix_key) + if matrix is None: + matrix = frame.get("colmap_transform_matrix") or frame.get("transform_matrix") + if matrix is None: + raise SystemExit(f"pose-init frame has no usable camera matrix: {image_path}") + c2w = np.asarray(matrix, dtype=np.float32) + if c2w.shape != (4, 4): + raise SystemExit(f"camera matrix must be 4x4 for {image_path}") + + fl_x = float(frame.get("fl_x", transforms.get("fl_x", 0.0))) * scale + fl_y = float(frame.get("fl_y", transforms.get("fl_y", fl_x))) * scale + cx = float(frame.get("cx", transforms.get("cx", original_width * 0.5))) * scale + cy = float(frame.get("cy", transforms.get("cy", original_height * 0.5))) * scale + K = np.asarray([[fl_x, 0.0, cx], [0.0, fl_y, cy], [0.0, 0.0, 1.0]], dtype=np.float32) + samples.append(FrameSample(image_path=image_path, pixels=pixels, c2w=c2w, K=K, width=width, height=height)) + + if not samples: + raise SystemExit("no pose-init frames were loaded") + return samples + + +def select_evenly_spaced(items: list[Any], max_count: int) -> list[Any]: + if len(items) <= max_count: + return list(items) + indices = np.linspace(0, len(items) - 1, max_count, dtype=np.int64) + return [items[int(index)] for index in indices] + + +def load_sparse_points(path: Path) -> tuple[np.ndarray, np.ndarray]: + if not path.is_file(): + raise SystemExit(f"pose-init sparse point PLY not found: {path}") + with path.open("r", encoding="ascii") as handle: + first = handle.readline().strip() + if first != "ply": + raise SystemExit(f"unsupported sparse point file, expected PLY: {path}") + vertex_count = None + properties: list[str] = [] + in_vertex = False + for line in handle: + stripped = line.strip() + if stripped.startswith("format ") and stripped != "format ascii 1.0": + raise SystemExit(f"only ascii PLY sparse points are supported: {path}") + if stripped.startswith("element vertex "): + vertex_count = int(stripped.split()[-1]) + in_vertex = True + continue + if stripped.startswith("element ") and not stripped.startswith("element vertex "): + in_vertex = False + if in_vertex and stripped.startswith("property "): + properties.append(stripped.split()[-1]) + if stripped == "end_header": + break + if vertex_count is None: + raise SystemExit(f"PLY has no vertex count: {path}") + required = {"x", "y", "z"} + if not required.issubset(set(properties)): + raise SystemExit(f"PLY missing xyz properties: {path}") + names = {name: index for index, name in enumerate(properties)} + points = np.zeros((vertex_count, 3), dtype=np.float32) + colors = np.ones((vertex_count, 3), dtype=np.float32) * 0.7 + for row in range(vertex_count): + values = handle.readline().split() + if len(values) < len(properties): + raise SystemExit(f"PLY vertex row is truncated at row {row}: {path}") + points[row] = [float(values[names["x"]]), float(values[names["y"]]), float(values[names["z"]])] + if {"red", "green", "blue"}.issubset(names): + colors[row] = [ + float(values[names["red"]]) / 255.0, + float(values[names["green"]]) / 255.0, + float(values[names["blue"]]) / 255.0, + ] + finite = np.isfinite(points).all(axis=1) & np.isfinite(colors).all(axis=1) + points = points[finite] + colors = np.clip(colors[finite], 0.0, 1.0) + return points.astype(np.float32, copy=False), colors.astype(np.float32, copy=False) + + +def trim_sparse_outliers( + points: np.ndarray, + colors: np.ndarray, + *, + percentile: float, +) -> tuple[np.ndarray, np.ndarray, dict[str, Any]]: + if len(points) == 0: + return points, colors, {"input_point_count": 0, "kept_point_count": 0, "percentile": percentile} + if percentile <= 0.0 or percentile >= 100.0: + return points, colors, { + "input_point_count": int(len(points)), + "kept_point_count": int(len(points)), + "percentile": percentile, + "enabled": False, + } + center = np.median(points, axis=0) + radius = np.linalg.norm(points - center[None, :], axis=1) + threshold = float(np.percentile(radius, percentile)) + keep = radius <= threshold + if int(keep.sum()) < max(256, int(len(points) * 0.5)): + keep = np.ones(len(points), dtype=bool) + return points[keep], colors[keep], { + "input_point_count": int(len(points)), + "kept_point_count": int(keep.sum()), + "percentile": percentile, + "radius_threshold": threshold, + "center": [float(value) for value in center], + } + + +def densify_sparse_points( + points: np.ndarray, + colors: np.ndarray, + *, + target_gaussians: int, + max_points: int, + seed: int, +) -> tuple[np.ndarray, np.ndarray, dict[str, Any]]: + if len(points) == 0: + return points, colors, {"input_count": 0, "output_count": 0} + target = min(max(int(target_gaussians), len(points)), max(int(max_points), len(points))) + if target <= len(points): + return points, colors, {"input_count": int(len(points)), "output_count": int(len(points)), "jittered_count": 0} + + rng = np.random.default_rng(seed) + extra_count = target - len(points) + base_indices = rng.integers(0, len(points), size=extra_count) + span = np.percentile(points, 95, axis=0) - np.percentile(points, 5, axis=0) + jitter_scale = max(float(np.max(span)) / 900.0, 1e-4) + jitter = rng.normal(0.0, jitter_scale, size=(extra_count, 3)).astype(np.float32) + extra_points = points[base_indices] + jitter + extra_colors = np.clip(colors[base_indices] + rng.normal(0.0, 0.015, size=(extra_count, 3)), 0.0, 1.0).astype(np.float32) + return ( + np.concatenate([points, extra_points], axis=0).astype(np.float32, copy=False), + np.concatenate([colors, extra_colors], axis=0).astype(np.float32, copy=False), + { + "input_count": int(len(points)), + "output_count": int(target), + "jittered_count": int(extra_count), + "jitter_scale": jitter_scale, + }, + ) + + +def augment_points_from_registered_images( + points: np.ndarray, + colors: np.ndarray, + frames: list[FrameSample], + *, + points_per_frame: int, + neighbors: int, + max_distance_px: float, + seed: int, +) -> tuple[np.ndarray, np.ndarray, dict[str, Any]]: + if points_per_frame <= 0 or len(points) == 0: + return ( + np.zeros((0, 3), dtype=np.float32), + np.zeros((0, 3), dtype=np.float32), + {"enabled": False, "output_count": 0}, + ) + + rng = np.random.default_rng(seed) + all_points: list[np.ndarray] = [] + all_colors: list[np.ndarray] = [] + frame_reports: list[dict[str, Any]] = [] + for frame_index, frame in enumerate(frames): + projected = project_world_points(points, frame) + if projected["uv"].shape[0] < max(32, neighbors): + frame_reports.append( + { + "frame": frame.image_path.name, + "projected_sparse_count": int(projected["uv"].shape[0]), + "sampled_count": 0, + } + ) + continue + pixels = sample_candidate_pixels(frame.width, frame.height, points_per_frame, rng=rng) + depths, nearest_distances = estimate_depths_from_sparse_projection( + pixels, + projected["uv"], + projected["depth"], + neighbors=max(1, neighbors), + ) + valid = np.isfinite(depths) & (depths > 0.0) & (nearest_distances <= max_distance_px) + if int(valid.sum()) < max(64, int(points_per_frame * 0.15)): + valid = np.isfinite(depths) & (depths > 0.0) + pixels = pixels[valid] + depths = depths[valid] + if len(pixels) == 0: + frame_reports.append( + { + "frame": frame.image_path.name, + "projected_sparse_count": int(projected["uv"].shape[0]), + "sampled_count": 0, + } + ) + continue + world = unproject_pixels_to_world(pixels, depths, frame) + rgb = frame.pixels[ + np.clip(np.rint(pixels[:, 1]).astype(np.int32), 0, frame.height - 1), + np.clip(np.rint(pixels[:, 0]).astype(np.int32), 0, frame.width - 1), + ] + all_points.append(world.astype(np.float32, copy=False)) + all_colors.append(rgb.astype(np.float32, copy=False)) + frame_reports.append( + { + "frame": frame.image_path.name, + "projected_sparse_count": int(projected["uv"].shape[0]), + "sampled_count": int(len(world)), + "nearest_distance_mean": float(np.mean(nearest_distances[valid])), + "depth_median": float(np.median(depths)), + } + ) + + if not all_points: + return ( + np.zeros((0, 3), dtype=np.float32), + np.zeros((0, 3), dtype=np.float32), + { + "enabled": True, + "points_per_frame": points_per_frame, + "output_count": 0, + "frames": frame_reports, + }, + ) + dense_points = np.concatenate(all_points, axis=0) + dense_colors = np.concatenate(all_colors, axis=0) + return dense_points, dense_colors, { + "enabled": True, + "points_per_frame": points_per_frame, + "neighbors": neighbors, + "max_distance_px": max_distance_px, + "output_count": int(len(dense_points)), + "frames": frame_reports, + } + + +def project_world_points(points: np.ndarray, frame: FrameSample) -> dict[str, np.ndarray]: + w2c = np.linalg.inv(frame.c2w) + homogeneous = np.concatenate([points, np.ones((len(points), 1), dtype=np.float32)], axis=1) + camera = homogeneous @ w2c.T + depth = camera[:, 2] + valid_depth = depth > 1e-5 + camera = camera[valid_depth] + depth = depth[valid_depth] + if len(camera) == 0: + return {"uv": np.zeros((0, 2), dtype=np.float32), "depth": np.zeros((0,), dtype=np.float32)} + uv = np.empty((len(camera), 2), dtype=np.float32) + uv[:, 0] = frame.K[0, 0] * (camera[:, 0] / depth) + frame.K[0, 2] + uv[:, 1] = frame.K[1, 1] * (camera[:, 1] / depth) + frame.K[1, 2] + in_frame = (uv[:, 0] >= 0.0) & (uv[:, 0] < frame.width) & (uv[:, 1] >= 0.0) & (uv[:, 1] < frame.height) + return {"uv": uv[in_frame], "depth": depth[in_frame].astype(np.float32, copy=False)} + + +def sample_candidate_pixels(width: int, height: int, count: int, *, rng: np.random.Generator) -> np.ndarray: + count = max(1, int(count)) + stride = max(1, int(math.sqrt((width * height) / float(count)))) + xs = np.arange(stride // 2, width, stride, dtype=np.float32) + ys = np.arange(stride // 2, height, stride, dtype=np.float32) + grid_x, grid_y = np.meshgrid(xs, ys) + pixels = np.stack([grid_x.reshape(-1), grid_y.reshape(-1)], axis=1) + if len(pixels) > count: + selected = rng.choice(len(pixels), size=count, replace=False) + pixels = pixels[np.sort(selected)] + return pixels.astype(np.float32, copy=False) + + +def estimate_depths_from_sparse_projection( + pixels: np.ndarray, + uv: np.ndarray, + depth: np.ndarray, + *, + neighbors: int, +) -> tuple[np.ndarray, np.ndarray]: + estimated = np.zeros((len(pixels),), dtype=np.float32) + nearest = np.zeros((len(pixels),), dtype=np.float32) + chunk = 512 + k = min(max(1, neighbors), len(uv)) + for start in range(0, len(pixels), chunk): + stop = min(start + chunk, len(pixels)) + delta = pixels[start:stop, None, :] - uv[None, :, :] + dist2 = np.sum(delta * delta, axis=2) + if k == 1: + idx = np.argmin(dist2, axis=1)[:, None] + else: + idx = np.argpartition(dist2, kth=k - 1, axis=1)[:, :k] + chosen_dist2 = np.take_along_axis(dist2, idx, axis=1) + chosen_depth = depth[idx] + weights = 1.0 / np.maximum(chosen_dist2, 1e-4) + estimated[start:stop] = np.sum(weights * chosen_depth, axis=1) / np.sum(weights, axis=1) + nearest[start:stop] = np.sqrt(np.min(chosen_dist2, axis=1)) + return estimated, nearest + + +def unproject_pixels_to_world(pixels: np.ndarray, depths: np.ndarray, frame: FrameSample) -> np.ndarray: + x = (pixels[:, 0] - frame.K[0, 2]) / frame.K[0, 0] * depths + y = (pixels[:, 1] - frame.K[1, 2]) / frame.K[1, 1] * depths + camera = np.stack([x, y, depths, np.ones_like(depths)], axis=1).astype(np.float32, copy=False) + world = camera @ frame.c2w.T + return world[:, :3] + + +def estimate_initial_scale(points: np.ndarray) -> float: + span = np.percentile(points, 95, axis=0) - np.percentile(points, 5, axis=0) + extent = max(float(np.max(span)), 1e-3) + return max(min(extent / 220.0, 0.08), 0.004) + + +def image_metrics(render: Any, target: Any) -> dict[str, float]: + import torch + + diff = torch.clamp(render, 0.0, 1.0) - torch.clamp(target, 0.0, 1.0) + mae = float(torch.mean(torch.abs(diff)).detach().cpu()) + mse = float(torch.mean(diff * diff).detach().cpu()) + psnr = 99.0 if mse <= 1e-12 else -10.0 * math.log10(mse) + return {"psnr": psnr, "mae": mae} + + +def tensor_image_to_numpy(image: Any) -> np.ndarray: + array = image.detach().clamp(0.0, 1.0).cpu().numpy() + return (array * 255.0 + 0.5).astype(np.uint8) + + +def write_side_by_side_preview(source: np.ndarray, render: np.ndarray, output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + h = max(source.shape[0], render.shape[0]) + w = source.shape[1] + render.shape[1] + canvas = np.zeros((h, w, 3), dtype=np.uint8) + canvas[: source.shape[0], : source.shape[1]] = source + canvas[: render.shape[0], source.shape[1] :] = render + Image.fromarray(canvas).save(output) + + +def write_3dgs_ply( + path: Path, + means: np.ndarray, + colors: np.ndarray, + log_scales: np.ndarray, + opacity_logits: np.ndarray, +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + sh = (np.clip(colors, 0.0, 1.0) - 0.5) / SH_C0 + with path.open("w", encoding="ascii") as handle: + handle.write("ply\nformat ascii 1.0\n") + handle.write(f"element vertex {len(means)}\n") + for name in ("x", "y", "z", "nx", "ny", "nz", "f_dc_0", "f_dc_1", "f_dc_2"): + handle.write(f"property float {name}\n") + for index in range(45): + handle.write(f"property float f_rest_{index}\n") + for name in ("opacity", "scale_0", "scale_1", "scale_2", "rot_0", "rot_1", "rot_2", "rot_3"): + handle.write(f"property float {name}\n") + handle.write("end_header\n") + zeros_rest = " ".join(["0"] * 45) + for point, color, scale, opacity in zip(means, sh, log_scales, opacity_logits): + handle.write( + f"{point[0]:.7g} {point[1]:.7g} {point[2]:.7g} 0 0 0 " + f"{color[0]:.7g} {color[1]:.7g} {color[2]:.7g} {zeros_rest} " + f"{opacity:.7g} {scale[0]:.7g} {scale[1]:.7g} {scale[2]:.7g} 1 0 0 0\n" + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_sam3d_movable_batch.py b/vendor/gs_playground/scripts/real2sim_sam3d_movable_batch.py new file mode 100644 index 0000000..f9837f7 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_sam3d_movable_batch.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Run SAM-3D over movable extraction tasks and emit production object assets.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from real2sim.contracts import write_json +from real2sim.image_io import read_gray, read_rgb, write_gray, write_rgb + + +def main() -> int: + parser = argparse.ArgumentParser(description="Convert movable extraction tasks into production SAM-3D object assets") + parser.add_argument("--package", type=Path, required=True, help="Real2Sim package root") + parser.add_argument("--tasks", type=Path, default=None, help="movable_extraction_tasks.json; defaults under package") + parser.add_argument("--output-objects-json", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--command", default=os.environ.get("REAL2SIM_SAM3D_COMMAND", "")) + parser.add_argument("--max-objects", type=int, default=3) + parser.add_argument("--dry-run", action="store_true", help="Write planned SAM-3D jobs without executing them") + args = parser.parse_args() + + result = run_sam3d_movable_batch( + package_root=args.package, + tasks_path=args.tasks or args.package / "artifacts" / "assets" / "movable_extraction_tasks.json", + output_objects_json=args.output_objects_json, + output_dir=args.output_dir, + command_template=args.command, + max_objects=max(1, int(args.max_objects)), + dry_run=bool(args.dry_run), + ) + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 + + +def run_sam3d_movable_batch( + *, + package_root: Path, + tasks_path: Path, + output_objects_json: Path, + output_dir: Path, + command_template: str, + max_objects: int, + dry_run: bool = False, +) -> dict[str, Any]: + tasks_data = json.loads(tasks_path.read_text(encoding="utf-8")) + tasks = tasks_data.get("tasks", []) if isinstance(tasks_data, dict) else [] + if not dry_run and not command_template.strip(): + raise SystemExit("SAM-3D command is required via --command or REAL2SIM_SAM3D_COMMAND") + output_dir.mkdir(parents=True, exist_ok=True) + objects: list[dict[str, Any]] = [] + jobs: list[dict[str, Any]] = [] + for index, task in enumerate(tasks[:max_objects]): + if not isinstance(task, dict): + continue + job = prepare_sam3d_task(package_root=package_root, output_dir=output_dir, task=task, index=index) + jobs.append(job) + if dry_run: + continue + command = render_command(command_template, job["context"]) + completed = subprocess.run(command, cwd=REPO_ROOT, check=False) + if completed.returncode != 0: + raise SystemExit(f"SAM-3D command failed for {job['id']} with exit code {completed.returncode}") + objects.append(production_object_from_job(job, package_root=package_root)) + payload = { + "format": "gs-playground-real2sim-production-objects-v1", + "objects": objects, + "quality": { + "object_count": len(objects), + "source_task_count": len(tasks), + "planned_job_count": len(jobs), + "production": bool(objects) and len(objects) == len(jobs), + "dry_run": bool(dry_run), + }, + "sam3d_jobs": [ + { + "id": job["id"], + "label": job["label"], + "instance_json": job["instance_json"], + "image": job["image"], + "mask": job["mask"], + "output_dir": job["output_dir"], + } + for job in jobs + ], + } + write_json(output_objects_json, payload) + return {"objects_json": output_objects_json.as_posix(), **payload["quality"]} + + +def prepare_sam3d_task(*, package_root: Path, output_dir: Path, task: dict[str, Any], index: int) -> dict[str, Any]: + task_id = str(task.get("id") or f"task_{index:03d}") + label = best_prompt_label(task) + name = f"{index:02d}_{safe_token(label)}" + object_dir = output_dir / name + object_dir.mkdir(parents=True, exist_ok=True) + crop_path = package_root / str(task["crop"]) + mask_path = package_root / str(task["mask"]) + rgb = read_rgb(crop_path) + mask = read_gray(mask_path) + local_mask = local_crop_mask(mask, task.get("bbox_xyxy"), rgb.shape[:2]) + image_out = object_dir / "sam3d_image.ppm" + mask_out = object_dir / "sam3d_mask.pgm" + write_rgb(image_out, rgb) + write_gray(mask_out, local_mask.astype(np.float32)) + instance_json = object_dir / "instance.json" + write_json( + instance_json, + { + "id": task_id, + "label": label, + "bbox_xyxy": task.get("bbox_xyxy"), + "prompt_candidates": task.get("prompt_candidates"), + "source_task": task, + }, + ) + gaussian = object_dir / "gaussian.ply" + mesh = object_dir / "mesh.obj" + pose = object_dir / "pose.json" + cameras = package_root / "artifacts" / "cameras_json.json" + context = { + "image": image_out, + "mask": mask_out, + "instance_json": instance_json, + "object_dir": object_dir, + "object_ply": gaussian, + "object_mesh": mesh, + "object_pose": pose, + "cameras_json": cameras if cameras.is_file() else "", + } + return { + "id": task_id, + "name": name, + "label": label, + "image": image_out.as_posix(), + "mask": mask_out.as_posix(), + "instance_json": instance_json.as_posix(), + "output_dir": object_dir.as_posix(), + "gaussian": gaussian.as_posix(), + "mesh": mesh.as_posix(), + "pose": pose.as_posix(), + "context": context, + } + + +def render_command(template: str, context: dict[str, Any]) -> list[str]: + rendered = template.format_map({key: str(value) for key, value in context.items()}) + return shlex.split(rendered) + + +def production_object_from_job(job: dict[str, Any], *, package_root: Path) -> dict[str, Any]: + gaussian = require_file(Path(job["gaussian"]), "SAM-3D object gaussian") + mesh = require_file(Path(job["mesh"]), "SAM-3D object mesh") + pose_path = require_file(Path(job["pose"]), "SAM-3D object pose") + pose = json.loads(pose_path.read_text(encoding="utf-8")) + metadata = Path(job["output_dir"]) / "sam3d_metadata.json" + return { + "name": job["name"], + "label": job["label"], + "gaussian_ply": gaussian.as_posix(), + "mesh_obj": mesh.as_posix(), + "pose": pose, + "dynamic": True, + "source_instance_id": f"sam3d_{job['id']}", + "provisional": False, + "reconstruction_quality": { + "reconstruction_source": "sam3d_objects", + "commercial_asset_grade": True, + "has_gaussian_asset": True, + "has_mesh_asset": True, + "has_pose": True, + "sam3d_metadata": metadata.as_posix() if metadata.is_file() else None, + }, + } + + +def require_file(path: Path, label: str) -> Path: + if not path.is_file() or path.stat().st_size <= 0: + raise SystemExit(f"missing {label}: {path}") + return path + + +def local_crop_mask(global_mask: np.ndarray, bbox: Any, crop_shape: tuple[int, int]) -> np.ndarray: + if not isinstance(bbox, list) or len(bbox) != 4: + return np.ones(crop_shape, dtype=bool) + x0, y0, x1, y1 = [int(value) for value in bbox] + cropped = global_mask[max(0, y0) : max(0, y1), max(0, x0) : max(0, x1)] > 0.5 + if cropped.shape != crop_shape: + return np.ones(crop_shape, dtype=bool) + return cropped + + +def best_prompt_label(task: dict[str, Any]) -> str: + prompts = task.get("prompt_candidates") + if isinstance(prompts, list) and prompts: + return str(prompts[0]) + return "movable_object" + + +def safe_token(value: str) -> str: + return "".join(ch if ch.isalnum() else "_" for ch in value.lower()).strip("_") or "object" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_scene_viewer.py b/vendor/gs_playground/scripts/real2sim_scene_viewer.py new file mode 100644 index 0000000..c48952c --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_scene_viewer.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""Desktop viewer for a generated Real2Sim MotrixSim scene package. + +The viewer can inspect only the MJCF scene, or load a Real2Sim package config +and render the corresponding 3D Gaussian assets without attaching a robot. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +os.environ.setdefault("SDL_AUDIODRIVER", "dummy") +os.environ.setdefault("ALSOFT_DRIVERS", "null") +os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "8.9") +if "XDG_RUNTIME_DIR" not in os.environ: + runtime_dir = Path(f"/run/user/{os.getuid()}") + if runtime_dir.is_dir(): + os.environ["XDG_RUNTIME_DIR"] = runtime_dir.as_posix() + +sys.path[:] = [ + entry + for entry in sys.path + if not entry + or ( + Path(entry).name != "speedy-splat" + and "real2sim_paper_runtime/speedy-splat" not in Path(entry).as_posix() + ) +] + +import numpy as np +from motrixsim import SceneData, msd, run +from motrixsim.render import Layout, RenderApp, RenderSettings + +REPO_ROOT = Path(__file__).resolve().parents[1] +if REPO_ROOT.as_posix() not in sys.path: + sys.path.insert(0, REPO_ROOT.as_posix()) + +from real2sim.camera import load_anysplat_source_camera + + +def main() -> int: + parser = argparse.ArgumentParser(description="Open a Real2Sim scene package without attaching a robot") + parser.add_argument("--config", type=Path, default=None, help="Path to a generated configs/*_real2sim.json") + parser.add_argument("--scene", type=Path, default=None, help="Path to the generated mjcf/scene.xml") + parser.add_argument( + "--camera", + default="overview", + help="Camera name from scene.xml to use as the main view; use 'system' for the interactive free camera", + ) + parser.add_argument("--no-gs", action="store_true", help="Disable 3DGS preview widgets and load only the Motrix/MJCF scene") + parser.add_argument("--gs-width", type=int, default=640, help="3DGS camera widget width") + parser.add_argument("--gs-height", type=int, default=480, help="3DGS camera widget height") + parser.add_argument("--gs-widget-left", type=int, default=10, help="Left offset for 3DGS preview widgets") + parser.add_argument("--gs-widget-top", type=int, default=10, help="Top offset for 3DGS preview widgets") + parser.add_argument("--gs-render-fps", type=float, default=12.0, help="Maximum 3DGS widget render refresh rate; use 0 for every frame") + parser.add_argument("--static-gs-preview", action="store_true", help="Render 3DGS preview widgets once, then leave interaction to MotrixRender") + parser.add_argument("--viewer-fps", type=float, default=60.0, help="MotrixRender window refresh rate") + parser.add_argument("--frame-sleep-ms", type=float, default=0.0, help="Extra sleep after each render frame to keep desktop interaction responsive") + parser.add_argument("--show-source-gs", action="store_true", help="Render the AnySplat source camera from camera_artifacts.cameras_json") + parser.add_argument("--source-gs-only", action="store_true", help="Use the source-camera 3DGS render as the only 3DGS preview widget") + parser.add_argument("--source-camera-json", type=Path, default=None, help="Override source-view cameras.json artifact") + parser.add_argument("--save-source-gs", type=Path, default=None, help="Optional PNG path for the rendered source-camera 3DGS view") + parser.add_argument("--save-main-gs", type=Path, default=None, help="Optional PNG path for the rendered main-camera 3DGS view") + parser.add_argument("--show-system-gs", action="store_true", help="Also render a 3DGS view from the interactive camera") + parser.add_argument("--exit-after-static-preview", action="store_true", help="Exit after static 3DGS preview images have been rendered and saved") + parser.add_argument("--no-sync", action="store_true", help="Disable real-time sync") + args = parser.parse_args() + + config = load_config(args.config) + scene_path = resolve_scene_path(args.scene, config, args.config) + scene_gaussians = {} if args.no_gs else collect_scene_gaussians(config, args.config) + camera_artifacts = collect_camera_artifacts(config, args.config) + source_camera_json = args.source_camera_json or camera_artifacts.get("cameras_json") + gaussian_links = { + str(gs_name): str(link_name) + for gs_name, link_name in config.get("gaussian_links", {}).items() + } + + world = msd.from_file(scene_path.as_posix()) + model = world.build() + data = SceneData(model) + + render_target_fps = max(int(round(float(args.viewer_fps))), 1) + with RenderApp(fps=render_target_fps) as render: + print(f"Launching Real2Sim scene viewer: {scene_path}", flush=True) + render.launch(model, render_settings=RenderSettings.performance()) + for group_id in range(6): + try: + render.opt.set_group_vis(group_id, True) + except Exception: + pass + + use_system_camera = args.camera.lower() in {"system", "free", "interactive"} + camera = None if use_system_camera else find_camera(model, args.camera) + camera_id = None if use_system_camera else find_camera_id(model, args.camera) + if use_system_camera: + render.set_main_camera(None) + seed_camera = find_camera(model, "overview") or (model.cameras[0] if len(model.cameras) else None) + if seed_camera is not None: + try: + render.system_camera.pose = seed_camera.get_pose(data).astype(np.float32) + except Exception: + pass + print("Using system camera", flush=True) + elif camera is not None: + render.set_main_camera(camera) + print(f"Using camera: {args.camera}", flush=True) + + gs_renderer = None + gs_image = None + gs_source_image = None + gs_system_image = None + gs_debug_printed = False + source_rendered = False + source_render_failed = False + source_camera = None + if scene_gaussians: + print(f"Loading 3DGS assets: {', '.join(sorted(scene_gaussians))}", flush=True) + from gaussian_renderer import GSRendererMotrixSim + + gs_renderer = GSRendererMotrixSim(scene_gaussians, model) + apply_gaussian_links(gs_renderer, model, gaussian_links) + widget_top = int(args.gs_widget_top) + widget_left = int(args.gs_widget_left) + if args.source_gs_only: + args.show_source_gs = True + if args.show_source_gs or args.save_source_gs is not None: + if source_camera_json is None: + print("Source 3DGS view requested but no camera_artifacts.cameras_json was found.", flush=True) + elif source_camera_json.is_file(): + source_camera = load_anysplat_source_camera(source_camera_json) + gs_source_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8)) + if args.show_source_gs: + render.widgets.create_image_widget( + gs_source_image, + layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height), + ) + widget_top += args.gs_height + 10 + print(f"Source 3DGS camera ready: {source_camera_json}", flush=True) + else: + print(f"Source camera JSON is missing: {source_camera_json}", flush=True) + if not args.source_gs_only: + gs_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8)) + render.widgets.create_image_widget( + gs_image, + layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height), + ) + widget_top += args.gs_height + 10 + if args.show_system_gs: + gs_system_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8)) + render.widgets.create_image_widget( + gs_system_image, + layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height), + ) + print("3DGS renderer ready.", flush=True) + + print(f"Scene viewer ready. Bodies: {len(model.body_names)} Links: {len(model.link_names)}", flush=True) + + def phys_step() -> None: + return None + + last_gs_render_at = 0.0 + gs_preview_rendered = False + gs_render_interval = 0.0 if args.gs_render_fps <= 0 else 1.0 / max(float(args.gs_render_fps), 1e-6) + + def render_step() -> bool: + nonlocal gs_debug_printed, source_rendered, source_render_failed, last_gs_render_at, gs_preview_rendered + if render.input.is_key_just_pressed("esc") or render.input.is_key_just_pressed("escape"): + return False + if gs_renderer is not None: + now = time.monotonic() + should_render_gs = ( + not (args.static_gs_preview and gs_preview_rendered) + and (gs_render_interval <= 0.0 or (now - last_gs_render_at) >= gs_render_interval) + ) + if source_camera is not None and gs_source_image is not None and not source_rendered and not source_render_failed: + try: + gs_renderer.update_gaussians(data) + source_rgb, _source_depth = gs_renderer.render_batch( + source_camera.position.reshape(1, 3), + source_camera.xmat.reshape(1, 9), + args.gs_height, + args.gs_width, + np.array([source_camera.fovy_degrees], dtype=np.float32), + y_up=source_camera.y_up, + ) + gs_source_image.pixels = rgb_to_uint8(source_rgb[0]) + if args.save_source_gs is not None: + args.save_source_gs.parent.mkdir(parents=True, exist_ok=True) + gs_source_image.save_to_disk(args.save_source_gs.as_posix()) + print(f"Saved source 3DGS render: {args.save_source_gs}", flush=True) + source_rendered = True + print("3DGS rendered source camera.", flush=True) + if args.source_gs_only and args.static_gs_preview and not gs_preview_rendered: + gs_preview_rendered = True + print("3DGS static preview rendered.", flush=True) + except Exception as exc: + source_render_failed = True + print(f"Source 3DGS render failed: {exc}", flush=True) + if should_render_gs and not args.source_gs_only: + last_gs_render_at = now + gs_renderer.update_gaussians(data) + render_ids = [] + if camera_id is not None: + render_ids.append(camera_id) + if args.show_system_gs: + render_ids.append(-1) + if render_ids: + results = gs_renderer.render( + model, + data, + render_ids, + args.gs_width, + args.gs_height, + system_camera=render.system_camera, + ) + if not gs_debug_printed: + print(f"3DGS rendered camera ids: {sorted(results.keys())}", flush=True) + gs_debug_printed = True + if camera_id in results and gs_image is not None: + gs_image.pixels = rgb_to_uint8(results[camera_id][0]) + if args.save_main_gs is not None: + args.save_main_gs.parent.mkdir(parents=True, exist_ok=True) + gs_image.save_to_disk(args.save_main_gs.as_posix()) + print(f"Saved main 3DGS render: {args.save_main_gs}", flush=True) + if -1 in results and gs_system_image is not None: + gs_system_image.pixels = rgb_to_uint8(results[-1][0]) + if args.static_gs_preview: + gs_preview_rendered = True + print("3DGS static preview rendered.", flush=True) + should_exit_after_preview = args.exit_after_static_preview and args.static_gs_preview and gs_preview_rendered + render.sync(data) + if args.frame_sleep_ms > 0: + time.sleep(float(args.frame_sleep_ms) / 1000.0) + if should_exit_after_preview: + # Motrix's render loop can ignore a false-y callback return on + # the Wuying headless display. This flag is only for automated + # static screenshot capture, so exit explicitly after flushing. + sys.stdout.flush() + sys.stderr.flush() + os._exit(0) + return True + + render_fps = 1000000.0 if args.no_sync else max(float(args.viewer_fps), 1.0) + run.render_loop(model.options.timestep, render_fps, phys_step, render_step) + return 0 + + +def find_camera(model, name: str): + try: + return model.cameras[name] + except Exception: + pass + for camera in model.cameras: + if getattr(camera, "name", None) == name: + return camera + return model.cameras[0] if len(model.cameras) else None + + +def find_camera_id(model, name: str) -> int | None: + for idx, camera in enumerate(model.cameras): + if getattr(camera, "name", None) == name: + return idx + return 0 if len(model.cameras) else None + + +def load_config(config_path: Path | None) -> dict: + if config_path is None: + return {} + with config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + if not isinstance(config, dict): + raise ValueError(f"Real2Sim viewer config must be a JSON object: {config_path}") + return config + + +def resolve_config_path(path: str | Path, config_path: Path | None) -> Path: + resolved = Path(path) + if resolved.is_absolute(): + return resolved + if config_path is not None: + return (config_path.resolve().parent / resolved).resolve() + return resolved.resolve() + + +def resolve_scene_path(scene_arg: Path | None, config: dict, config_path: Path | None) -> Path: + scene = scene_arg if scene_arg is not None else config.get("scene") + if not scene: + raise ValueError("Provide --scene or a --config containing a scene path") + return resolve_config_path(scene, config_path) + + +def collect_scene_gaussians(config: dict, config_path: Path | None) -> dict[str, str]: + scene_gaussians = config.get("scene_gaussians", {}) + if not isinstance(scene_gaussians, dict): + raise ValueError("scene_gaussians must be an object mapping Gaussian names to PLY paths") + + gaussians: dict[str, str] = {} + missing: list[Path] = [] + for name, path in scene_gaussians.items(): + ply = resolve_config_path(path, config_path) + if ply.exists(): + gaussians[str(name)] = ply.as_posix() + else: + missing.append(ply) + if missing: + print("Skipping missing 3DGS assets: " + ", ".join(path.as_posix() for path in missing), flush=True) + return gaussians + + +def collect_camera_artifacts(config: dict, config_path: Path | None) -> dict[str, Path]: + raw_artifacts = config.get("camera_artifacts", {}) + if not isinstance(raw_artifacts, dict): + raise ValueError("camera_artifacts must be an object mapping artifact names to paths") + + artifacts: dict[str, Path] = {} + missing: list[Path] = [] + for name, path in raw_artifacts.items(): + artifact = resolve_config_path(path, config_path) + if artifact.exists(): + artifacts[str(name)] = artifact + else: + missing.append(artifact) + if missing: + print("Skipping missing camera artifacts: " + ", ".join(path.as_posix() for path in missing), flush=True) + return artifacts + + +def apply_gaussian_links(gs_renderer, model, gaussian_links: dict[str, str]) -> None: + if not gaussian_links: + return + + link_ids = {name: idx for idx, name in enumerate(model.link_names)} + objects_info = [] + body_ids = [] + skipped = [] + for gs_name, link_name in gaussian_links.items(): + if gs_name not in gs_renderer.gaussian_start_indices: + skipped.append(f"{gs_name}:missing-gaussian") + continue + if link_name not in link_ids: + skipped.append(f"{gs_name}:{link_name}:missing-link") + continue + objects_info.append((gs_name, gs_renderer.gaussian_start_indices[gs_name], gs_renderer.gaussian_end_indices[gs_name])) + body_ids.append(link_ids[link_name]) + + if skipped: + print("Skipping Gaussian links: " + ", ".join(skipped), flush=True) + if not objects_info: + return + + gs_renderer.gs_idx_start = np.array([start for _, start, _ in objects_info]) + gs_renderer.gs_idx_end = np.array([end for _, _, end in objects_info]) + gs_renderer.gs_body_ids = np.array(body_ids) + gs_renderer.set_objects_mapping(objects_info) + print(f"Bound 3DGS assets to MJCF links: {', '.join(name for name, _, _ in objects_info)}", flush=True) + + +def rgb_to_uint8(rgb_tensor) -> np.ndarray: + rgb = rgb_tensor.detach().cpu().numpy() + if rgb.dtype == np.uint8: + return rgb + return np.clip(rgb * 255.0, 0, 255).astype(np.uint8) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_sweep_motrix_scene_completion.py b/vendor/gs_playground/scripts/real2sim_sweep_motrix_scene_completion.py new file mode 100644 index 0000000..d499be8 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_sweep_motrix_scene_completion.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Sweep MotrixSim scene-completion parameters for a Real2Sim package. + +The sweep keeps the reconstructed 3DGS and textured scene mesh fixed, varies +the photo-completion backdrop margin, packages each candidate, runs the MuJoCo +offscreen verifier, and ranks candidates by the verifier's render-quality +score. It is intended for same-capture candidate comparison, not as an absolute +commercial-quality metric. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Sweep Real2Sim Motrix scene-completion backdrop margins") + parser.add_argument("--source", type=Path, required=True, help="Original capture directory or source path") + parser.add_argument("--scene-ply", type=Path, required=True) + parser.add_argument("--scene-mesh", type=Path, required=True) + parser.add_argument("--cameras-json", type=Path, required=True) + parser.add_argument("--preview-image", type=Path, required=True) + parser.add_argument("--mesh-cleanup-report", type=Path, required=True) + parser.add_argument("--texture-bake-report", type=Path, required=True) + parser.add_argument("--completion-report", type=Path, required=True) + parser.add_argument("--completion-texture", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--name-prefix", default="real2sim_scene_completion_sweep") + parser.add_argument("--margins", default="0,0.15,0.25,0.35,0.55", help="Comma-separated backdrop margins to test") + parser.add_argument("--scene-faces", type=int, default=30_000, help="Default scene mesh LOD face budget") + parser.add_argument("--scene-face-options", default="", help="Comma-separated scene mesh LOD face budgets to sweep") + parser.add_argument("--min-scene-vertices", type=int, default=100_000) + parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"]) + parser.add_argument("--prompt", default="photogrammetry textured scene with swept photo completion backdrop") + parser.add_argument("--python", default=sys.executable) + parser.add_argument("--archive", action="store_true", help="Write a .tar.gz for each candidate package") + parser.add_argument("--skip-existing", action="store_true", help="Reuse package and verifier output directories when present") + args = parser.parse_args() + + margins = parse_margins(args.margins) + face_options = parse_int_options(args.scene_face_options) if args.scene_face_options else [int(args.scene_faces)] + args.output_root.mkdir(parents=True, exist_ok=True) + candidates: list[dict[str, Any]] = [] + for margin in margins: + for scene_faces in face_options: + candidates.append(run_candidate(args, margin, scene_faces)) + + ranked = rank_candidates(candidates) + report = { + "format": "gs-playground-real2sim-scene-completion-sweep-v1", + "target_quality": "polycam_realitycapture_like_space", + "margins": margins, + "scene_face_options": face_options, + "best": ranked[0] if ranked else None, + "candidates": ranked, + "notes": [ + "Scores are comparable only within the same capture, cameras, render size, and verifier version.", + "A high score does not prove Polycam/RealityCapture-level fidelity; inspect contact sheets before accepting.", + ], + } + report_path = args.output_root / "sweep_report.json" + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + + +def parse_margins(raw: str) -> list[float]: + values: list[float] = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + value = float(item) + if value < 0: + raise SystemExit(f"margin must be non-negative: {value}") + values.append(value) + if not values: + raise SystemExit("at least one margin is required") + return values + + +def parse_int_options(raw: str) -> list[int]: + values: list[int] = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + value = int(item) + if value <= 0: + raise SystemExit(f"scene face option must be positive: {value}") + values.append(value) + if not values: + raise SystemExit("at least one scene face option is required") + return values + + +def run_candidate(args: argparse.Namespace, margin: float, scene_faces: int) -> dict[str, Any]: + token = candidate_token(margin, scene_faces) + package_dir = args.output_root / f"{args.name_prefix}_{token}_package" + verify_dir = args.output_root / f"{args.name_prefix}_{token}_verify" + report_path = verify_dir / "report.json" + + if not (args.skip_existing and report_path.is_file()): + if package_dir.exists(): + shutil.rmtree(package_dir) + if verify_dir.exists(): + shutil.rmtree(verify_dir) + verify_dir.mkdir(parents=True, exist_ok=True) + + env = os.environ.copy() + env.update( + { + "REAL2SIM_SCENE_COMPLETION_REPORT": args.completion_report.as_posix(), + "REAL2SIM_SCENE_COMPLETION_TEXTURE": args.completion_texture.as_posix(), + "REAL2SIM_SCENE_COMPLETION_BACKDROP_MARGIN": str(margin), + "REAL2SIM_MOTRIX_SCENE_VISUAL": "photogrammetry", + "REAL2SIM_MOTRIX_PREVIEW_MESH_FACES": str(max(int(scene_faces), 1)), + } + ) + package_cmd = [ + args.python, + "-m", + "real2sim", + "package-existing", + "--source", + args.source.as_posix(), + "--scene-ply", + args.scene_ply.as_posix(), + "--scene-mesh", + args.scene_mesh.as_posix(), + "--cameras-json", + args.cameras_json.as_posix(), + "--preview-image", + args.preview_image.as_posix(), + "--mesh-cleanup-report", + args.mesh_cleanup_report.as_posix(), + "--texture-bake-report", + args.texture_bake_report.as_posix(), + "--output", + package_dir.as_posix(), + "--prompt", + args.prompt, + "--robot", + args.robot, + ] + run(package_cmd, env=env, cwd=REPO_ROOT) + + verify_cmd = [ + args.python, + (REPO_ROOT / "scripts" / "real2sim_verify_motrix_package.py").as_posix(), + "--package", + package_dir.as_posix(), + "--min-scene-vertices", + str(int(args.min_scene_vertices)), + "--mujoco-offscreen-render-dir", + verify_dir.as_posix(), + "--visual-report-dir", + verify_dir.as_posix(), + ] + with report_path.open("w", encoding="utf-8") as handle: + run(verify_cmd, env={"MUJOCO_GL": "egl", **env}, cwd=REPO_ROOT, stdout=handle) + + if args.archive: + archive_path = package_dir.with_suffix(".tar.gz") + run(["tar", "-C", package_dir.parent.as_posix(), "-czf", archive_path.as_posix(), package_dir.name], cwd=REPO_ROOT) + + verification = json.loads(report_path.read_text(encoding="utf-8")) + return candidate_summary(margin, scene_faces, package_dir, verify_dir, verification) + + +def run( + cmd: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, + stdout: Any | None = None, +) -> None: + process_env = os.environ.copy() + if env: + process_env.update(env) + subprocess.run(cmd, cwd=cwd, env=process_env, stdout=stdout, check=True) + + +def candidate_summary( + margin: float, + scene_faces: int, + package_dir: Path, + verify_dir: Path, + verification: dict[str, Any], +) -> dict[str, Any]: + renders = verification.get("mujoco_offscreen_render", {}).get("renders", []) + scores = [ + float(render.get("quality", {}).get("score", 0.0)) + for render in renders + if isinstance(render, dict) + ] + overview = next((render for render in renders if render.get("camera") == "overview"), {}) if isinstance(renders, list) else {} + closeup = next((render for render in renders if render.get("camera") == "closeup"), {}) if isinstance(renders, list) else {} + avg_score = sum(scores) / len(scores) if scores else 0.0 + return { + "margin": float(margin), + "scene_faces": int(scene_faces), + "score": float(avg_score), + "package": package_dir.as_posix(), + "verify_dir": verify_dir.as_posix(), + "contact_sheet": (verify_dir / "visual_acceptance_contact_sheet.png").as_posix(), + "overview": render_quality_summary(overview), + "closeup": render_quality_summary(closeup), + } + + +def render_quality_summary(render: dict[str, Any]) -> dict[str, Any]: + quality = render.get("quality") if isinstance(render, dict) else {} + return { + "path": render.get("path") if isinstance(render, dict) else None, + "score": float(quality.get("score", 0.0)) if isinstance(quality, dict) else 0.0, + "black_ratio": float(quality.get("black_ratio", 0.0)) if isinstance(quality, dict) else 0.0, + "high_saturation_ratio": float(quality.get("high_saturation_ratio", 0.0)) if isinstance(quality, dict) else 0.0, + } + + +def rank_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + return sorted( + candidates, + key=lambda item: ( + float(item.get("score", 0.0)), + int(item.get("scene_faces", 0)), + -float(item.get("margin", 0.0)), + ), + reverse=True, + ) + + +def margin_token(margin: float) -> str: + return f"margin_{margin:.3f}".replace(".", "p").replace("-", "m") + + +def candidate_token(margin: float, scene_faces: int) -> str: + return f"{margin_token(margin)}_faces_{int(scene_faces)}" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_verify_motrix_package.py b/vendor/gs_playground/scripts/real2sim_verify_motrix_package.py new file mode 100644 index 0000000..a08b949 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_verify_motrix_package.py @@ -0,0 +1,1827 @@ +#!/usr/bin/env python3 +"""Verify a MotrixSim-importable Real2Sim scene package. + +The verifier accepts either an unpacked package directory or a ``.tar.gz`` +archive. It checks the Motrix import entrypoints, required scene assets, and +quality metrics written by the one-image AnySplat workflow. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import tarfile +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any + + +COMMERCIAL_ACCEPTANCE_FORMAT = "gs-playground-real2sim-commercial-acceptance-v1" +COMMERCIAL_ACCEPTANCE_REPORT_PATH = "artifacts/quality/commercial_acceptance.json" + +REQUIRED_COMMERCIAL_ACCEPTANCE_ENTRYPOINTS = { + "motrix_import": "motrix_import.json", + "package_manifest": "package_manifest.json", + "scene_xml": "mjcf/scene.xml", + "viewer_config": "configs/go2_real2sim.json", + "scene_gaussians": "3dgs/scene.ply", +} + +REQUIRED_COMMERCIAL_ACCEPTANCE_REPORTS = { + "commercial_readiness": "artifacts/quality/commercial_readiness.json", + "room_completeness": "artifacts/quality/room_completeness.json", + "metric_scale": "artifacts/quality/metric_scale.json", + "navigation_acceptance": "artifacts/quality/navigation_acceptance.json", + "material_readiness": "artifacts/quality/material_readiness.json", + "task_affordance": "artifacts/quality/task_affordance.json", + "movable_assets": "artifacts/assets/movable_assets.json", + "static_structure_layers": "artifacts/scene_structure/static_structure.json", +} + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify a Real2Sim MotrixSim import package") + parser.add_argument("--package", type=Path, required=True, help="Package directory or .tar.gz archive") + parser.add_argument("--min-scene-vertices", type=int, default=100_000) + parser.add_argument("--min-source-preview-psnr", type=float, default=18.0) + parser.add_argument("--max-source-preview-mae", type=float, default=0.10) + parser.add_argument("--visual-report-dir", type=Path, help="Write a visual acceptance JSON report and contact sheet") + parser.add_argument("--motrix-viewer-render", type=Path, help="Optional Motrix viewer screenshot/render to include") + parser.add_argument("--mujoco-offscreen-render-dir", type=Path, help="Render MJCF cameras with mujoco.Renderer into this directory") + parser.add_argument("--mujoco-offscreen-cameras", default="overview,closeup", help="Comma-separated MJCF cameras to render") + parser.add_argument("--mujoco-offscreen-width", type=int, default=640) + parser.add_argument("--mujoco-offscreen-height", type=int, default=480) + parser.add_argument("--mujoco-offscreen-max-geom", type=int, default=0, help="Override mujoco.Renderer max_geom; 0 derives it from model.ngeom") + parser.add_argument( + "--require-commercial-acceptance", + action="store_true", + help="Fail strict commercial visual quality unless the package declares and passes commercial_acceptance.json", + ) + parser.add_argument( + "--require-capture-acceptance", + action="store_true", + help="Fail strict commercial visual quality unless capture acceptance is present and accepted", + ) + args = parser.parse_args() + if (args.require_commercial_acceptance or args.require_capture_acceptance) and args.visual_report_dir is None: + parser.error("--require-commercial-acceptance/--require-capture-acceptance requires --visual-report-dir") + + with package_root(args.package) as root: + report = verify_package( + root, + min_scene_vertices=args.min_scene_vertices, + min_source_preview_psnr=args.min_source_preview_psnr, + max_source_preview_mae=args.max_source_preview_mae, + ) + report["physics_task_readiness"] = physics_task_readiness_report(root, report) + offscreen_report = None + if args.mujoco_offscreen_render_dir is not None: + cameras = [camera.strip() for camera in args.mujoco_offscreen_cameras.split(",") if camera.strip()] + offscreen_report = render_mujoco_offscreen( + root, + report, + args.mujoco_offscreen_render_dir, + cameras=cameras, + width=args.mujoco_offscreen_width, + height=args.mujoco_offscreen_height, + max_geom=args.mujoco_offscreen_max_geom, + ) + report["mujoco_offscreen_render"] = offscreen_report + if isinstance(report.get("physics_task_readiness"), dict): + report["physics_task_readiness"]["mujoco_simulation"] = offscreen_report.get("simulation_stability") + report["physics_task_readiness"]["initial_contact_diagnostics"] = offscreen_report.get( + "initial_contact_diagnostics" + ) + report["physics_task_readiness"]["settle_drift_probe"] = offscreen_report.get("settle_drift_probe") + report["physics_task_readiness"]["passed"] = physics_task_readiness_passed( + report["physics_task_readiness"] + ) + report["robot_task_readiness"] = robot_task_readiness_report(root, report) + report["task_execution_probe"] = task_execution_probe_report(root, report) + if args.visual_report_dir is not None: + viewer_render = args.motrix_viewer_render + viewer_render_label = "Motrix viewer render" + if viewer_render is None and offscreen_report is not None: + renders = offscreen_report.get("renders") + if isinstance(renders, list) and renders: + first_render = renders[0] + if isinstance(first_render, dict) and first_render.get("path"): + viewer_render = Path(str(first_render["path"])) + viewer_render_label = "MuJoCo offscreen render" + report["visual_acceptance_report"] = write_visual_acceptance_report( + root, + report, + args.visual_report_dir, + motrix_viewer_render=viewer_render, + motrix_viewer_render_label=viewer_render_label, + ) + report["strict_commercial_visual_quality"] = write_strict_commercial_visual_quality_report( + root, + report, + args.visual_report_dir, + visual_acceptance=report["visual_acceptance_report"], + require_commercial_acceptance=args.require_commercial_acceptance, + require_capture_acceptance=args.require_capture_acceptance, + ) + print(json.dumps(report, indent=2, ensure_ascii=False)) + if args.require_commercial_acceptance or args.require_capture_acceptance: + strict_summary = report.get("strict_commercial_visual_quality", {}).get("summary", {}) + if not strict_summary.get("strict_commercial_visual_grade"): + return 1 + return 0 + + +def package_root(path: Path): + class _PackageRoot: + def __init__(self, source: Path) -> None: + self.source = source + self.tmp: tempfile.TemporaryDirectory[str] | None = None + self.root: Path | None = None + + def __enter__(self) -> Path: + if self.source.is_dir(): + self.root = self.source.resolve() + return self.root + if not self.source.is_file(): + raise SystemExit(f"package path does not exist: {self.source}") + self.tmp = tempfile.TemporaryDirectory() + tmp_root = Path(self.tmp.name) + with tarfile.open(self.source, "r:gz") as tar: + tar.extractall(tmp_root, filter="data") + candidates = [path for path in tmp_root.iterdir() if path.is_dir()] + if len(candidates) != 1: + raise SystemExit(f"archive must contain exactly one package directory: {self.source}") + self.root = candidates[0] + return self.root + + def __exit__(self, exc_type, exc, tb) -> None: + if self.tmp is not None: + self.tmp.cleanup() + + return _PackageRoot(path) + + +def verify_package( + root: Path, + *, + min_scene_vertices: int, + min_source_preview_psnr: float, + max_source_preview_mae: float, +) -> dict[str, Any]: + motrix_import_path = require_file(root / "motrix_import.json", "Motrix import manifest") + motrix_import = read_json(motrix_import_path) + if motrix_import.get("format") != "gs-playground-real2sim-motrix-import-v1": + raise SystemExit("motrix_import.json has an unsupported format") + if motrix_import.get("import_type") != "mjcf_scene_with_3dgs": + raise SystemExit("motrix_import.json import_type must be mjcf_scene_with_3dgs") + + scene_xml = require_file(root / str(motrix_import.get("scene_xml", "")), "MJCF scene") + config_json = require_file(root / str(motrix_import.get("config", "")), "viewer config") + ET.parse(scene_xml) + config = read_json(config_json) + + scene_gaussians = motrix_import.get("scene_gaussians") + if not isinstance(scene_gaussians, dict) or not scene_gaussians.get("scene"): + raise SystemExit("motrix_import.json missing scene_gaussians.scene") + scene_ply = require_file(root / str(scene_gaussians["scene"]), "scene 3DGS PLY") + vertices = count_ply_vertices(scene_ply) + if vertices < min_scene_vertices: + raise SystemExit(f"scene vertex count too low: {vertices} < {min_scene_vertices}") + + config_gaussians = config.get("scene_gaussians") + if not isinstance(config_gaussians, dict) or not config_gaussians.get("scene"): + raise SystemExit("viewer config missing scene_gaussians.scene") + require_file(resolve_config_path(config_json, config_gaussians["scene"]), "config scene 3DGS PLY") + + camera_artifacts = motrix_import.get("camera_artifacts") + if not isinstance(camera_artifacts, dict): + raise SystemExit("motrix_import.json missing camera_artifacts") + for key in ("cameras_json", "source_gs_preview"): + require_file(root / str(camera_artifacts.get(key, "")), f"camera artifact {key}") + + preview_path = require_file(root / str(motrix_import.get("preview", "")), "preview image") + reconstruction_quality = load_reconstruction_quality( + root, + motrix_import, + min_source_preview_psnr=min_source_preview_psnr, + max_source_preview_mae=max_source_preview_mae, + ) + + viewer_args = motrix_import.get("viewer_args") + if not isinstance(viewer_args, list) or "--show-source-gs" not in viewer_args: + raise SystemExit("viewer_args must include --show-source-gs") + wuying_launch_command = motrix_import.get("wuying_archive_launch") + if not isinstance(wuying_launch_command, str) or "--with-gs" not in wuying_launch_command: + raise SystemExit("wuying_archive_launch must be present and include --with-gs") + viewer_verification = viewer_verification_summary( + wuying_launch_command=wuying_launch_command, + viewer_args=viewer_args, + ) + + synthetic_quality = None + quality_path = root / "synthetic_multiview_quality.json" + if quality_path.is_file(): + synthetic_quality = read_json(quality_path) + + return { + "ok": True, + "root": root.as_posix(), + "scene_xml": scene_xml.relative_to(root).as_posix(), + "config": config_json.relative_to(root).as_posix(), + "scene_ply": scene_ply.relative_to(root).as_posix(), + "scene_ply_vertices": vertices, + "preview": preview_path.relative_to(root).as_posix(), + "source_preview_metrics": reconstruction_quality["source_preview_metrics"], + "source_preview_quality_gate": reconstruction_quality["source_preview_quality_gate"], + "reconstruction_quality": reconstruction_quality, + "motrix_viewer_verification": viewer_verification, + "synthetic_multiview_quality": synthetic_quality, + } + + +def require_file(path: Path, label: str) -> Path: + if not path.is_file() or path.stat().st_size <= 0: + raise SystemExit(f"missing or empty {label}: {path}") + return path + + +def read_json(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise SystemExit(f"JSON file must contain an object: {path}") + return data + + +def resolve_config_path(config_json: Path, value: Any) -> Path: + path = Path(str(value)) + if path.is_absolute(): + return path + return (config_json.parent / path).resolve() + + +def count_ply_vertices(path: Path) -> int: + with path.open("rb") as handle: + for raw in handle: + line = raw.decode("ascii", errors="ignore").strip() + if line.startswith("element vertex "): + return int(line.split()[-1]) + if line == "end_header": + break + raise SystemExit(f"PLY vertex count not found: {path}") + + +def require_metrics(metadata: dict[str, Any], min_psnr: float, max_mae: float) -> dict[str, float]: + metrics = metadata.get("source_preview_metrics") + if not isinstance(metrics, dict): + raise SystemExit("reconstruction metadata missing source_preview_metrics") + psnr = float(metrics.get("psnr", 0.0)) + mae = float(metrics.get("mae", 1.0)) + if psnr < min_psnr: + raise SystemExit(f"source preview PSNR too low: {psnr:.3f} < {min_psnr:.3f}") + if mae > max_mae: + raise SystemExit(f"source preview MAE too high: {mae:.4f} > {max_mae:.4f}") + return {"psnr": psnr, "mae": mae, "mse": float(metrics.get("mse", 0.0))} + + +def load_reconstruction_quality( + root: Path, + motrix_import: dict[str, Any], + *, + min_source_preview_psnr: float, + max_source_preview_mae: float, +) -> dict[str, Any]: + metadata_value = motrix_import.get("reconstruction_metadata") + if metadata_value: + metadata_path = root / str(metadata_value) + metadata = read_json(require_file(metadata_path, "reconstruction metadata")) + metrics = require_metrics(metadata, min_source_preview_psnr, max_source_preview_mae) + gate = metadata.get("source_preview_quality_gate") + if not isinstance(gate, dict) or not gate.get("passed"): + raise SystemExit("source preview quality gate did not pass") + return { + "mode": "source_preview_metadata", + "metadata": metadata_path.relative_to(root).as_posix(), + "source_preview_metrics": metrics, + "source_preview_quality_gate": gate, + "mesh_cleanup": None, + "texture_bake": None, + } + + manifest_path = require_file(root / "package_manifest.json", "package manifest") + manifest = read_json(manifest_path) + package_metadata = manifest.get("metadata") + if not isinstance(package_metadata, dict): + raise SystemExit("package_manifest.json missing metadata") + + mesh_cleanup = read_optional_json(root / "artifacts" / "mesh_cleanup" / "cleanup_report.json") + texture_bake = read_optional_json(root / "artifacts" / "colmap_texture_bake" / "texture_bake_report.json") + if mesh_cleanup is None and texture_bake is None: + raise SystemExit("missing reconstruction metadata or scene reconstruction quality reports") + if texture_bake is not None: + face_count = int(texture_bake.get("face_count", 0) or 0) + drawn_faces = int(texture_bake.get("drawn_face_count", 0) or 0) + coverage = float(texture_bake.get("texture_coverage_ratio", 0.0) or 0.0) + if face_count <= 0 or drawn_faces <= 0 or coverage <= 0.0: + raise SystemExit("texture bake report does not show a textured scene mesh") + + gate = { + "passed": True, + "mode": "scene_reconstruction_reports", + "reason": "Existing multi-image scene package verified with mesh/texture reconstruction reports.", + } + return { + "mode": "scene_reconstruction_reports", + "metadata": "package_manifest.json", + "source_preview_metrics": None, + "source_preview_quality_gate": gate, + "mesh_cleanup": mesh_cleanup, + "texture_bake": texture_bake, + "package_metadata": package_metadata, + } + + +def read_optional_json(path: Path) -> dict[str, Any] | None: + if not path.is_file() or path.stat().st_size <= 0: + return None + return read_json(path) + + +def viewer_verification_summary(*, wuying_launch_command: str, viewer_args: list[Any]) -> dict[str, Any]: + return { + "wuying_launch_command": wuying_launch_command, + "viewer_args": [str(arg) for arg in viewer_args], + "required_log_markers": [ + "Loading 3DGS assets: scene", + "3DGS renderer ready.", + "Scene viewer ready.", + "3DGS rendered source camera.", + "3DGS static preview rendered.", + ], + } + + +def render_mujoco_offscreen( + root: Path, + verification_report: dict[str, Any], + output_dir: Path, + *, + cameras: list[str], + width: int, + height: int, + max_geom: int = 0, +) -> dict[str, Any]: + os.environ.setdefault("MUJOCO_GL", "egl") + try: + import mujoco + import numpy as np + from PIL import Image + except ImportError as exc: + raise SystemExit("MuJoCo offscreen rendering requires mujoco, Pillow, and numpy") from exc + + scene_xml = root / str(verification_report["scene_xml"]) + model = mujoco.MjModel.from_xml_path(scene_xml.as_posix()) + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + initial_contact_diagnostics = mujoco_contact_diagnostics(mujoco, model, data) + settle_drift_probe = mujoco_settle_drift_probe( + mujoco, + model, + physics_task_body_names(verification_report), + ) + output_dir.mkdir(parents=True, exist_ok=True) + renderer_max_geom = mujoco_renderer_max_geom(int(model.ngeom), requested=max_geom) + + renders: list[dict[str, Any]] = [] + for camera_name in cameras: + camera_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_CAMERA, camera_name) + if camera_id < 0: + raise SystemExit(f"MJCF camera not found for offscreen render: {camera_name}") + try: + renderer = mujoco.Renderer(model, height=height, width=width, max_geom=renderer_max_geom) + except TypeError: + renderer = mujoco.Renderer(model, height=height, width=width) + try: + renderer.update_scene(data, camera=camera_id) + rgb = renderer.render() + finally: + renderer.close() + image = Image.fromarray(rgb) + output_path = output_dir / f"{camera_name}_mujoco_offscreen.png" + image.save(output_path) + array = np.asarray(rgb, dtype=np.float32) + quality = offscreen_image_quality(array) + renders.append( + { + "camera": camera_name, + "path": output_path.as_posix(), + "size": [int(width), int(height)], + "mean": float(array.mean()), + "std": float(array.std()), + "nonblank": bool(array.std() > 1.0), + "quality": quality, + } + ) + + if not renders or not all(render["nonblank"] for render in renders): + raise SystemExit("MuJoCo offscreen render was blank or missing") + simulation_stability = mujoco_simulation_stability(mujoco, model, data) + report = { + "ok": True, + "scene_xml": scene_xml.as_posix(), + "model": { + "nbody": int(model.nbody), + "ngeom": int(model.ngeom), + "nmesh": int(model.nmesh), + "ntex": int(model.ntex), + "nmat": int(model.nmat), + "ncam": int(model.ncam), + "renderer_max_geom": int(renderer_max_geom), + }, + "renders": renders, + "initial_contact_diagnostics": initial_contact_diagnostics, + "settle_drift_probe": settle_drift_probe, + "simulation_stability": simulation_stability, + } + report_path = output_dir / "mujoco_offscreen_render_report.json" + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return report + + +def mujoco_simulation_stability(mujoco: Any, model: Any, data: Any, *, step_count: int = 240) -> dict[str, Any]: + import numpy as np + + if not hasattr(mujoco, "mj_step"): + return { + "passed": True, + "skipped": True, + "reason": "mujoco_mj_step_unavailable", + "step_count": int(step_count), + "finite_state": True, + "max_abs_qpos": 0.0, + "max_abs_qvel": 0.0, + "max_abs_qpos_threshold": 10.0, + "max_abs_qvel_threshold": 100.0, + "ncon": int(getattr(data, "ncon", 0)), + } + max_abs_qpos = 0.0 + max_abs_qvel = 0.0 + finite = True + for _ in range(max(int(step_count), 1)): + mujoco.mj_step(model, data) + qpos = np.asarray(data.qpos, dtype=np.float64) + qvel = np.asarray(data.qvel, dtype=np.float64) + finite = finite and bool(np.all(np.isfinite(qpos)) and np.all(np.isfinite(qvel))) + max_abs_qpos = max(max_abs_qpos, float(np.max(np.abs(qpos), initial=0.0))) + max_abs_qvel = max(max_abs_qvel, float(np.max(np.abs(qvel), initial=0.0))) + if not finite: + break + max_qpos_threshold = 10.0 + max_qvel_threshold = 100.0 + passed = bool(finite and max_abs_qpos < max_qpos_threshold and max_abs_qvel < max_qvel_threshold) + return { + "passed": passed, + "step_count": int(step_count), + "finite_state": bool(finite), + "max_abs_qpos": round(max_abs_qpos, 6), + "max_abs_qvel": round(max_abs_qvel, 6), + "max_abs_qpos_threshold": max_qpos_threshold, + "max_abs_qvel_threshold": max_qvel_threshold, + "ncon": int(getattr(data, "ncon", 0)), + } + + +def mujoco_contact_diagnostics(mujoco: Any, model: Any, data: Any, *, max_allowed_penetration: float = 0.25) -> dict[str, Any]: + contacts: list[dict[str, Any]] = [] + max_penetration = 0.0 + for index in range(int(getattr(data, "ncon", 0))): + contact = data.contact[index] + dist = float(contact.dist) + penetration = max(0.0, -dist) + max_penetration = max(max_penetration, penetration) + geom1 = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_GEOM, int(contact.geom1)) + geom2 = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_GEOM, int(contact.geom2)) + contacts.append( + { + "geom1": geom1, + "geom2": geom2, + "dist": round(dist, 6), + "penetration": round(penetration, 6), + } + ) + contacts.sort(key=lambda item: float(item["penetration"]), reverse=True) + return { + "passed": bool(max_penetration <= max_allowed_penetration), + "contact_count": int(getattr(data, "ncon", 0)), + "max_penetration": round(max_penetration, 6), + "max_allowed_penetration": float(max_allowed_penetration), + "deepest_contacts": contacts[:16], + } + + +def mujoco_settle_drift_probe( + mujoco: Any, + model: Any, + body_names: list[str], + *, + step_count: int = 240, + max_allowed_displacement: float = 0.15, + max_allowed_qvel: float = 2.0, +) -> dict[str, Any]: + import numpy as np + + if not hasattr(mujoco, "mj_step"): + return { + "passed": True, + "skipped": True, + "reason": "mujoco_mj_step_unavailable", + "step_count": int(step_count), + "body_count": 0, + "missing_bodies": [], + "max_displacement": 0.0, + "max_allowed_displacement": float(max_allowed_displacement), + "max_abs_qvel": 0.0, + "max_allowed_qvel": float(max_allowed_qvel), + "bodies": [], + } + data = mujoco.MjData(model) + mujoco.mj_forward(model, data) + body_ids: list[tuple[str, int]] = [] + missing: list[str] = [] + for name in body_names: + body_id = mujoco.mj_name2id(model, mujoco.mjtObj.mjOBJ_BODY, name) + if body_id < 0: + missing.append(name) + else: + body_ids.append((name, int(body_id))) + start_positions = {name: np.asarray(data.xpos[body_id], dtype=np.float64).copy() for name, body_id in body_ids} + for _ in range(max(int(step_count), 1)): + mujoco.mj_step(model, data) + body_reports: list[dict[str, Any]] = [] + max_displacement = 0.0 + for name, body_id in body_ids: + end = np.asarray(data.xpos[body_id], dtype=np.float64) + displacement = float(np.linalg.norm(end - start_positions[name])) + max_displacement = max(max_displacement, displacement) + body_reports.append( + { + "body": name, + "displacement": round(displacement, 6), + "start": [round(float(value), 6) for value in start_positions[name]], + "end": [round(float(value), 6) for value in end], + } + ) + qvel = np.asarray(data.qvel, dtype=np.float64) + max_abs_qvel = float(np.max(np.abs(qvel), initial=0.0)) + passed = bool(body_ids and not missing and max_displacement <= max_allowed_displacement and max_abs_qvel <= max_allowed_qvel) + return { + "passed": passed, + "step_count": int(step_count), + "body_count": len(body_ids), + "missing_bodies": missing, + "max_displacement": round(max_displacement, 6), + "max_allowed_displacement": float(max_allowed_displacement), + "max_abs_qvel": round(max_abs_qvel, 6), + "max_allowed_qvel": float(max_allowed_qvel), + "bodies": body_reports, + } + + +def physics_task_readiness_report(root: Path, verification_report: dict[str, Any]) -> dict[str, Any]: + scene_xml = root / str(verification_report["scene_xml"]) + tree = ET.parse(scene_xml) + dynamic_bodies: list[dict[str, Any]] = [] + for body in tree.getroot().iter("body"): + free_joints = free_joint_elements(body) + if not free_joints: + continue + body_name = body.get("name") or "" + inertials = body.findall("inertial") + collision_geoms = [ + geom + for geom in body.findall("geom") + if geom_is_collision_enabled(geom) + ] + masses = [parse_positive_float(inertial.get("mass")) for inertial in inertials] + diaginertias = [parse_float_triplet(inertial.get("diaginertia")) for inertial in inertials] + damping_values = [parse_positive_float(joint.get("damping")) for joint in free_joints if joint.tag == "joint"] + collision_half_extents = [geom_half_extents(geom) for geom in collision_geoms] + max_collision_half_extent = max( + (max(extents) for extents in collision_half_extents if extents is not None), + default=0.0, + ) + friction_values = [geom_friction(geom) for geom in collision_geoms] + damping_ready = bool(damping_values and all(value is not None and value > 0.0 for value in damping_values)) + friction_ready = bool(friction_values and all(value is not None and value[0] >= 0.2 for value in friction_values)) + bounded_collision_proxy = bool(collision_geoms and 0.0 < max_collision_half_extent <= 0.25) + has_positive_mass = any(mass is not None and mass > 0.0 for mass in masses) + plausible_mass = bool(masses and all(mass is not None and 0.02 <= mass <= 5.0 for mass in masses)) + plausible_diaginertia = bool( + diaginertias + and all(values is not None and all(1e-6 <= item <= 1.0 for item in values) for values in diaginertias) + ) + dynamic_bodies.append( + { + "body": body_name, + "freejoint_count": len(free_joints), + "inertial_count": len(inertials), + "positive_mass_count": sum(1 for mass in masses if mass is not None and mass > 0.0), + "mass_values": [round(float(mass), 6) for mass in masses if mass is not None], + "diaginertia_values": [list(values) for values in diaginertias if values is not None], + "plausible_mass": plausible_mass, + "plausible_diaginertia": plausible_diaginertia, + "collision_geom_count": len(collision_geoms), + "free_joint_damping": [joint.get("damping") for joint in free_joints if joint.tag == "joint"], + "damping_ready": damping_ready, + "friction_ready": friction_ready, + "bounded_collision_proxy": bounded_collision_proxy, + "max_collision_half_extent": round(float(max_collision_half_extent), 6), + "collision_geoms": [ + { + "name": geom.get("name"), + "type": geom.get("type", "sphere"), + "contype": geom.get("contype", "1"), + "conaffinity": geom.get("conaffinity", "1"), + "size": geom.get("size"), + "friction": geom.get("friction"), + } + for geom in collision_geoms + ], + "ready": bool( + inertials + and has_positive_mass + and plausible_mass + and plausible_diaginertia + and collision_geoms + and damping_ready + and friction_ready + and bounded_collision_proxy + ), + } + ) + summary = { + "dynamic_body_count": len(dynamic_bodies), + "ready_dynamic_body_count": sum(1 for item in dynamic_bodies if item["ready"]), + "freejoint_body_count": len(dynamic_bodies), + } + report = { + "format": "gs-playground-real2sim-physics-task-readiness-v1", + "scene_xml": scene_xml.as_posix(), + "summary": summary, + "dynamic_bodies": dynamic_bodies, + } + report["passed"] = physics_task_readiness_passed(report) + return report + + +def physics_task_readiness_passed(report: dict[str, Any]) -> bool: + summary = report.get("summary") if isinstance(report.get("summary"), dict) else {} + dynamic_count = int(summary.get("dynamic_body_count") or 0) + ready_count = int(summary.get("ready_dynamic_body_count") or 0) + static_passed = bool(dynamic_count > 0 and ready_count == dynamic_count) + simulation = report.get("mujoco_simulation") + contact = report.get("initial_contact_diagnostics") + settle = report.get("settle_drift_probe") + if isinstance(simulation, dict): + contact_passed = bool(contact.get("passed")) if isinstance(contact, dict) else True + settle_passed = bool(settle.get("passed")) if isinstance(settle, dict) else True + return bool(static_passed and simulation.get("passed") and contact_passed and settle_passed) + return static_passed + + +def physics_task_body_names(verification_report: dict[str, Any]) -> list[str]: + physics = verification_report.get("physics_task_readiness") + physics = physics if isinstance(physics, dict) else {} + bodies = physics.get("dynamic_bodies") if isinstance(physics.get("dynamic_bodies"), list) else [] + return [str(item["body"]) for item in bodies if isinstance(item, dict) and item.get("body")] + + +def robot_task_readiness_report(root: Path, verification_report: dict[str, Any]) -> dict[str, Any]: + scene_xml = root / str(verification_report["scene_xml"]) + config_json = root / str(verification_report["config"]) + config = read_json(config_json) + tree = ET.parse(scene_xml) + cameras = sorted({camera.get("name") for camera in tree.getroot().iter("camera") if camera.get("name")}) + manifest = read_optional_json(root / "package_manifest.json") or {} + objects = manifest.get("objects") if isinstance(manifest.get("objects"), list) else [] + dynamic_names = dynamic_body_names(tree) + target_objects = [ + { + "name": item.get("name"), + "label": item.get("label"), + "dynamic": bool(item.get("dynamic")), + "source_instance_id": item.get("source_instance_id"), + } + for item in objects + if isinstance(item, dict) and bool(item.get("dynamic")) + ] + if not target_objects: + target_objects = [{"name": name, "label": None, "dynamic": True, "source_instance_id": None} for name in dynamic_names] + body_positions = dynamic_body_positions(tree) + finite_targets = all(all(isinstance(value, (int, float)) and abs(float(value)) < 10.0 for value in pos) for pos in body_positions.values()) + physics = verification_report.get("physics_task_readiness") + physics = physics if isinstance(physics, dict) else {} + render_summary = strict_render_quality_summary(verification_report) + robot = config.get("robot") + gates = [ + strict_gate( + "robot_config_present", + isinstance(robot, str) and bool(robot.strip()), + "viewer config names the robot platform expected to consume the Real2Sim package", + {"robot": robot}, + ), + strict_gate( + "task_targets_present", + bool(target_objects), + "package exposes at least one dynamic target object for downstream navigation, reach, or manipulation tasks", + {"target_count": len(target_objects), "targets": target_objects}, + ), + strict_gate( + "task_targets_physics_ready", + bool(physics.get("passed")), + "all dynamic task targets have contact-ready physics parameters and stable MuJoCo simulation", + physics, + ), + strict_gate( + "observation_cameras_present", + {"overview", "closeup"}.issubset(set(cameras)), + "MJCF includes standard overview and closeup cameras for task observation and review", + {"cameras": cameras, "required": ["overview", "closeup"]}, + ), + strict_gate( + "observation_render_ready", + bool(render_summary.get("passed")), + "offscreen task-observation renders are nonblank and balanced", + render_summary, + ), + strict_gate( + "finite_task_target_poses", + bool(body_positions and finite_targets), + "dynamic task target body poses are finite and inside the verifier workspace envelope", + {"body_positions": body_positions, "max_abs_coordinate": 10.0}, + ), + ] + blockers = [gate["id"] for gate in gates if not gate["passed"]] + return { + "format": "gs-playground-real2sim-robot-task-readiness-v1", + "passed": not blockers, + "summary": { + "robot": robot, + "target_count": len(target_objects), + "camera_count": len(cameras), + "blockers": blockers, + }, + "gates": gates, + "task_targets": target_objects, + } + + +def task_execution_probe_report(root: Path, verification_report: dict[str, Any]) -> dict[str, Any]: + scene_xml = root / str(verification_report["scene_xml"]) + tree = ET.parse(scene_xml) + probes = dynamic_task_execution_probes(tree) + robot_task = verification_report.get("robot_task_readiness") + robot_task = robot_task if isinstance(robot_task, dict) else {} + gates = [ + strict_gate( + "execution_targets_present", + bool(probes), + "at least one dynamic task target has an execution probe", + {"probe_count": len(probes)}, + ), + strict_gate( + "collision_targets_present", + bool(probes and all(probe["collision_geom_count"] > 0 for probe in probes)), + "each execution target has an enabled collision proxy for task contact", + {"probes": probes}, + ), + strict_gate( + "manipulation_proxy_scale", + bool(probes and all(0.015 <= float(probe["max_collision_half_extent"]) <= 0.25 for probe in probes)), + "target collision proxies are within the desktop manipulation scale envelope", + {"min_half_extent": 0.015, "max_half_extent": 0.25, "probes": probes}, + ), + strict_gate( + "approach_waypoints_valid", + bool(probes and all(probe["approach_waypoint_count"] >= 4 and probe["approach_waypoints_in_workspace"] for probe in probes)), + "each target has four finite cardinal approach waypoints inside the verifier workspace", + {"workspace_max_abs_coordinate": 10.0, "probes": probes}, + ), + strict_gate( + "robot_task_gate_already_passed", + bool(robot_task.get("passed")), + "task execution probes are only accepted after robot task readiness passes", + robot_task, + ), + ] + blockers = [gate["id"] for gate in gates if not gate["passed"]] + return { + "format": "gs-playground-real2sim-task-execution-probe-v1", + "passed": not blockers, + "summary": { + "probe_count": len(probes), + "blockers": blockers, + }, + "gates": gates, + "probes": probes, + } + + +def dynamic_task_execution_probes(tree: ET.ElementTree) -> list[dict[str, Any]]: + probes: list[dict[str, Any]] = [] + for body in tree.getroot().iter("body"): + if not free_joint_elements(body): + continue + body_name = body.get("name") or "" + body_pos = parse_float_list(body.get("pos"), expected=3) or [0.0, 0.0, 0.0] + collision_geoms = [geom for geom in body.findall("geom") if geom_is_collision_enabled(geom)] + collision_centers: list[list[float]] = [] + half_extents: list[tuple[float, float, float]] = [] + for geom in collision_geoms: + local_pos = parse_float_list(geom.get("pos"), expected=3) or [0.0, 0.0, 0.0] + collision_centers.append([body_pos[idx] + local_pos[idx] for idx in range(3)]) + extents = geom_half_extents(geom) + if extents is not None: + half_extents.append(extents) + if collision_centers: + target_center = [ + sum(center[idx] for center in collision_centers) / float(len(collision_centers)) + for idx in range(3) + ] + else: + target_center = [float(value) for value in body_pos] + max_half = max((max(extents) for extents in half_extents), default=0.0) + standoff = max(0.25, min(0.6, 2.5 * max_half + 0.1)) + z_clearance = min(max_half + 0.05, 0.35) + waypoints = [ + [target_center[0] + standoff, target_center[1], target_center[2] + z_clearance], + [target_center[0] - standoff, target_center[1], target_center[2] + z_clearance], + [target_center[0], target_center[1] + standoff, target_center[2] + z_clearance], + [target_center[0], target_center[1] - standoff, target_center[2] + z_clearance], + ] + in_workspace = all(all(abs(float(value)) <= 10.0 for value in waypoint) for waypoint in waypoints) + probes.append( + { + "body": body_name, + "target_center": [round(float(value), 6) for value in target_center], + "collision_geom_count": len(collision_geoms), + "max_collision_half_extent": round(float(max_half), 6), + "standoff_distance": round(float(standoff), 6), + "approach_waypoint_count": len(waypoints), + "approach_waypoints_in_workspace": bool(in_workspace), + "approach_waypoints": [[round(float(value), 6) for value in waypoint] for waypoint in waypoints], + } + ) + return probes + + +def dynamic_body_names(tree: ET.ElementTree) -> list[str]: + names: list[str] = [] + for body in tree.getroot().iter("body"): + if not free_joint_elements(body): + continue + if body.get("name"): + names.append(str(body.get("name"))) + return names + + +def dynamic_body_positions(tree: ET.ElementTree) -> dict[str, list[float]]: + positions: dict[str, list[float]] = {} + for body in tree.getroot().iter("body"): + if not free_joint_elements(body): + continue + name = body.get("name") or "" + pos = parse_float_list(body.get("pos"), expected=3) + if pos is not None: + positions[name] = [round(float(value), 6) for value in pos] + return positions + + +def free_joint_elements(body: ET.Element) -> list[ET.Element]: + free_joints = list(body.findall("freejoint")) + free_joints.extend(joint for joint in body.findall("joint") if joint.get("type") == "free") + return free_joints + + +def geom_is_collision_enabled(geom: ET.Element) -> bool: + contype = parse_int_attr(geom.get("contype"), default=1) + conaffinity = parse_int_attr(geom.get("conaffinity"), default=1) + return bool(contype != 0 or conaffinity != 0) + + +def parse_int_attr(value: str | None, *, default: int) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except ValueError: + return default + + +def parse_positive_float(value: str | None) -> float | None: + if value is None: + return None + try: + parsed = float(str(value).split()[0]) + except (TypeError, ValueError): + return None + return parsed if parsed > 0.0 else None + + +def parse_float_triplet(value: str | None) -> tuple[float, float, float] | None: + if value is None: + return None + try: + values = [float(item) for item in str(value).split()] + except ValueError: + return None + if len(values) != 3: + return None + return tuple(values) + + +def parse_float_list(value: str | None, *, expected: int) -> list[float] | None: + if value is None: + return None + try: + values = [float(item) for item in str(value).split()] + except ValueError: + return None + if len(values) != expected: + return None + return values + + +def geom_half_extents(geom: ET.Element) -> tuple[float, float, float] | None: + size = geom.get("size") + if not size: + return None + try: + values = [abs(float(item)) for item in str(size).split()] + except ValueError: + return None + if not values: + return None + if len(values) == 1: + values = values * 3 + elif len(values) == 2: + values = [values[0], values[0], values[1]] + return tuple(values[:3]) + + +def geom_friction(geom: ET.Element) -> tuple[float, float, float] | None: + value = geom.get("friction") + if not value: + return None + try: + values = [float(item) for item in str(value).split()] + except ValueError: + return None + if len(values) < 3: + return None + return tuple(values[:3]) + + +def mujoco_renderer_max_geom(model_ngeom: int, *, requested: int = 0) -> int: + if requested > 0: + return max(int(requested), 10_000) + return max(int(model_ngeom) + 1_000, 10_000) + + +def offscreen_image_quality(array: Any) -> dict[str, float]: + import numpy as np + + rgb = np.asarray(array, dtype=np.float32) + if rgb.max(initial=0.0) > 1.0: + rgb = rgb / 255.0 + if rgb.ndim != 3 or rgb.shape[2] < 3: + raise SystemExit("offscreen render must be an RGB image") + rgb = np.clip(rgb[:, :, :3], 0.0, 1.0) + luminance = 0.2126 * rgb[:, :, 0] + 0.7152 * rgb[:, :, 1] + 0.0722 * rgb[:, :, 2] + chroma = rgb.max(axis=2) - rgb.min(axis=2) + black_ratio = float(np.mean(luminance < 0.03)) + dark_ratio = float(np.mean(luminance < 0.08)) + mean_luminance = float(np.mean(luminance)) + std_luminance = float(np.std(luminance)) + high_saturation_ratio = float(np.mean(chroma > 0.35)) + score = 1.0 + score -= 1.5 * black_ratio + score -= 2.0 * max(0.0, high_saturation_ratio - 0.75) + score -= 0.8 * max(0.0, mean_luminance - 0.72) + score -= 0.7 * max(0.0, 0.18 - std_luminance) + return { + "black_ratio": black_ratio, + "dark_ratio": dark_ratio, + "mean_luminance": mean_luminance, + "std_luminance": std_luminance, + "high_saturation_ratio": high_saturation_ratio, + "score": float(min(max(score, 0.0), 1.0)), + } + + +def write_visual_acceptance_report( + root: Path, + verification_report: dict[str, Any], + output_dir: Path, + *, + motrix_viewer_render: Path | None = None, + motrix_viewer_render_label: str = "Motrix viewer render", +) -> dict[str, Any]: + try: + import numpy as np + from PIL import Image, ImageDraw + except ImportError as exc: + raise SystemExit("visual report generation requires Pillow and numpy") from exc + + output_dir.mkdir(parents=True, exist_ok=True) + source_preview = require_file(root / str(verification_report["preview"]), "source GS preview") + source_image, source_selection = select_visual_source_image(root, reference_image=source_preview) + pointcloud_preview = root / "pointcloud_projection_preview.png" + panels: list[tuple[str, Path]] = [ + ("source image", source_image), + ("3DGS source render", source_preview), + ] + if pointcloud_preview.is_file(): + panels.append(("pointcloud projection", pointcloud_preview)) + if motrix_viewer_render is not None: + panels.append((motrix_viewer_render_label, require_file(motrix_viewer_render, motrix_viewer_render_label))) + + panel_size = (360, 240) + header_h = 34 + contact = Image.new("RGB", (panel_size[0] * len(panels), panel_size[1] + header_h), "white") + draw = ImageDraw.Draw(contact) + image_summaries: dict[str, dict[str, Any]] = {} + for index, (label, path) in enumerate(panels): + image = Image.open(path).convert("RGB") + image_summaries[label] = {"path": path.as_posix(), "size": list(image.size)} + thumb = fit_image(image, panel_size) + x = index * panel_size[0] + draw.text((x + 10, 10), label, fill=(20, 20, 20)) + contact.paste(thumb, (x + (panel_size[0] - thumb.width) // 2, header_h + (panel_size[1] - thumb.height) // 2)) + + contact_sheet = output_dir / "visual_acceptance_contact_sheet.png" + contact.save(contact_sheet) + + source = Image.open(source_image).convert("RGB") + preview = Image.open(source_preview).convert("RGB").resize(source.size) + diff = np.asarray(source, dtype=np.float32) / 255.0 - np.asarray(preview, dtype=np.float32) / 255.0 + resized_mae = float(np.mean(np.abs(diff))) + + report = { + "ok": bool(verification_report["ok"]), + "contact_sheet": contact_sheet.as_posix(), + "images": image_summaries, + "source_selection": source_selection, + "source_to_preview_resized_mae": resized_mae, + "quality_gate": verification_report["source_preview_quality_gate"], + "source_preview_metrics": verification_report["source_preview_metrics"], + "reconstruction_quality": verification_report.get("reconstruction_quality"), + "notes": [ + "Use the contact sheet for human visual acceptance.", + "The resized MAE is a coarse sanity check because source and preview cameras/aspect ratios can differ.", + ], + } + report_path = output_dir / "visual_acceptance_report.json" + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + return {"json": report_path.as_posix(), "contact_sheet": contact_sheet.as_posix(), "resized_mae": resized_mae} + + +def write_strict_commercial_visual_quality_report( + root: Path, + verification_report: dict[str, Any], + output_dir: Path, + *, + visual_acceptance: dict[str, Any] | None = None, + require_commercial_acceptance: bool = False, + require_capture_acceptance: bool = False, +) -> dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + report = strict_commercial_visual_quality_report( + root, + verification_report, + visual_acceptance=visual_acceptance, + require_commercial_acceptance=require_commercial_acceptance, + require_capture_acceptance=require_capture_acceptance, + ) + report_path = output_dir / "strict_commercial_visual_quality.json" + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return { + "json": report_path.as_posix(), + "summary": report["summary"], + "blockers": report["summary"]["blockers"], + } + + +def strict_commercial_visual_quality_report( + root: Path, + verification_report: dict[str, Any], + *, + visual_acceptance: dict[str, Any] | None = None, + require_commercial_acceptance: bool = False, + require_capture_acceptance: bool = False, +) -> dict[str, Any]: + commercial = read_optional_json(root / "artifacts" / "quality" / "commercial_readiness.json") or {} + motrix_import = read_optional_json(root / "motrix_import.json") or {} + commercial_acceptance = read_optional_json(root / "artifacts" / "quality" / "commercial_acceptance.json") or {} + room = read_optional_json(root / "artifacts" / "quality" / "room_completeness.json") or {} + metric_scale = read_optional_json(root / "artifacts" / "quality" / "metric_scale.json") or {} + navigation_acceptance = read_optional_json(root / "artifacts" / "quality" / "navigation_acceptance.json") or {} + material_readiness = read_optional_json(root / "artifacts" / "quality" / "material_readiness.json") or {} + task_affordance = read_optional_json(root / "artifacts" / "quality" / "task_affordance.json") or {} + movable = read_optional_json(root / "artifacts" / "assets" / "movable_assets.json") or {} + commercial_summary = commercial.get("summary") if isinstance(commercial.get("summary"), dict) else {} + commercial_acceptance_summary = commercial_acceptance.get("summary") if isinstance(commercial_acceptance.get("summary"), dict) else {} + room_summary = room.get("summary") if isinstance(room.get("summary"), dict) else {} + metric_scale_summary = metric_scale.get("summary") if isinstance(metric_scale.get("summary"), dict) else {} + navigation_summary = navigation_acceptance.get("summary") if isinstance(navigation_acceptance.get("summary"), dict) else {} + material_summary = material_readiness.get("summary") if isinstance(material_readiness.get("summary"), dict) else {} + task_affordance_summary = task_affordance.get("summary") if isinstance(task_affordance.get("summary"), dict) else {} + movable_quality = movable_asset_quality_summary(movable) + reconstruction = verification_report.get("reconstruction_quality") + reconstruction = reconstruction if isinstance(reconstruction, dict) else {} + texture = reconstruction.get("texture_bake") if isinstance(reconstruction.get("texture_bake"), dict) else {} + mesh_cleanup = reconstruction.get("mesh_cleanup") if isinstance(reconstruction.get("mesh_cleanup"), dict) else {} + visual_detail = load_visual_acceptance_detail(visual_acceptance) + render_summary = strict_render_quality_summary(verification_report) + physics_task = verification_report.get("physics_task_readiness") + physics_task = physics_task if isinstance(physics_task, dict) else {} + robot_task = verification_report.get("robot_task_readiness") + robot_task = robot_task if isinstance(robot_task, dict) else {} + execution_probe = verification_report.get("task_execution_probe") + execution_probe = execution_probe if isinstance(execution_probe, dict) else {} + capture_acceptance = capture_space_acceptance_summary(root) + capture_acceptance_required = bool(require_capture_acceptance or require_commercial_acceptance) + material_required = material_readiness_required(commercial, motrix_import) + task_affordance_required = task_affordance_required_for_package(commercial, motrix_import) + commercial_acceptance_required = require_commercial_acceptance or commercial_acceptance_required_for_package( + commercial, + motrix_import, + ) + commercial_acceptance_declaration = commercial_acceptance_declaration_report( + motrix_import, + required=commercial_acceptance_required, + ) + commercial_acceptance_schema = commercial_acceptance_schema_report( + commercial_acceptance, + required=commercial_acceptance_required, + ) + commercial_acceptance_integrity = commercial_acceptance_integrity_report( + root, + commercial_acceptance, + required=commercial_acceptance_required, + ) + source_to_preview_mae = visual_detail.get("source_to_preview_resized_mae") + source_mae_target = 0.16 + source_mae_score = source_fidelity_score(source_to_preview_mae, target=source_mae_target) + texture_face_ratio = texture_textured_face_ratio(texture) + room_score = float(room_summary.get("score") or 0.0) + engineering_score = 1.0 if commercial_summary.get("commercial_grade") else float(commercial_summary.get("score") or 0.0) + score = round( + min( + 1.0, + 0.25 * engineering_score + + 0.25 * float(render_summary["average_score"]) + + 0.2 * source_mae_score + + 0.15 * min(room_score, 1.0) + + 0.1 * min(texture_face_ratio, 1.0) + + 0.02 * min(float(mesh_cleanup.get("kept_area_ratio") or 0.0), 1.0) + + 0.02 * (1.0 if physics_task.get("passed") else 0.0) + + 0.005 * (1.0 if robot_task.get("passed") else 0.0) + + 0.005 * (1.0 if execution_probe.get("passed") else 0.0), + ), + 6, + ) + gates = [ + strict_gate( + "engineering_import_readiness", + bool(commercial_summary.get("commercial_grade")), + "package passes core Real2Sim import, structure, object, collision, navigation, and texture gates", + {"commercial_readiness": commercial_summary}, + ), + strict_gate( + "commercial_acceptance_manifest", + bool( + (not commercial_acceptance_required and not commercial_acceptance_summary) + or ( + commercial_acceptance_declaration["passed"] + and commercial_acceptance_schema["passed"] + and commercial_acceptance_summary.get("accepted") + and commercial_acceptance_integrity["passed"] + ) + ), + "when the package declares a commercial acceptance manifest, the manifest must exist, be accepted, and match declared artifact hashes", + { + "evaluated": bool(commercial_acceptance_summary), + "required": commercial_acceptance_required, + "require_commercial_acceptance": bool(require_commercial_acceptance), + "declaration": commercial_acceptance_declaration, + "schema": commercial_acceptance_schema, + "summary": commercial_acceptance_summary, + "artifact_integrity": commercial_acceptance_integrity, + }, + ), + strict_gate( + "render_nonblank_and_balanced", + bool(render_summary["passed"]), + "MuJoCo/Motrix renders are nonblank and visually balanced across requested cameras", + render_summary, + ), + strict_gate( + "source_view_fidelity", + isinstance(source_to_preview_mae, (int, float)) and float(source_to_preview_mae) <= source_mae_target, + "source image and reconstructed source-view preview remain close after resize", + {"source_to_preview_resized_mae": source_to_preview_mae, "target_max": source_mae_target, "score": round(source_mae_score, 6)}, + ), + strict_gate( + "observed_room_floor", + bool(room_summary.get("has_observed_floor")), + "commercial room reconstruction should observe a real floor plane rather than only using an inferred simulation floor", + { + "has_observed_floor": room_summary.get("has_observed_floor"), + "has_floor_completion": room_summary.get("has_floor_completion"), + "room_completeness_score": room_summary.get("score"), + }, + ), + strict_gate( + "texture_mesh_coverage", + texture_face_ratio >= 0.98, + "nearly all reconstructed mesh faces have projected/baked texture", + {"textured_face_ratio": round(texture_face_ratio, 6)}, + ), + strict_gate( + "production_movable_assets", + bool(movable_quality["commercial_asset_grade"]), + "movable objects are production-grade reconstructed assets, not provisional extraction-task billboards", + movable_quality, + ), + strict_gate( + "physics_task_readiness", + bool(physics_task.get("passed")), + "dynamic assets have free joints, positive inertial mass, enabled collision geometry, and stable MuJoCo simulation when rendered", + physics_task, + ), + strict_gate( + "robot_task_readiness", + bool(robot_task.get("passed")), + "package exposes robot config, task targets, observation cameras, and render-ready task views", + robot_task, + ), + strict_gate( + "task_execution_probe", + bool(execution_probe.get("passed")), + "task targets expose finite approach waypoints and manipulation-scale collision proxies for execution probes", + execution_probe, + ), + strict_gate( + "task_affordance_targets", + bool((not task_affordance_required and not task_affordance_summary) or task_affordance_summary.get("passed")), + "when task affordance analysis is available, production movable assets must expose finite poses and actionable contact roles", + { + "evaluated": bool(task_affordance_summary), + "required": task_affordance_required, + "summary": task_affordance_summary, + }, + ), + strict_gate( + "capture_space_acceptance", + bool((not capture_acceptance_required and not capture_acceptance["evaluated"]) or capture_acceptance["accepted"]), + "when SfM capture acceptance is available, camera path coverage must support space-level reconstruction", + { + **capture_acceptance, + "required": capture_acceptance_required, + "require_capture_acceptance": bool(require_capture_acceptance), + }, + ), + strict_gate( + "metric_scale_plausibility", + bool((not metric_scale_summary) or metric_scale_summary.get("passed")), + "when metric-scale analysis is available, room bounds and structure proxies must be plausible for robot simulation", + { + "evaluated": bool(metric_scale_summary), + "summary": metric_scale_summary, + }, + ), + strict_gate( + "navigation_acceptance", + bool((not navigation_summary) or navigation_summary.get("passed")), + "when navigation acceptance is available, package must expose a robot-usable walkable surface", + { + "evaluated": bool(navigation_summary), + "summary": navigation_summary, + }, + ), + strict_gate( + "material_readiness", + bool((not material_required and not material_summary) or material_summary.get("passed")), + "when material readiness is available, package must include texture-backed material assets and referenced texture files", + { + "evaluated": bool(material_summary), + "required": material_required, + "summary": material_summary, + }, + ), + ] + blockers = [gate["id"] for gate in gates if not gate["passed"]] + return { + "format": "gs-playground-real2sim-strict-commercial-visual-quality-v1", + "summary": { + "score": score, + "target": 0.85, + "strict_commercial_visual_grade": bool(score >= 0.85 and not blockers), + "blocker_count": len(blockers), + "blockers": blockers, + }, + "gates": gates, + "evidence": { + "commercial_readiness": commercial_summary, + "commercial_acceptance": commercial_acceptance_summary, + "commercial_acceptance_declaration": commercial_acceptance_declaration, + "commercial_acceptance_schema": commercial_acceptance_schema, + "commercial_acceptance_integrity": commercial_acceptance_integrity, + "room_completeness": room_summary, + "metric_scale": metric_scale_summary, + "navigation_acceptance": navigation_summary, + "material_readiness": material_summary, + "task_affordance": task_affordance_summary, + "render_quality": render_summary, + "source_to_preview_resized_mae": source_to_preview_mae, + "texture_textured_face_ratio": round(texture_face_ratio, 6), + "mesh_cleanup_kept_area_ratio": mesh_cleanup.get("kept_area_ratio"), + "movable_asset_quality": movable_quality, + "physics_task_readiness": physics_task, + "robot_task_readiness": robot_task, + "task_execution_probe": execution_probe, + "capture_space_acceptance": capture_acceptance, + }, + "next_actions": strict_visual_next_actions(blockers), + "notes": [ + "This strict report is intentionally harder than package import readiness.", + "It is meant to track the remaining gap to Polycam/RealityCapture-like visual and geometric quality.", + ], + } + + +def material_readiness_required(commercial: dict[str, Any], motrix_import: dict[str, Any]) -> bool: + if isinstance(motrix_import, dict) and motrix_import.get("material_readiness_report"): + return True + gates = commercial.get("gates") if isinstance(commercial, dict) else None + if not isinstance(gates, list): + return False + return any(isinstance(gate, dict) and gate.get("id") == "material_readiness" for gate in gates) + + +def task_affordance_required_for_package(commercial: dict[str, Any], motrix_import: dict[str, Any]) -> bool: + if isinstance(motrix_import, dict) and motrix_import.get("task_affordance_report"): + return True + gates = commercial.get("gates") if isinstance(commercial, dict) else None + if not isinstance(gates, list): + return False + return any(isinstance(gate, dict) and gate.get("id") == "task_affordance_targets" for gate in gates) + + +def commercial_acceptance_required_for_package(commercial: dict[str, Any], motrix_import: dict[str, Any]) -> bool: + if isinstance(motrix_import, dict) and motrix_import.get("commercial_acceptance_report"): + return True + gates = commercial.get("gates") if isinstance(commercial, dict) else None + if not isinstance(gates, list): + return False + return any(isinstance(gate, dict) and gate.get("id") == "commercial_acceptance_manifest" for gate in gates) + + +def commercial_acceptance_declaration_report(motrix_import: dict[str, Any], *, required: bool = False) -> dict[str, Any]: + actual_path = motrix_import.get("commercial_acceptance_report") if isinstance(motrix_import, dict) else None + if not actual_path: + return { + "evaluated": False, + "passed": not required, + "expected_path": COMMERCIAL_ACCEPTANCE_REPORT_PATH, + "actual_path": None, + "issues": [{"reason": "missing_commercial_acceptance_report"}] if required else [], + } + issues = [] + if actual_path != COMMERCIAL_ACCEPTANCE_REPORT_PATH: + issues.append( + { + "reason": "commercial_acceptance_report_path_mismatch", + "expected": COMMERCIAL_ACCEPTANCE_REPORT_PATH, + "actual": actual_path, + } + ) + return { + "evaluated": True, + "passed": not issues, + "expected_path": COMMERCIAL_ACCEPTANCE_REPORT_PATH, + "actual_path": actual_path, + "issues": issues, + } + + +def commercial_acceptance_schema_report(manifest: dict[str, Any], *, required: bool = False) -> dict[str, Any]: + if not isinstance(manifest, dict) or not manifest: + return { + "evaluated": False, + "passed": not required, + "expected_format": COMMERCIAL_ACCEPTANCE_FORMAT, + "actual_format": None, + "issues": [{"reason": "missing_commercial_acceptance_manifest"}] if required else [], + } + actual_format = manifest.get("format") + issues = [] + if actual_format != COMMERCIAL_ACCEPTANCE_FORMAT: + issues.append( + { + "reason": "unsupported_commercial_acceptance_format", + "expected": COMMERCIAL_ACCEPTANCE_FORMAT, + "actual": actual_format, + } + ) + if not isinstance(manifest.get("summary"), dict): + issues.append({"reason": "missing_commercial_acceptance_summary"}) + return { + "evaluated": True, + "passed": not issues, + "expected_format": COMMERCIAL_ACCEPTANCE_FORMAT, + "actual_format": actual_format, + "issues": issues, + } + + +def commercial_acceptance_integrity_report(root: Path, manifest: dict[str, Any], *, required: bool = False) -> dict[str, Any]: + artifact_integrity = manifest.get("artifact_integrity") if isinstance(manifest, dict) else None + if not isinstance(artifact_integrity, dict): + return { + "evaluated": False, + "passed": not required, + "checked_count": 0, + "issues": [{"reason": "missing_artifact_integrity"}] if required else [], + } + issues: list[dict[str, Any]] = [] + checked_count = 0 + groups: dict[str, Any] = {} + required_groups: dict[str, dict[str, str]] = { + "entrypoints": REQUIRED_COMMERCIAL_ACCEPTANCE_ENTRYPOINTS, + "reports": REQUIRED_COMMERCIAL_ACCEPTANCE_REPORTS, + } + for group_name in ("entrypoints", "reports"): + group = artifact_integrity.get(group_name) + if not isinstance(group, dict): + if required: + issues.append({"group": group_name, "reason": "missing_integrity_group"}) + continue + group_result: dict[str, Any] = {} + if required: + for artifact_id in sorted(set(required_groups[group_name]) - set(str(key) for key in group)): + issue = {"group": group_name, "artifact": artifact_id, "reason": "missing_integrity_entry"} + group_result[artifact_id] = {"passed": False, "reason": "missing_integrity_entry"} + issues.append(issue) + for artifact_id, declared in group.items(): + checked_count += 1 + expected_path = required_groups[group_name].get(str(artifact_id)) if required else None + result = validate_declared_artifact(root, declared, expected_path=expected_path) + group_result[str(artifact_id)] = result + if not result["passed"]: + issue = {"group": group_name, "artifact": str(artifact_id)} + issue.update(result) + issues.append(issue) + groups[group_name] = group_result + if checked_count == 0: + issues.append({"reason": "empty_artifact_integrity"}) + return { + "evaluated": True, + "passed": not issues, + "checked_count": checked_count, + "issues": issues, + "groups": groups, + } + + +def validate_declared_artifact(root: Path, declared: Any, *, expected_path: str | None = None) -> dict[str, Any]: + if not isinstance(declared, dict): + return {"passed": False, "reason": "invalid_manifest_entry"} + package_path = declared.get("path") + if not isinstance(package_path, str) or not package_path: + return {"passed": False, "reason": "missing_path"} + if Path(package_path).is_absolute() or ".." in Path(package_path).parts: + return {"path": package_path, "passed": False, "reason": "unsafe_package_path"} + path = root / package_path + if not path.is_file(): + return {"path": package_path, "passed": False, "reason": "missing_file"} + actual_size = path.stat().st_size + actual_sha256 = sha256_file(path) if actual_size > 0 else None + expected_size = declared.get("size") + expected_sha256 = declared.get("sha256") + issues = [] + if expected_path is not None and package_path != expected_path: + issues.append("path_mismatch") + if not isinstance(expected_size, int) or expected_size != actual_size: + issues.append("size_mismatch") + if not isinstance(expected_sha256, str) or expected_sha256 != actual_sha256: + issues.append("sha256_mismatch") + return { + "path": package_path, + "passed": not issues, + "issues": issues, + "expected_path": expected_path, + "expected_size": expected_size, + "actual_size": actual_size, + "expected_sha256": expected_sha256, + "actual_sha256": actual_sha256, + } + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_visual_acceptance_detail(visual_acceptance: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(visual_acceptance, dict): + return {} + value = visual_acceptance.get("json") + if not value: + return {"source_to_preview_resized_mae": visual_acceptance.get("resized_mae")} + path = Path(str(value)) + if not path.is_file(): + return {"source_to_preview_resized_mae": visual_acceptance.get("resized_mae")} + try: + return read_json(path) + except Exception: + return {"source_to_preview_resized_mae": visual_acceptance.get("resized_mae")} + + +def capture_space_acceptance_summary(root: Path) -> dict[str, Any]: + report = read_optional_json(root / "artifacts" / "capture_acceptance_report.json") + if not isinstance(report, dict) or not report: + return { + "evaluated": False, + "accepted": None, + "decision": "not_evaluated", + "blockers": [], + "metrics": {}, + } + metrics = report.get("metrics") if isinstance(report.get("metrics"), dict) else {} + metric_keys = [ + "frame_count", + "registered_image_count", + "registered_ratio", + "sparse_point_count", + "verified_pair_count", + "camera_path_length", + "camera_planar_coverage_ratio", + "camera_max_step_ratio", + "camera_loop_closure_ratio", + ] + return { + "evaluated": True, + "accepted": bool(report.get("accepted")), + "decision": report.get("decision"), + "blockers": report.get("blockers", []), + "metrics": {key: metrics.get(key) for key in metric_keys if key in metrics}, + } + + +def strict_render_quality_summary(verification_report: dict[str, Any]) -> dict[str, Any]: + offscreen = verification_report.get("mujoco_offscreen_render") + renders = offscreen.get("renders") if isinstance(offscreen, dict) else [] + renders = renders if isinstance(renders, list) else [] + scores: list[float] = [] + nonblank = True + max_black = 1.0 + max_saturation = 1.0 + for render in renders: + if not isinstance(render, dict): + continue + nonblank = nonblank and bool(render.get("nonblank")) + quality = render.get("quality") if isinstance(render.get("quality"), dict) else {} + scores.append(float(quality.get("score") or 0.0)) + max_black = min(max_black, float(quality.get("black_ratio") or 0.0)) if max_black == 1.0 else max(max_black, float(quality.get("black_ratio") or 0.0)) + max_saturation = min(max_saturation, float(quality.get("high_saturation_ratio") or 0.0)) if max_saturation == 1.0 else max(max_saturation, float(quality.get("high_saturation_ratio") or 0.0)) + average = sum(scores) / len(scores) if scores else 0.0 + minimum = min(scores) if scores else 0.0 + if not renders: + max_black = 1.0 + max_saturation = 1.0 + return { + "render_count": len(renders), + "all_nonblank": bool(nonblank and renders), + "average_score": round(float(average), 6), + "min_score": round(float(minimum), 6), + "max_black_ratio": round(float(max_black), 6), + "max_high_saturation_ratio": round(float(max_saturation), 6), + "passed": bool(renders and nonblank and average >= 0.78 and minimum >= 0.70 and max_black <= 0.16), + } + + +def texture_textured_face_ratio(texture: dict[str, Any]) -> float: + value = texture.get("textured_face_ratio") if isinstance(texture, dict) else None + if isinstance(value, (int, float)): + return float(value) + if not isinstance(texture, dict): + return 0.0 + drawn = int(texture.get("drawn_face_count") or 0) + selected = int(texture.get("selected_face_count") or texture.get("face_count") or 0) + return drawn / float(max(selected, 1)) + + +def movable_asset_quality_summary(movable: dict[str, Any]) -> dict[str, Any]: + quality = movable.get("quality") if isinstance(movable.get("quality"), dict) else {} + assets = movable.get("movable_assets") if isinstance(movable.get("movable_assets"), list) else [] + asset_count = int(quality.get("movable_asset_count") or len(assets)) + production_count = int(quality.get("production_asset_count") or 0) + provisional_count = int(quality.get("provisional_asset_count") or 0) + incomplete_count = int(quality.get("incomplete_asset_count") or 0) + if assets: + asset_count = len(assets) + production_count = 0 + provisional_count = 0 + incomplete_count = 0 + for asset in assets: + if not isinstance(asset, dict): + incomplete_count += 1 + continue + rq = asset.get("reconstruction_quality") if isinstance(asset.get("reconstruction_quality"), dict) else {} + completeness_blockers = rq.get("asset_completeness_blockers") if isinstance(rq.get("asset_completeness_blockers"), list) else [] + if completeness_blockers or not rq.get("has_gaussian_asset") or not rq.get("has_mesh_asset") or not rq.get("has_pose"): + incomplete_count += 1 + if rq.get("commercial_asset_grade") and not completeness_blockers: + production_count += 1 + elif str(asset.get("source_instance_id", "")).startswith("candidate_slot_"): + provisional_count += 1 + if asset_count and production_count == 0 and provisional_count == 0: + for asset in assets: + rq = asset.get("reconstruction_quality") if isinstance(asset, dict) and isinstance(asset.get("reconstruction_quality"), dict) else {} + if rq.get("commercial_asset_grade"): + production_count += 1 + elif str(asset.get("source_instance_id", "")).startswith("candidate_slot_") if isinstance(asset, dict) else False: + provisional_count += 1 + commercial_asset_grade = bool(asset_count > 0 and production_count == asset_count and provisional_count == 0 and incomplete_count == 0) + return { + "movable_asset_count": asset_count, + "production_asset_count": production_count, + "provisional_asset_count": provisional_count, + "incomplete_asset_count": incomplete_count, + "commercial_asset_grade": commercial_asset_grade, + "required_upgrade": None + if commercial_asset_grade + else "replace provisional movable assets with SAM/SAM3D or multi-view object reconstructions and ensure gaussian, mesh, and pose assets are present", + } + + +def source_fidelity_score(value: Any, *, target: float) -> float: + if not isinstance(value, (int, float)): + return 0.0 + mae = max(float(value), 0.0) + target = max(float(target), 1e-6) + if mae <= target: + return max(0.0, min(1.0, 0.75 + 0.25 * (1.0 - mae / target))) + return max(0.0, min(0.75, 0.75 * (1.0 - (mae - target) / max(target * 0.75, 1e-6)))) + + +def strict_gate(gate_id: str, passed: bool, requirement: str, evidence: dict[str, Any]) -> dict[str, Any]: + return { + "id": gate_id, + "passed": bool(passed), + "requirement": requirement, + "evidence": evidence, + } + + +def strict_visual_next_actions(blockers: list[str]) -> list[str]: + actions = { + "engineering_import_readiness": "Fix core package readiness before evaluating visual fidelity.", + "commercial_acceptance_manifest": "Regenerate or attach artifacts/quality/commercial_acceptance.json and resolve every acceptance blocker before final handoff.", + "render_nonblank_and_balanced": "Tune camera framing, scene completion, material colors, and render exposure until all verifier cameras are balanced.", + "source_view_fidelity": "Improve texture projection, source-camera alignment, and mesh/3DGS registration to lower source-view MAE.", + "observed_room_floor": "Capture more floor and room-corner views or reconstruct a real floor plane instead of relying on inferred floor completion.", + "texture_mesh_coverage": "Increase projected/baked face coverage and inspect untextured regions before final export.", + "production_movable_assets": "Replace provisional movable assets with SAM/SAM3D or multi-view object reconstructions before commercial export.", + "physics_task_readiness": "Add task-ready collision proxies, inertial mass, free joints, and a stable MuJoCo simulation step check for every movable asset.", + "robot_task_readiness": "Add robot config, dynamic task targets, standard observation cameras, and render-ready task views before task-level acceptance.", + "task_execution_probe": "Add finite approach waypoints and manipulation-scale collision proxies for every dynamic task target.", + "task_affordance_targets": "Assign dynamic task targets finite poses and actionable graspable or pushable contact roles before commercial export.", + "capture_space_acceptance": "Recapture with a continuous room-scale camera path, stronger lateral parallax, and no discontinuous jumps before claiming space-level commercial reconstruction.", + "metric_scale_plausibility": "Calibrate metric scale from COLMAP poses, known robot/object dimensions, or manual room measurements before claiming robot-ready commercial reconstruction.", + "navigation_acceptance": "Generate and validate an observed walkable floor/nav surface before claiming robot-navigation-ready commercial reconstruction.", + "material_readiness": "Export texture-backed mesh/MJCF materials and ensure every referenced texture file is included in the simulator package.", + } + return [actions[item] for item in blockers if item in actions] + + +def fit_image(image: Any, size: tuple[int, int]) -> Any: + fitted = image.copy() + fitted.thumbnail(size) + return fitted + + +def select_visual_source_image(root: Path, *, reference_image: Path) -> tuple[Path, dict[str, Any]]: + candidates = visual_source_candidates(root) + if not candidates: + return require_file(root / "artifacts" / "background_image.jpg", "source/background image"), { + "strategy": "fallback_background_image", + "candidate_count": 0, + } + explicit = [item for item in candidates if item["priority"] <= 1] + if explicit: + selected = require_file(explicit[0]["path"], "source/background image") + return selected, { + "strategy": "explicit_camera_artifact", + "selected": selected.as_posix(), + "candidate_count": len(candidates), + "selected_source": explicit[0]["source"], + } + scored: list[dict[str, Any]] = [] + for item in candidates[:512]: + path = item["path"] + try: + score = image_resized_mae(path, reference_image, size=(160, 120)) + except Exception: + continue + scored.append({**item, "resized_mae_to_preview": round(float(score), 6)}) + if not scored: + selected = require_file(candidates[0]["path"], "source/background image") + return selected, { + "strategy": "first_readable_candidate", + "selected": selected.as_posix(), + "candidate_count": len(candidates), + } + scored.sort(key=lambda item: float(item["resized_mae_to_preview"])) + selected_item = scored[0] + selected = require_file(selected_item["path"], "source/background image") + return selected, { + "strategy": "best_resized_mae_to_source_gs_preview", + "selected": selected.as_posix(), + "candidate_count": len(candidates), + "scored_candidate_count": len(scored), + "selected_source": selected_item["source"], + "selected_resized_mae_to_preview": selected_item["resized_mae_to_preview"], + "top_candidates": [ + { + "path": item["path"].as_posix(), + "source": item["source"], + "resized_mae_to_preview": item["resized_mae_to_preview"], + } + for item in scored[:5] + ], + } + + +def visual_source_candidates(root: Path) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [] + motrix_import_path = root / "motrix_import.json" + if motrix_import_path.is_file(): + motrix_import = read_json(motrix_import_path) + camera_artifacts = motrix_import.get("camera_artifacts") + if isinstance(camera_artifacts, dict): + for key in ("background_image", "source_image"): + value = camera_artifacts.get(key) + if value: + path = root / str(value) + if path.is_file(): + candidates.append({"path": path, "source": f"camera_artifacts.{key}", "priority": 0}) + for pattern in ("background_image.*", "source_image.*"): + matches = sorted((root / "artifacts").glob(pattern)) + for match in matches: + candidates.append({"path": match, "source": f"artifacts/{pattern}", "priority": 1}) + manifest_path = root / "package_manifest.json" + if manifest_path.is_file(): + manifest = read_json(manifest_path) + source = manifest.get("source") + if source: + source_path = Path(str(source)) + if source_path.is_dir(): + for pattern in ("*.jpg", "*.jpeg", "*.png", "*.JPG", "*.JPEG", "*.PNG"): + matches = sorted(source_path.glob(pattern)) + for match in matches: + candidates.append({"path": match, "source": "package_manifest.source", "priority": 2}) + return dedupe_visual_candidates(candidates) + + +def dedupe_visual_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + seen: set[str] = set() + deduped: list[dict[str, Any]] = [] + for item in sorted(candidates, key=lambda value: (int(value["priority"]), value["path"].as_posix())): + key = item["path"].resolve().as_posix() + if key in seen: + continue + seen.add(key) + deduped.append(item) + return deduped + + +def image_resized_mae(a: Path, b: Path, *, size: tuple[int, int]) -> float: + from PIL import Image + import numpy as np + + left = Image.open(a).convert("RGB").resize(size) + right = Image.open(b).convert("RGB").resize(size) + diff = np.asarray(left, dtype=np.float32) / 255.0 - np.asarray(right, dtype=np.float32) / 255.0 + return float(np.mean(np.abs(diff))) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_video_motrix_scene.py b/vendor/gs_playground/scripts/real2sim_video_motrix_scene.py new file mode 100644 index 0000000..2b4513a --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_video_motrix_scene.py @@ -0,0 +1,1902 @@ +#!/usr/bin/env python3 +"""Video Real2Sim scene reconstruction workflow for MotrixSim. + +This script upgrades short video captures from the built-in pseudo-depth path +to the AnySplat-direct path. It extracts video frames, selects sharp and +temporally spread keyframes, reconstructs a scene-level 3DGS, packages it for +MotrixSim, and writes verification/visual acceptance artifacts. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np +from PIL import Image, ImageOps + +PIL_BILINEAR = getattr(getattr(Image, "Resampling", Image), "BILINEAR") + +REPO_ROOT = Path(__file__).resolve().parents[1] +if REPO_ROOT.as_posix() not in sys.path: + sys.path.insert(0, REPO_ROOT.as_posix()) + +VIDEO_PROMPT = "office aisle scene with desks, cabinets, floor, plants and fixtures" +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"} +DEFAULT_VIDEO_ANYSPLAT_MAX_VIEWS = 12 +DEFAULT_VIDEO_MIN_SCENE_VERTICES = 600_000 +MOTRIX_HIGH_FRAME_FPS = 3.0 +MOTRIX_HIGH_MESH_GRID = 160 +MOTRIX_HIGH_MIN_SCENE_VERTICES = 900_000 +MOTRIX_HIGH_SCENE_SURFELS = 30_000 +MOTRIX_HIGH_POSE_INIT_STEPS = 320 +MOTRIX_HIGH_POSE_INIT_MAX_FRAMES = 10 +MOTRIX_HIGH_POSE_INIT_IMAGE_MAX_SIZE = 448 +MOTRIX_HIGH_POSE_INIT_TARGET_GAUSSIANS = 60_000 +MOTRIX_HIGH_POSE_INIT_MAX_POINTS = 90_000 +MOTRIX_HIGH_POSE_INIT_DENSE_POINTS_PER_FRAME = 2_600 +MOTRIX_HIGH_POSE_INIT_ADAPTIVE_EVERY = 25 +MOTRIX_HIGH_POSE_INIT_ADAPTIVE_STOP = 125 +MOTRIX_HIGH_POSE_INIT_ADAPTIVE_POINTS = 2_500 +MOTRIX_HIGH_POSE_INIT_ADAPTIVE_ERROR_PERCENTILE = 88.0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate and verify a MotrixSim scene package from a video") + parser.add_argument("--video", type=Path, default=None, help="Source video") + parser.add_argument("--frames-dir", type=Path, default=None, help="Use pre-extracted video frames or a still-image capture directory") + parser.add_argument("--images-dir", type=Path, default=None, help="Alias for --frames-dir when the input is a still-image capture set") + parser.add_argument("--output", type=Path, required=True, help="Output Real2Sim scene package directory") + parser.add_argument("--work-dir", type=Path, required=True, help="Intermediate workflow directory") + parser.add_argument("--prompt", default=VIDEO_PROMPT) + parser.add_argument("--scene-prompt", default="", help="Scene reconstruction prompt; defaults to --prompt") + parser.add_argument( + "--quality-preset", + default=os.environ.get("REAL2SIM_VIDEO_QUALITY_PRESET", "balanced"), + choices=["balanced", "motrix-high", "capture-high"], + help="Balanced keeps default package size; capture-high also runs COLMAP/SfM capture diagnostics", + ) + parser.add_argument("--frame-fps", type=float, default=float(os.environ.get("REAL2SIM_VIDEO_FRAME_FPS", "2.0"))) + parser.add_argument("--anysplat-python", default=os.environ.get("REAL2SIM_ANY_SPLAT_PYTHON", "")) + parser.add_argument("--anysplat-root", type=Path, default=os.environ.get("REAL2SIM_ANYSPLAT_ROOT")) + parser.add_argument( + "--anysplat-max-views", + type=int, + default=int(os.environ.get("REAL2SIM_VIDEO_ANYSPLAT_MAX_VIEWS", str(DEFAULT_VIDEO_ANYSPLAT_MAX_VIEWS))), + ) + parser.add_argument("--anysplat-mesh-grid", type=int, default=int(os.environ.get("REAL2SIM_VIDEO_ANYSPLAT_MESH_GRID", "96"))) + parser.add_argument( + "--anysplat-keyframe-strategy", + choices=["quality-diversity", "sfm-registered"], + default=os.environ.get("REAL2SIM_ANYSPLAT_KEYFRAME_STRATEGY"), + help="How to select frames passed to AnySplat; capture-high defaults to SfM-registered pose bins", + ) + parser.add_argument( + "--anysplat-keyframe-orientation-weight", + type=float, + default=float(os.environ["REAL2SIM_ANYSPLAT_KEYFRAME_ORIENTATION_WEIGHT"]) + if os.environ.get("REAL2SIM_ANYSPLAT_KEYFRAME_ORIENTATION_WEIGHT") + else None, + help="SfM-registered keyframe score weight for camera up-vector consistency", + ) + parser.add_argument("--min-video-quality", type=float, default=float(os.environ.get("REAL2SIM_MIN_VIDEO_FRAME_QUALITY", "0.38"))) + parser.add_argument("--allow-low-quality", action="store_true") + parser.add_argument("--min-source-preview-psnr", type=float, default=float(os.environ.get("REAL2SIM_MIN_SOURCE_PREVIEW_PSNR", "18.0"))) + parser.add_argument("--max-source-preview-mae", type=float, default=float(os.environ.get("REAL2SIM_MAX_SOURCE_PREVIEW_MAE", "0.10"))) + parser.add_argument( + "--min-scene-vertices", + type=int, + default=int(os.environ.get("REAL2SIM_VIDEO_MIN_SCENE_VERTICES", str(DEFAULT_VIDEO_MIN_SCENE_VERTICES))), + ) + parser.add_argument("--skip-final-package-verify", action="store_true") + parser.add_argument("--sfm-report", action="store_true", default=_env_bool("REAL2SIM_VIDEO_SFM_REPORT")) + parser.add_argument("--colmap-bin", default=os.environ.get("REAL2SIM_COLMAP_BIN", "colmap")) + parser.add_argument("--sfm-matcher", choices=["exhaustive", "sequential"], default=os.environ.get("REAL2SIM_COLMAP_MATCHER", "sequential")) + parser.add_argument("--sfm-sequential-overlap", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_SEQUENTIAL_OVERLAP", "10"))) + parser.add_argument("--sfm-camera-model", default=os.environ.get("REAL2SIM_COLMAP_CAMERA_MODEL")) + parser.add_argument("--sfm-max-image-size", type=int, default=_env_int("REAL2SIM_COLMAP_MAX_IMAGE_SIZE")) + parser.add_argument("--sfm-sift-max-num-features", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_SIFT_MAX_FEATURES", "8192"))) + parser.add_argument("--sfm-sift-peak-threshold", type=float, default=float(os.environ["REAL2SIM_COLMAP_SIFT_PEAK_THRESHOLD"]) if os.environ.get("REAL2SIM_COLMAP_SIFT_PEAK_THRESHOLD") else None) + parser.add_argument("--sfm-feature-threads", type=int, default=_env_int("REAL2SIM_COLMAP_FEATURE_THREADS")) + parser.add_argument("--sfm-matcher-threads", type=int, default=_env_int("REAL2SIM_COLMAP_MATCHER_THREADS")) + parser.add_argument("--sfm-guided-matching", action="store_true", default=_env_bool("REAL2SIM_COLMAP_GUIDED_MATCHING")) + parser.add_argument("--sfm-frames", choices=["extracted", "selected"], default=os.environ.get("REAL2SIM_SFM_FRAMES", "extracted")) + parser.add_argument("--min-sfm-registered-ratio", type=float, default=float(os.environ.get("REAL2SIM_MIN_SFM_REGISTERED_RATIO", "0.70"))) + parser.add_argument("--min-sfm-points", type=int, default=int(os.environ.get("REAL2SIM_MIN_SFM_POINTS", "500"))) + parser.add_argument("--require-sfm", action="store_true", default=_env_bool("REAL2SIM_REQUIRE_SFM")) + parser.add_argument("--colmap-dense", action="store_true", default=_env_bool("REAL2SIM_COLMAP_DENSE")) + parser.add_argument("--require-colmap-dense", action="store_true", default=_env_bool("REAL2SIM_REQUIRE_COLMAP_DENSE")) + parser.add_argument("--colmap-dense-bin", default=os.environ.get("REAL2SIM_COLMAP_DENSE_BIN")) + parser.add_argument("--colmap-dense-max-image-size", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_DENSE_MAX_IMAGE_SIZE", "960"))) + parser.add_argument("--colmap-dense-min-fused-points", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_DENSE_MIN_FUSED_POINTS", "100000"))) + parser.add_argument("--colmap-dense-gpu-index", default=os.environ.get("REAL2SIM_COLMAP_DENSE_GPU_INDEX", "")) + parser.add_argument("--colmap-dense-run-poisson", action="store_true", default=_env_bool("REAL2SIM_COLMAP_DENSE_POISSON")) + parser.add_argument("--colmap-dense-poisson-depth", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_DENSE_POISSON_DEPTH", "10"))) + parser.add_argument("--colmap-dense-cleanup-keep-largest", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_DENSE_CLEANUP_KEEP_LARGEST", "1"))) + parser.add_argument("--colmap-dense-cleanup-min-faces", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_DENSE_CLEANUP_MIN_FACES", "0"))) + parser.add_argument("--colmap-dense-cleanup-min-area-ratio", type=float, default=float(os.environ.get("REAL2SIM_COLMAP_DENSE_CLEANUP_MIN_AREA_RATIO", "0"))) + parser.add_argument("--colmap-dense-cleanup-max-face-area-percentile", type=float, default=float(os.environ.get("REAL2SIM_COLMAP_DENSE_CLEANUP_MAX_FACE_AREA_PERCENTILE", "100"))) + parser.add_argument("--colmap-dense-cleanup-max-edge-length-percentile", type=float, default=float(os.environ.get("REAL2SIM_COLMAP_DENSE_CLEANUP_MAX_EDGE_LENGTH_PERCENTILE", "100"))) + parser.add_argument("--colmap-dense-cleanup-max-centroid-z-percentile", type=float, default=float(os.environ.get("REAL2SIM_COLMAP_DENSE_CLEANUP_MAX_CENTROID_Z_PERCENTILE", "100"))) + parser.add_argument( + "--colmap-dense-replace-scene-mesh", + action="store_true", + default=_env_bool("REAL2SIM_COLMAP_DENSE_REPLACE_SCENE_MESH"), + help="Repackage the Motrix scene with the COLMAP dense mesh as the scene mesh", + ) + parser.add_argument("--colmap-texture-bake", action="store_true", default=_env_bool("REAL2SIM_COLMAP_TEXTURE_BAKE")) + parser.add_argument("--require-colmap-texture-bake", action="store_true", default=_env_bool("REAL2SIM_REQUIRE_COLMAP_TEXTURE_BAKE")) + parser.add_argument("--colmap-texture-bake-unwrap-uv", action="store_true", default=_env_bool("REAL2SIM_COLMAP_TEXTURE_BAKE_UNWRAP_UV")) + parser.add_argument("--require-colmap-texture-bake-unwrap-uv", action="store_true", default=_env_bool("REAL2SIM_REQUIRE_COLMAP_TEXTURE_BAKE_UNWRAP_UV")) + parser.add_argument("--colmap-texture-bake-atlas-size", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_ATLAS_SIZE", "4096"))) + parser.add_argument("--colmap-texture-bake-max-cameras", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_MAX_CAMERAS", "72"))) + parser.add_argument("--colmap-texture-bake-max-faces", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_MAX_FACES", "0"))) + parser.add_argument("--colmap-texture-bake-max-output-faces", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_MAX_OUTPUT_FACES", "0"))) + parser.add_argument("--colmap-texture-bake-fill-mode", choices=["face", "warp"], default=os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_FILL_MODE", "face")) + parser.add_argument("--colmap-texture-bake-color-blend-cameras", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_COLOR_BLEND_CAMERAS", "1"))) + parser.add_argument("--colmap-texture-bake-visibility-depth-tolerance", type=float, default=float(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_VISIBILITY_DEPTH_TOLERANCE", "0"))) + parser.add_argument("--colmap-texture-bake-visibility-grid-size", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_VISIBILITY_GRID_SIZE", "320"))) + parser.add_argument("--colmap-texture-bake-max-warped-faces", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_MAX_WARPED_FACES", "0"))) + parser.add_argument("--colmap-texture-bake-postprocess-median-radius", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_POSTPROCESS_MEDIAN_RADIUS", "0"))) + parser.add_argument("--colmap-texture-bake-bleed-iterations", type=int, default=int(os.environ.get("REAL2SIM_COLMAP_TEXTURE_BAKE_BLEED_ITERATIONS", "0"))) + parser.add_argument("--motrix-preview-mesh-faces", type=int, default=int(os.environ.get("REAL2SIM_MOTRIX_PREVIEW_MESH_FACES", "0"))) + parser.add_argument("--motrix-preview-mode", choices=["proxy", "lod"], default=os.environ.get("REAL2SIM_MOTRIX_PREVIEW_MODE")) + parser.add_argument("--capture-acceptance", action="store_true", default=_env_bool("REAL2SIM_CAPTURE_ACCEPTANCE")) + parser.add_argument("--require-capture-acceptance", action="store_true", default=_env_bool("REAL2SIM_REQUIRE_CAPTURE_ACCEPTANCE")) + parser.add_argument( + "--min-capture-camera-path-length", + type=float, + default=float(os.environ.get("REAL2SIM_MIN_CAPTURE_CAMERA_PATH_LENGTH", "1.0")), + ) + parser.add_argument( + "--min-capture-camera-planar-coverage-ratio", + type=float, + default=float(os.environ.get("REAL2SIM_MIN_CAPTURE_CAMERA_PLANAR_COVERAGE_RATIO", "0.12")), + ) + parser.add_argument( + "--max-capture-camera-step-ratio", + type=float, + default=float(os.environ.get("REAL2SIM_MAX_CAPTURE_CAMERA_STEP_RATIO", "0.55")), + ) + parser.add_argument( + "--skip-pose-init-dataset", + action="store_true", + help="Do not copy registered COLMAP frames/poses into artifacts/pose_init_dataset", + ) + parser.add_argument("--pose-init-train", action="store_true", default=_env_bool("REAL2SIM_POSE_INIT_TRAIN")) + parser.add_argument( + "--pose-init-train-python", + default=os.environ.get("REAL2SIM_POSE_INIT_TRAIN_PYTHON", os.environ.get("REAL2SIM_ANY_SPLAT_PYTHON", "")), + help="Python interpreter with torch and gsplat for pose-init training", + ) + parser.add_argument("--pose-init-train-steps", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_STEPS", "200"))) + parser.add_argument("--pose-init-train-max-frames", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_MAX_FRAMES", "16"))) + parser.add_argument("--pose-init-train-image-max-size", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_IMAGE_MAX_SIZE", "448"))) + parser.add_argument("--pose-init-train-target-gaussians", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_TARGET_GAUSSIANS", "50000"))) + parser.add_argument("--pose-init-train-max-points", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_MAX_POINTS", "80000"))) + parser.add_argument( + "--pose-init-train-dense-image-points-per-frame", + type=int, + default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_DENSE_IMAGE_POINTS_PER_FRAME", "1600")), + ) + parser.add_argument("--pose-init-train-dense-depth-neighbors", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_DENSE_DEPTH_NEIGHBORS", "4"))) + parser.add_argument("--pose-init-train-dense-depth-max-distance", type=float, default=float(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_DENSE_DEPTH_MAX_DISTANCE", "48.0"))) + parser.add_argument("--pose-init-train-adaptive-densify-every", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_ADAPTIVE_DENSIFY_EVERY", "0"))) + parser.add_argument("--pose-init-train-adaptive-densify-start", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_ADAPTIVE_DENSIFY_START", "50"))) + parser.add_argument("--pose-init-train-adaptive-densify-stop", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_ADAPTIVE_DENSIFY_STOP", "0"))) + parser.add_argument("--pose-init-train-adaptive-densify-points", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_ADAPTIVE_DENSIFY_POINTS", "2500"))) + parser.add_argument("--pose-init-train-adaptive-densify-error-percentile", type=float, default=float(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_ADAPTIVE_DENSIFY_ERROR_PERCENTILE", "82.0"))) + parser.add_argument("--pose-init-train-preview-frames", type=int, default=int(os.environ.get("REAL2SIM_POSE_INIT_TRAIN_PREVIEW_FRAMES", "4"))) + parser.add_argument( + "--pose-init-train-replace-scene", + action="store_true", + default=_env_bool("REAL2SIM_POSE_INIT_TRAIN_REPLACE_SCENE"), + help="Replace 3dgs/scene.ply with the trained pose-init scene after training", + ) + parser.add_argument("--archive", type=Path, default=None, help="Optional .tar.gz path; defaults to .tar.gz") + parser.add_argument("--visual-report-dir", type=Path, default=None) + parser.add_argument("--motrix-viewer-render", type=Path, default=None) + args = parser.parse_args() + if args.images_dir is not None: + if args.frames_dir is not None and args.frames_dir != args.images_dir: + raise SystemExit("--images-dir and --frames-dir point to different directories") + args.frames_dir = args.images_dir + apply_quality_preset(args) + + result = run_video_workflow(args) + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 + + +def run_video_workflow(args: argparse.Namespace) -> dict[str, Any]: + if args.frames_dir is None and args.video is None: + raise SystemExit("either --video or --frames-dir/--images-dir is required") + if args.frames_dir is None and not args.video.is_file(): + raise SystemExit(f"source video not found: {args.video}") + if args.anysplat_max_views <= 0: + raise SystemExit("--anysplat-max-views must be positive") + if args.frame_fps <= 0: + raise SystemExit("--frame-fps must be positive") + + args.work_dir.mkdir(parents=True, exist_ok=True) + if args.frames_dir is not None: + extracted_frames = normalize_existing_frames( + collect_existing_frames(args.frames_dir), + args.work_dir / "input_frames", + ) + else: + extracted_frames = extract_video_frames(args.video, args.work_dir / "video_frames", fps=args.frame_fps) + selected_frames = args.work_dir / "selected_frames" + video_quality = args.work_dir / "video_frame_quality.json" + selected = select_video_keyframes( + extracted_frames, + selected_frames, + max_views=args.anysplat_max_views, + quality_report=video_quality, + min_quality_score=args.min_video_quality, + ) + annotate_video_quality_report(video_quality, args) + quality = json.loads(video_quality.read_text(encoding="utf-8")) + if not quality["passed"] and not args.allow_low_quality: + raise SystemExit( + "video frame quality gate failed: " + f"score {float(quality['score']):.3f} < {args.min_video_quality:.3f}; " + "inspect video_frame_quality.json or rerun with --allow-low-quality" + ) + sfm_report = run_sfm_capture_report(args, sfm_frames_dir(args, extracted_frames, selected_frames)) if args.sfm_report else None + acceptance_report = run_capture_acceptance_report(args, sfm_report) if args.capture_acceptance and sfm_report is not None else None + selected = maybe_reselect_anysplat_keyframes_from_sfm( + args, + extracted_frames, + selected_frames, + video_quality, + sfm_report, + selected, + ) + + helpers = load_one_image_helpers() + helpers.run = run_child_command + source_image = selected[0] + helper_args = SimpleNamespace( + image=source_image, + output=args.output, + work_dir=args.work_dir, + prompt=args.prompt, + scene_prompt=args.scene_prompt or args.prompt, + anysplat_python=args.anysplat_python, + anysplat_root=args.anysplat_root, + anysplat_max_views=args.anysplat_max_views, + anysplat_mesh_grid=args.anysplat_mesh_grid, + source_gs_preview=None, + min_source_preview_psnr=args.min_source_preview_psnr, + max_source_preview_mae=args.max_source_preview_mae, + allow_low_quality=args.allow_low_quality, + min_scene_vertices=args.min_scene_vertices, + ) + anysplat_metadata = helpers.run_anysplat_direct_generation(helper_args, selected_frames) + package_report = helpers.verify_motrix_package(args.output) + projection_preview = helpers.write_projection_preview(args.output / "3dgs" / "scene.ply", args.output / "pointcloud_projection_preview.png") + if anysplat_metadata is not None: + helpers.attach_anysplat_metadata_to_package(args.output, anysplat_metadata) + attach_video_quality_to_package(args.output, video_quality) + if sfm_report is not None: + attach_sfm_report_to_package(args.output, sfm_report) + if acceptance_report is not None: + attach_capture_acceptance_to_package(args.output, acceptance_report) + colmap_dense_report = ( + run_colmap_dense_reconstruction_to_package(args, args.output, sfm_report, helpers) + if should_run_colmap_dense(args, sfm_report) + else None + ) + if colmap_dense_report is not None and bool(args.colmap_dense_replace_scene_mesh): + if anysplat_metadata is not None: + helpers.attach_anysplat_metadata_to_package(args.output, anysplat_metadata) + attach_video_quality_to_package(args.output, video_quality) + if sfm_report is not None: + attach_sfm_report_to_package(args.output, sfm_report) + if acceptance_report is not None: + attach_capture_acceptance_to_package(args.output, acceptance_report) + attach_colmap_dense_to_package(args.output, colmap_dense_report) + pose_init_dataset = ( + export_pose_init_dataset_to_package(sfm_report, args.output) + if should_export_pose_init_dataset(args, sfm_report) + else None + ) + pose_init_training = run_pose_init_training_to_package(args, args.output, pose_init_dataset) + + archive = helpers.create_archive(args.output, args.archive or args.output.with_suffix(".tar.gz")) + final_verification = None + if not args.skip_final_package_verify: + final_verification = helpers.run_final_package_verification(helper_args, archive) + archive = helpers.create_archive(args.output, archive) + + visual_report = None + visual_dir = args.visual_report_dir or (args.work_dir / "visual_acceptance") + visual_report = write_visual_report( + args.output, + visual_dir, + min_scene_vertices=args.min_scene_vertices, + min_source_preview_psnr=args.min_source_preview_psnr, + max_source_preview_mae=args.max_source_preview_mae, + motrix_viewer_render=args.motrix_viewer_render, + ) + + return { + "input": args.video.as_posix() if args.video is not None else args.frames_dir.as_posix(), + "input_kind": "video" if args.frames_dir is None else "image_directory", + "video": args.video.as_posix() if args.video is not None else None, + "frames_dir": args.frames_dir.as_posix() if args.frames_dir is not None else None, + "quality_preset": args.quality_preset, + "selected_frames": [path.as_posix() for path in selected], + "video_frame_quality": video_quality.as_posix(), + "sfm_capture_report": sfm_report.as_posix() if sfm_report is not None else None, + "capture_acceptance_report": acceptance_report.as_posix() if acceptance_report is not None else None, + "package": package_report, + "projection_preview": projection_preview.as_posix(), + "archive": archive.as_posix(), + "final_package_verification": final_verification.as_posix() if final_verification is not None else None, + "visual_acceptance_report": visual_report, + "colmap_dense_reconstruction": colmap_dense_report.as_posix() if colmap_dense_report is not None else None, + "pose_init_dataset": pose_init_dataset.as_posix() if pose_init_dataset is not None else None, + "pose_init_training": pose_init_training.as_posix() if pose_init_training is not None else None, + } + + +def apply_quality_preset(args: argparse.Namespace) -> None: + apply_quality_preset_arg_defaults(args) + if args.require_colmap_dense: + args.colmap_dense = True + if args.colmap_dense: + args.sfm_report = True + if args.sfm_camera_model is None: + args.sfm_camera_model = "SIMPLE_RADIAL" + if args.sfm_max_image_size is None or int(args.sfm_max_image_size) <= 0: + args.sfm_max_image_size = 1600 + if args.sfm_feature_threads is None or int(args.sfm_feature_threads) <= 0: + args.sfm_feature_threads = min(8, max(2, (os.cpu_count() or 2) // 2)) + if args.sfm_matcher_threads is None or int(args.sfm_matcher_threads) <= 0: + args.sfm_matcher_threads = min(8, max(2, (os.cpu_count() or 2) // 2)) + args.sfm_sequential_overlap = max(int(args.sfm_sequential_overlap), 20) + args.sfm_sift_max_num_features = max(int(args.sfm_sift_max_num_features), 16_384) + if args.sfm_sift_peak_threshold is None: + args.sfm_sift_peak_threshold = 0.003 + args.colmap_dense_max_image_size = max(int(args.colmap_dense_max_image_size), 1) + args.colmap_dense_min_fused_points = max(int(args.colmap_dense_min_fused_points), 1) + if args.quality_preset not in {"motrix-high", "capture-high"}: + return + args.frame_fps = max(float(args.frame_fps), MOTRIX_HIGH_FRAME_FPS) + args.anysplat_max_views = max(int(args.anysplat_max_views), DEFAULT_VIDEO_ANYSPLAT_MAX_VIEWS) + args.anysplat_mesh_grid = max(int(args.anysplat_mesh_grid), MOTRIX_HIGH_MESH_GRID) + args.min_scene_vertices = max(int(args.min_scene_vertices), MOTRIX_HIGH_MIN_SCENE_VERTICES) + os.environ.setdefault("REAL2SIM_MOTRIX_PREVIEW_MODE", "lod") + os.environ.setdefault("REAL2SIM_MOTRIX_SCENE_VISUAL", "hybrid") + os.environ.setdefault("REAL2SIM_MOTRIX_SCENE_SURFELS", str(MOTRIX_HIGH_SCENE_SURFELS)) + args.pose_init_train_steps = max(int(args.pose_init_train_steps), MOTRIX_HIGH_POSE_INIT_STEPS) + args.pose_init_train_max_frames = max(int(args.pose_init_train_max_frames), MOTRIX_HIGH_POSE_INIT_MAX_FRAMES) + args.pose_init_train_image_max_size = max(int(args.pose_init_train_image_max_size), MOTRIX_HIGH_POSE_INIT_IMAGE_MAX_SIZE) + args.pose_init_train_target_gaussians = max(int(args.pose_init_train_target_gaussians), MOTRIX_HIGH_POSE_INIT_TARGET_GAUSSIANS) + args.pose_init_train_max_points = max(int(args.pose_init_train_max_points), MOTRIX_HIGH_POSE_INIT_MAX_POINTS) + args.pose_init_train_dense_image_points_per_frame = max( + int(args.pose_init_train_dense_image_points_per_frame), + MOTRIX_HIGH_POSE_INIT_DENSE_POINTS_PER_FRAME, + ) + args.pose_init_train_adaptive_densify_every = max( + int(args.pose_init_train_adaptive_densify_every), + MOTRIX_HIGH_POSE_INIT_ADAPTIVE_EVERY, + ) + if int(args.pose_init_train_adaptive_densify_stop) <= 0: + args.pose_init_train_adaptive_densify_stop = MOTRIX_HIGH_POSE_INIT_ADAPTIVE_STOP + args.pose_init_train_adaptive_densify_points = max( + int(args.pose_init_train_adaptive_densify_points), + MOTRIX_HIGH_POSE_INIT_ADAPTIVE_POINTS, + ) + args.pose_init_train_adaptive_densify_error_percentile = max( + float(args.pose_init_train_adaptive_densify_error_percentile), + MOTRIX_HIGH_POSE_INIT_ADAPTIVE_ERROR_PERCENTILE, + ) + if args.quality_preset == "capture-high": + args.sfm_report = True + args.capture_acceptance = True + if args.anysplat_keyframe_strategy is None: + args.anysplat_keyframe_strategy = "sfm-registered" + if args.anysplat_keyframe_orientation_weight is None: + args.anysplat_keyframe_orientation_weight = 0.30 + if args.sfm_camera_model is None: + args.sfm_camera_model = "SIMPLE_RADIAL" + if args.sfm_max_image_size is None or int(args.sfm_max_image_size) <= 0: + args.sfm_max_image_size = 1600 + if args.sfm_feature_threads is None or int(args.sfm_feature_threads) <= 0: + args.sfm_feature_threads = min(8, max(2, (os.cpu_count() or 2) // 2)) + if args.sfm_matcher_threads is None or int(args.sfm_matcher_threads) <= 0: + args.sfm_matcher_threads = min(8, max(2, (os.cpu_count() or 2) // 2)) + args.sfm_sequential_overlap = max(int(args.sfm_sequential_overlap), 20) + args.sfm_sift_max_num_features = max(int(args.sfm_sift_max_num_features), 16_384) + if args.sfm_sift_peak_threshold is None: + args.sfm_sift_peak_threshold = 0.003 + if args.anysplat_keyframe_strategy is None: + args.anysplat_keyframe_strategy = "quality-diversity" + if args.anysplat_keyframe_orientation_weight is None: + args.anysplat_keyframe_orientation_weight = 0.0 + + +def apply_quality_preset_arg_defaults(args: argparse.Namespace) -> None: + defaults = { + "require_colmap_dense": False, + "colmap_dense": False, + "sfm_camera_model": None, + "sfm_max_image_size": None, + "sfm_feature_threads": None, + "sfm_matcher_threads": None, + "sfm_sequential_overlap": 10, + "sfm_sift_max_num_features": 8192, + "sfm_sift_peak_threshold": None, + "colmap_dense_max_image_size": 960, + "colmap_dense_min_fused_points": 100000, + "pose_init_train_steps": 200, + "pose_init_train_max_frames": 16, + "pose_init_train_image_max_size": 448, + "pose_init_train_target_gaussians": 50000, + "pose_init_train_max_points": 80000, + "pose_init_train_dense_image_points_per_frame": 1600, + "pose_init_train_adaptive_densify_every": 0, + "pose_init_train_adaptive_densify_stop": 0, + "pose_init_train_adaptive_densify_points": 2500, + "pose_init_train_adaptive_densify_error_percentile": 82.0, + "anysplat_keyframe_strategy": None, + "anysplat_keyframe_orientation_weight": None, + } + for name, value in defaults.items(): + if not hasattr(args, name): + setattr(args, name, value) + + +def annotate_video_quality_report(path: Path, args: argparse.Namespace) -> None: + report = json.loads(path.read_text(encoding="utf-8")) + report["quality_preset"] = args.quality_preset + report["frame_fps"] = float(args.frame_fps) + report["anysplat_max_views"] = int(args.anysplat_max_views) + report["anysplat_mesh_grid"] = int(args.anysplat_mesh_grid) + path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def run_sfm_capture_report(args: argparse.Namespace, frames_dir: Path) -> Path: + report = args.work_dir / "sfm_capture_report.json" + command = [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_colmap_capture_report.py").as_posix(), + "--frames-dir", + frames_dir.as_posix(), + "--work-dir", + (args.work_dir / "colmap_sfm").as_posix(), + "--output", + report.as_posix(), + "--colmap-bin", + str(args.colmap_bin), + "--matcher", + str(args.sfm_matcher), + "--sequential-overlap", + str(max(int(args.sfm_sequential_overlap), 1)), + "--sift-max-num-features", + str(max(int(args.sfm_sift_max_num_features), 1)), + "--min-registered-ratio", + str(args.min_sfm_registered_ratio), + "--min-sparse-points", + str(args.min_sfm_points), + ] + if args.sfm_camera_model is not None: + command.extend(["--camera-model", str(args.sfm_camera_model)]) + if args.sfm_max_image_size is not None and int(args.sfm_max_image_size) > 0: + command.extend(["--max-image-size", str(int(args.sfm_max_image_size))]) + if args.sfm_feature_threads is not None and int(args.sfm_feature_threads) > 0: + command.extend(["--feature-threads", str(int(args.sfm_feature_threads))]) + if args.sfm_matcher_threads is not None and int(args.sfm_matcher_threads) > 0: + command.extend(["--matcher-threads", str(int(args.sfm_matcher_threads))]) + if args.sfm_sift_peak_threshold is not None: + command.extend(["--sift-peak-threshold", str(float(args.sfm_sift_peak_threshold))]) + if args.sfm_guided_matching: + command.append("--guided-matching") + if args.require_sfm: + command.append("--require-pass") + run_child_command(command) + return report + + +def sfm_frames_dir(args: argparse.Namespace, extracted_frames: list[Path], selected_frames: Path) -> Path: + if args.sfm_frames == "selected": + return selected_frames + if extracted_frames: + return extracted_frames[0].parent + return selected_frames + + +def run_capture_acceptance_report(args: argparse.Namespace, sfm_report: Path) -> Path: + report = args.work_dir / "capture_acceptance_report.json" + command = [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_capture_acceptance.py").as_posix(), + "--sfm-report", + sfm_report.as_posix(), + "--output-json", + report.as_posix(), + "--output-md", + (args.work_dir / "capture_acceptance_report.md").as_posix(), + "--min-camera-path-length", + str(float(args.min_capture_camera_path_length)), + "--min-camera-planar-coverage-ratio", + str(float(args.min_capture_camera_planar_coverage_ratio)), + "--max-camera-step-ratio", + str(float(args.max_capture_camera_step_ratio)), + ] + run_child_command_allowing_exit(command, allowed={0, 2}) + acceptance = json.loads(report.read_text(encoding="utf-8")) + if args.require_capture_acceptance and not acceptance.get("accepted", False): + raise SystemExit("capture acceptance gate failed; inspect capture_acceptance_report.json") + return report + + +def maybe_reselect_anysplat_keyframes_from_sfm( + args: argparse.Namespace, + frames: list[Path], + output_dir: Path, + quality_report: Path, + sfm_report: Path | None, + fallback_selected: list[Path], +) -> list[Path]: + if args.anysplat_keyframe_strategy != "sfm-registered" or sfm_report is None: + return fallback_selected + if not sfm_report.is_file(): + return fallback_selected + try: + report = json.loads(sfm_report.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return fallback_selected + if not report.get("passed"): + return fallback_selected + selected = select_sfm_registered_keyframes( + frames, + output_dir, + sfm_report=sfm_report, + sfm_report_data=report, + max_views=max(int(args.anysplat_max_views), 1), + orientation_weight=max(0.0, min(float(args.anysplat_keyframe_orientation_weight), 1.0)), + quality_report=quality_report, + min_quality_score=float(args.min_video_quality), + ) + return selected or fallback_selected + + +def attach_capture_acceptance_to_package(package_dir: Path, acceptance_report: Path) -> Path: + artifact_dir = package_dir / "artifacts" + artifact_dir.mkdir(parents=True, exist_ok=True) + dest = artifact_dir / "capture_acceptance_report.json" + dest.write_text(acceptance_report.read_text(encoding="utf-8"), encoding="utf-8") + source_md = acceptance_report.with_suffix(".md") + if source_md.is_file(): + (artifact_dir / "capture_acceptance_report.md").write_text(source_md.read_text(encoding="utf-8"), encoding="utf-8") + + import_path = package_dir / "motrix_import.json" + if import_path.is_file(): + manifest = json.loads(import_path.read_text(encoding="utf-8")) + report = json.loads(dest.read_text(encoding="utf-8")) + manifest["capture_acceptance_report"] = dest.relative_to(package_dir).as_posix() + metrics = report.get("metrics") if isinstance(report.get("metrics"), dict) else {} + manifest.setdefault("verification", {})["capture_acceptance"] = { + "accepted": report.get("accepted"), + "decision": report.get("decision"), + "blockers": report.get("blockers", []), + "camera_path_length": metrics.get("camera_path_length"), + "camera_planar_coverage_ratio": metrics.get("camera_planar_coverage_ratio"), + "camera_max_step_ratio": metrics.get("camera_max_step_ratio"), + } + import_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return dest + + +def attach_sfm_report_to_package(package_dir: Path, sfm_report: Path) -> Path: + artifact_dir = package_dir / "artifacts" + artifact_dir.mkdir(parents=True, exist_ok=True) + dest = artifact_dir / "sfm_capture_report.json" + report = json.loads(sfm_report.read_text(encoding="utf-8")) + coverage_dest = copy_sfm_coverage_preview(report, artifact_dir, package_dir) + dest.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + import_path = package_dir / "motrix_import.json" + if import_path.is_file(): + manifest = json.loads(import_path.read_text(encoding="utf-8")) + manifest["sfm_capture_report"] = dest.relative_to(package_dir).as_posix() + if coverage_dest is not None: + manifest["sfm_capture_coverage_preview"] = coverage_dest.relative_to(package_dir).as_posix() + manifest.setdefault("verification", {})["sfm_capture"] = { + "passed": report.get("passed"), + "frame_count": report.get("frame_count"), + "registered_image_count": report.get("registered_image_count"), + "registered_ratio": report.get("registered_ratio"), + "sparse_point_count": report.get("sparse_point_count"), + "camera_pose_count": (report.get("camera_path") or {}).get("pose_count"), + "camera_path_length": (report.get("camera_path") or {}).get("path_length"), + "coverage_preview": coverage_dest.relative_to(package_dir).as_posix() if coverage_dest is not None else None, + "warnings": report.get("warnings", []), + } + import_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return dest + + +def should_run_colmap_dense(args: argparse.Namespace, sfm_report: Path | None) -> bool: + if not getattr(args, "colmap_dense", False): + return False + if sfm_report is None: + if getattr(args, "require_colmap_dense", False): + raise SystemExit("--colmap-dense requires --sfm-report") + return False + try: + report = json.loads(sfm_report.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + if getattr(args, "require_colmap_dense", False): + raise SystemExit("COLMAP dense reconstruction requires a readable SfM report") + return False + if not report.get("passed", False): + if getattr(args, "require_colmap_dense", False): + raise SystemExit("COLMAP dense reconstruction requires a passing SfM report") + return False + return True + + +def run_colmap_dense_reconstruction_to_package( + args: argparse.Namespace, + package_dir: Path, + sfm_report: Path | None, + helpers: Any, +) -> Path: + if sfm_report is None: + raise SystemExit("--colmap-dense requires a passing SfM report") + output_dir = ( + args.work_dir / "colmap_dense" + if bool(args.colmap_dense_replace_scene_mesh) + else package_dir / "artifacts" / "colmap_dense" + ) + report_path = output_dir / "colmap_dense_report.json" + command = [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_colmap_dense_reconstruct.py").as_posix(), + "--sfm-report", + sfm_report.as_posix(), + "--output-dir", + output_dir.as_posix(), + "--report", + report_path.as_posix(), + "--colmap-bin", + str(args.colmap_dense_bin or args.colmap_bin), + "--max-image-size", + str(max(int(args.colmap_dense_max_image_size), 1)), + "--min-fused-points", + str(max(int(args.colmap_dense_min_fused_points), 1)), + "--poisson-depth", + str(max(int(args.colmap_dense_poisson_depth), 5)), + ] + if str(args.colmap_dense_gpu_index).strip(): + command.extend(["--gpu-index", str(args.colmap_dense_gpu_index).strip()]) + if bool(args.colmap_dense_run_poisson): + command.append("--run-poisson") + if bool(args.require_colmap_dense): + command.append("--require-pass") + run_child_command(command) + attach_colmap_dense_preview(report_path, package_dir, helpers) + if bool(args.colmap_dense_replace_scene_mesh): + replace_package_scene_mesh_with_colmap_dense(args, package_dir, report_path) + report_path = materialize_colmap_dense_package_artifacts(package_dir, report_path) + attach_colmap_dense_to_package(package_dir, report_path) + return report_path + + +def attach_colmap_dense_preview(report_path: Path, package_dir: Path, helpers: Any) -> None: + if not report_path.is_file(): + return + report = json.loads(report_path.read_text(encoding="utf-8")) + fused_value = report.get("fused_ply") + if not fused_value: + return + fused = Path(str(fused_value)) + if not fused.is_file(): + return + preview = report_path.parent / "fused_projection_preview.png" + try: + helpers.write_projection_preview(fused, preview) + report["projection_preview"] = preview.as_posix() + except (Exception, SystemExit) as exc: + report.setdefault("warnings", []).append(f"failed to write dense projection preview: {exc}") + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def replace_package_scene_mesh_with_colmap_dense( + args: argparse.Namespace, + package_dir: Path, + report_path: Path, +) -> None: + if not report_path.is_file(): + return + report = json.loads(report_path.read_text(encoding="utf-8")) + mesh_value = report.get("mesh_ply") or report.get("fused_ply") + if not mesh_value: + report.setdefault("warnings", []).append("dense scene mesh replacement skipped: no dense mesh/fused PLY") + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return + mesh_ply = Path(str(mesh_value)) + if not mesh_ply.is_file(): + report.setdefault("warnings", []).append(f"dense scene mesh replacement skipped: missing {mesh_ply}") + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return + + staging = args.work_dir / "colmap_dense_repackage" + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True, exist_ok=True) + manifest = json.loads((package_dir / "motrix_import.json").read_text(encoding="utf-8")) + package_manifest = json.loads((package_dir / "package_manifest.json").read_text(encoding="utf-8")) + + scene_ply = copy_package_relative(package_dir, manifest["scene_gaussians"]["scene"], staging / "scene.ply") + camera_artifacts = { + key: copy_package_relative(package_dir, value, staging / Path(value).name) + for key, value in (manifest.get("camera_artifacts") or {}).items() + if value + } + preview = copy_package_relative(package_dir, manifest.get("preview"), staging / "source_gs_preview.png") + source = Path(str(package_manifest.get("source") or "")) + if not source.is_file(): + source = camera_artifacts.get("background_image") or preview or scene_ply + + dense_obj = staging / "scene_colmap_dense.obj" + dense_obj_report = staging / "scene_colmap_dense_obj_report.json" + run_child_command( + [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_ply_mesh_to_obj.py").as_posix(), + "--input-ply", + mesh_ply.as_posix(), + "--output-obj", + dense_obj.as_posix(), + "--report", + dense_obj_report.as_posix(), + ] + ) + dense_obj = maybe_cleanup_colmap_dense_obj(args, dense_obj, staging, report) + scene_structure_report = maybe_write_scene_structure_report(dense_obj, staging, report) + scene_mesh_for_package = maybe_texture_bake_colmap_dense_obj(args, dense_obj, staging, report) + + package_command = [ + sys.executable, + "-m", + "real2sim", + "package-existing", + "--source", + source.as_posix(), + "--scene-ply", + scene_ply.as_posix(), + "--scene-mesh", + scene_mesh_for_package.as_posix(), + "--output", + package_dir.as_posix(), + "--prompt", + args.scene_prompt or args.prompt, + ] + optional_args = { + "--cameras-json": camera_artifacts.get("cameras_json"), + "--depth-map": camera_artifacts.get("depth_map"), + "--background-image": camera_artifacts.get("background_image"), + "--preview-image": preview, + } + for flag, path in optional_args.items(): + if path is not None and path.is_file(): + package_command.extend([flag, path.as_posix()]) + if report.get("mesh_cleanup_report"): + package_command.extend(["--mesh-cleanup-report", str(report["mesh_cleanup_report"])]) + if report.get("texture_bake_report"): + package_command.extend(["--texture-bake-report", str(report["texture_bake_report"])]) + if scene_structure_report is not None: + package_command.extend(["--scene-structure-report", scene_structure_report.as_posix()]) + run_child_command(package_command, env=motrix_package_env(args)) + + obj_report = json.loads(dense_obj_report.read_text(encoding="utf-8")) + report["scene_mesh_replacement"] = { + "enabled": True, + "mesh_source": mesh_ply.as_posix(), + "obj_report": obj_report, + "mesh_cleanup_report": report.get("mesh_cleanup_report"), + "scene_structure_report": report.get("scene_structure_report"), + "texture_bake_report": report.get("texture_bake_report"), + "package_scene_mesh": "meshes/scene.obj", + } + report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def maybe_write_scene_structure_report( + dense_obj: Path, + staging: Path, + report: dict[str, Any], +) -> Path | None: + structure_report = staging / "scene_structure_report.json" + command = [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_obj_structure_report.py").as_posix(), + "--input-obj", + dense_obj.as_posix(), + "--report", + structure_report.as_posix(), + ] + try: + run_child_command(command) + except (Exception, SystemExit) as exc: + report.setdefault("warnings", []).append(f"scene structure report failed: {exc}") + return None + if not structure_report.is_file(): + report.setdefault("warnings", []).append("scene structure report did not produce an output file") + return None + data = json.loads(structure_report.read_text(encoding="utf-8")) + report["scene_structure_report"] = structure_report.as_posix() + report["scene_structure_summary"] = { + "format": data.get("format"), + "structure_score": data.get("structure_score"), + "support_area_ratio": (data.get("support_planes") or {}).get("support_area_ratio") + if isinstance(data.get("support_planes"), dict) + else None, + } + return structure_report + + +def maybe_cleanup_colmap_dense_obj( + args: argparse.Namespace, + dense_obj: Path, + staging: Path, + report: dict[str, Any], +) -> Path: + max_face_percentile = float(getattr(args, "colmap_dense_cleanup_max_face_area_percentile", 100.0) or 100.0) + max_edge_percentile = float(getattr(args, "colmap_dense_cleanup_max_edge_length_percentile", 100.0) or 100.0) + max_centroid_z_percentile = float(getattr(args, "colmap_dense_cleanup_max_centroid_z_percentile", 100.0) or 100.0) + min_faces = int(getattr(args, "colmap_dense_cleanup_min_faces", 0) or 0) + min_area_ratio = float(getattr(args, "colmap_dense_cleanup_min_area_ratio", 0.0) or 0.0) + keep_largest = int(getattr(args, "colmap_dense_cleanup_keep_largest", 1) or 0) + cleanup_requested = ( + max_face_percentile < 100.0 + or max_edge_percentile < 100.0 + or max_centroid_z_percentile < 100.0 + or min_faces > 0 + or min_area_ratio > 0.0 + or keep_largest != 1 + ) + if not cleanup_requested: + return dense_obj + cleaned_obj = staging / "scene_colmap_dense_cleaned.obj" + cleanup_report = staging / "scene_colmap_dense_cleanup_report.json" + command = [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_obj_mesh_cleanup.py").as_posix(), + "--input-obj", + dense_obj.as_posix(), + "--output-obj", + cleaned_obj.as_posix(), + "--report", + cleanup_report.as_posix(), + "--keep-largest", + str(max(keep_largest, 0)), + "--min-faces", + str(max(min_faces, 0)), + "--min-area-ratio", + str(max(min_area_ratio, 0.0)), + "--max-face-area-percentile", + str(max_face_percentile), + "--max-edge-length-percentile", + str(max_edge_percentile), + "--max-centroid-z-percentile", + str(max_centroid_z_percentile), + ] + try: + run_child_command(command) + except (Exception, SystemExit) as exc: + message = f"COLMAP dense OBJ cleanup failed: {exc}" + report.setdefault("warnings", []).append(message) + return dense_obj + if not cleaned_obj.is_file() or not cleanup_report.is_file(): + report.setdefault("warnings", []).append("COLMAP dense OBJ cleanup produced incomplete outputs") + return dense_obj + cleanup_data = json.loads(cleanup_report.read_text(encoding="utf-8")) + report["mesh_cleanup_report"] = cleanup_report.as_posix() + report["mesh_cleanup_summary"] = { + "format": cleanup_data.get("format"), + "input_face_count": cleanup_data.get("input_face_count"), + "kept_face_count": cleanup_data.get("kept_face_count"), + "removed_face_count": cleanup_data.get("removed_face_count"), + "centroid_filter_removed_face_count": cleanup_data.get("centroid_filter_removed_face_count"), + "kept_area_ratio": cleanup_data.get("kept_area_ratio"), + } + return cleaned_obj + + +def motrix_package_env(args: argparse.Namespace) -> dict[str, str] | None: + env = os.environ.copy() + changed = False + preview_faces = int(getattr(args, "motrix_preview_mesh_faces", 0) or 0) + if preview_faces > 0: + env["REAL2SIM_MOTRIX_PREVIEW_MESH_FACES"] = str(preview_faces) + changed = True + preview_mode = getattr(args, "motrix_preview_mode", None) + if preview_mode: + env["REAL2SIM_MOTRIX_PREVIEW_MODE"] = str(preview_mode) + changed = True + return env if changed else None + + +def maybe_texture_bake_colmap_dense_obj( + args: argparse.Namespace, + dense_obj: Path, + staging: Path, + report: dict[str, Any], +) -> Path: + if not bool(getattr(args, "colmap_texture_bake", False)): + return dense_obj + sfm_report = Path(str(report.get("sfm_report") or "")) + if not sfm_report.is_file(): + message = "COLMAP texture bake skipped: dense report has no readable sfm_report" + report.setdefault("warnings", []).append(message) + if bool(getattr(args, "require_colmap_texture_bake", False)): + raise SystemExit(message) + return dense_obj + textured_obj = staging / "scene_colmap_dense_textured.obj" + textured_png = staging / "scene_colmap_dense_texture.png" + texture_report = staging / "scene_colmap_dense_texture_bake_report.json" + command = [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_colmap_texture_bake_obj.py").as_posix(), + "--input-obj", + dense_obj.as_posix(), + "--output-obj", + textured_obj.as_posix(), + "--output-texture", + textured_png.as_posix(), + "--report", + texture_report.as_posix(), + "--sfm-report", + sfm_report.as_posix(), + "--atlas-size", + str(max(int(args.colmap_texture_bake_atlas_size), 64)), + "--max-cameras", + str(max(int(args.colmap_texture_bake_max_cameras), 1)), + "--max-faces", + str(max(int(args.colmap_texture_bake_max_faces), 0)), + "--max-output-faces", + str(max(int(args.colmap_texture_bake_max_output_faces), 0)), + "--texture-fill-mode", + args.colmap_texture_bake_fill_mode, + "--color-blend-cameras", + str(max(int(args.colmap_texture_bake_color_blend_cameras), 1)), + "--visibility-depth-tolerance", + str(max(float(args.colmap_texture_bake_visibility_depth_tolerance), 0.0)), + "--visibility-grid-size", + str(max(int(args.colmap_texture_bake_visibility_grid_size), 32)), + "--max-warped-faces", + str(max(int(args.colmap_texture_bake_max_warped_faces), 0)), + "--postprocess-median-radius", + str(max(int(args.colmap_texture_bake_postprocess_median_radius), 0)), + "--bleed-iterations", + str(max(int(args.colmap_texture_bake_bleed_iterations), 0)), + ] + if bool(getattr(args, "colmap_texture_bake_unwrap_uv", False)): + command.append("--unwrap-uv") + if bool(getattr(args, "require_colmap_texture_bake_unwrap_uv", False)): + command.append("--require-unwrap-uv") + try: + run_child_command(command) + except (Exception, SystemExit) as exc: + message = f"COLMAP texture bake failed: {exc}" + report.setdefault("warnings", []).append(message) + if bool(getattr(args, "require_colmap_texture_bake", False)): + raise + return dense_obj + if not textured_obj.is_file() or not textured_png.is_file() or not texture_report.is_file(): + message = "COLMAP texture bake produced incomplete OBJ/texture/report outputs" + report.setdefault("warnings", []).append(message) + if bool(getattr(args, "require_colmap_texture_bake", False)): + raise SystemExit(message) + return dense_obj + texture_data = json.loads(texture_report.read_text(encoding="utf-8")) + report["texture_bake_report"] = texture_report.as_posix() + report["texture_baked_obj"] = textured_obj.as_posix() + report["texture_baked_texture"] = textured_png.as_posix() + report["texture_bake_summary"] = { + "format": texture_data.get("format"), + "projected_face_count": texture_data.get("projected_face_count"), + "drawn_face_count": texture_data.get("drawn_face_count"), + "texture_coverage_ratio": texture_data.get("texture_coverage_ratio"), + "texture_fill_mode": texture_data.get("texture_fill_mode"), + "warped_face_count": texture_data.get("warped_face_count"), + "warp_fallback_face_count": texture_data.get("warp_fallback_face_count"), + "texture_postprocess": texture_data.get("texture_postprocess"), + "atlas_size": texture_data.get("atlas_size"), + "selected_camera_count": texture_data.get("selected_camera_count"), + "selected_face_count": texture_data.get("selected_face_count"), + } + return textured_obj + + +def copy_package_relative(package_dir: Path, value: Any, dest: Path) -> Path | None: + if not value: + return None + source = package_dir / str(value) + if not source.is_file(): + return None + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, dest) + return dest + + +def materialize_colmap_dense_package_artifacts(package_dir: Path, report_path: Path) -> Path: + report = json.loads(report_path.read_text(encoding="utf-8")) + dest_dir = package_dir / "artifacts" / "colmap_dense" + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + copied: dict[str, str] = {} + for key, name in ( + ("fused_ply", "fused.ply"), + ("mesh_ply", "mesh_poisson.ply"), + ("projection_preview", "fused_projection_preview.png"), + ("texture_bake_report", "texture_bake_report.json"), + ("texture_baked_texture", "scene_texture_bake.png"), + ): + value = report.get(key) + if not value: + continue + source = Path(str(value)) + if not source.is_file(): + continue + dest = dest_dir / name + shutil.copy2(source, dest) + copied[key] = dest.as_posix() + report.update(copied) + report["output_dir"] = dest_dir.as_posix() + report["package_artifacts_are_summary_only"] = True + packaged_report = dest_dir / "colmap_dense_report.json" + packaged_report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return packaged_report + + +def attach_colmap_dense_to_package(package_dir: Path, report_path: Path) -> None: + import_path = package_dir / "motrix_import.json" + if not import_path.is_file() or not report_path.is_file(): + return + report = json.loads(report_path.read_text(encoding="utf-8")) + manifest = json.loads(import_path.read_text(encoding="utf-8")) + relative_root = report_path.parent.relative_to(package_dir).as_posix() + manifest["colmap_dense_reconstruction"] = { + "root": relative_root, + "report": report_path.relative_to(package_dir).as_posix(), + "fused_ply": relative_package_file(package_dir, report.get("fused_ply")), + "mesh_ply": relative_package_file(package_dir, report.get("mesh_ply")), + "projection_preview": relative_package_file(package_dir, report.get("projection_preview")), + "texture_bake_report": relative_package_file(package_dir, report.get("texture_bake_report")), + "texture_baked_texture": relative_package_file(package_dir, report.get("texture_baked_texture")), + } + manifest.setdefault("verification", {})["colmap_dense"] = { + "format": report.get("format"), + "passed": report.get("passed"), + "fused_point_count": report.get("fused_point_count"), + "mesh_vertex_count": report.get("mesh_vertex_count"), + "mesh_face_count": report.get("mesh_face_count"), + "max_image_size": report.get("max_image_size"), + "min_fused_points": report.get("min_fused_points"), + "scene_mesh_replacement": report.get("scene_mesh_replacement"), + "texture_bake_summary": report.get("texture_bake_summary"), + "warnings": report.get("warnings", []), + "projection_preview": relative_package_file(package_dir, report.get("projection_preview")), + } + import_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + append_colmap_dense_readme(package_dir, manifest["colmap_dense_reconstruction"]) + + +def relative_package_file(package_dir: Path, value: Any) -> str | None: + if value is None: + return None + path = Path(str(value)) + if not path.exists(): + return None + try: + return path.resolve().relative_to(package_dir.resolve()).as_posix() + except ValueError: + return path.as_posix() + + +def append_colmap_dense_readme(package_dir: Path, dense_manifest: dict[str, Any]) -> None: + readme = package_dir / "MOTRIX_IMPORT.md" + if not readme.is_file(): + return + lines = [ + "", + "COLMAP dense MVS artifacts:", + "", + f"- Report: `{dense_manifest.get('report')}`", + f"- Fused point cloud: `{dense_manifest.get('fused_ply')}`", + f"- Poisson mesh: `{dense_manifest.get('mesh_ply')}`", + f"- Projection preview: `{dense_manifest.get('projection_preview')}`", + "", + ] + with readme.open("a", encoding="utf-8") as handle: + handle.write("\n".join(lines)) + + +def should_export_pose_init_dataset(args: argparse.Namespace, sfm_report: Path | None) -> bool: + if sfm_report is None or bool(getattr(args, "skip_pose_init_dataset", False)): + return False + try: + report = json.loads(sfm_report.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return bool(report.get("passed")) + + +def export_pose_init_dataset_to_package(sfm_report: Path, package_dir: Path) -> Path: + output_dir = package_dir / "artifacts" / "pose_init_dataset" + command = [ + sys.executable, + (REPO_ROOT / "scripts" / "real2sim_export_pose_init_dataset.py").as_posix(), + "--sfm-report", + sfm_report.as_posix(), + "--output-dir", + output_dir.as_posix(), + ] + run_child_command(command) + attach_pose_init_dataset_to_manifest(package_dir, output_dir) + return output_dir + + +def run_pose_init_training_to_package(args: argparse.Namespace, package_dir: Path, pose_init_dir: Path | None) -> Path | None: + if not getattr(args, "pose_init_train", False): + return None + if pose_init_dir is None: + raise SystemExit("--pose-init-train requires a passing SfM report and exported pose-init dataset") + python_bin = str(args.pose_init_train_python or sys.executable) + output_dir = package_dir / "artifacts" / "pose_init_trained" + output_ply = output_dir / "pose_init_trained_scene.ply" + preview = output_dir / "pose_init_training_preview.png" + multi_preview = output_dir / "pose_init_training_contact_sheet.png" + metrics = output_dir / "pose_init_training_metrics.json" + command = [ + python_bin, + (REPO_ROOT / "scripts" / "real2sim_pose_init_gsplat_train.py").as_posix(), + "--pose-init-dir", + pose_init_dir.as_posix(), + "--output-dir", + output_dir.as_posix(), + "--output-ply", + output_ply.as_posix(), + "--preview-output", + preview.as_posix(), + "--multi-preview-output", + multi_preview.as_posix(), + "--metrics-output", + metrics.as_posix(), + "--max-steps", + str(max(int(args.pose_init_train_steps), 1)), + "--max-frames", + str(max(int(args.pose_init_train_max_frames), 1)), + "--image-max-size", + str(max(int(args.pose_init_train_image_max_size), 1)), + "--target-gaussians", + str(max(int(args.pose_init_train_target_gaussians), 1)), + "--max-points", + str(max(int(args.pose_init_train_max_points), 1)), + "--dense-image-points-per-frame", + str(max(int(args.pose_init_train_dense_image_points_per_frame), 1)), + "--dense-depth-neighbors", + str(max(int(args.pose_init_train_dense_depth_neighbors), 1)), + "--dense-depth-max-distance", + str(max(float(args.pose_init_train_dense_depth_max_distance), 1.0)), + "--adaptive-densify-every", + str(max(int(args.pose_init_train_adaptive_densify_every), 0)), + "--adaptive-densify-start", + str(max(int(args.pose_init_train_adaptive_densify_start), 1)), + "--adaptive-densify-stop", + str(max(int(args.pose_init_train_adaptive_densify_stop), 0)), + "--adaptive-densify-points", + str(max(int(args.pose_init_train_adaptive_densify_points), 1)), + "--adaptive-densify-error-percentile", + str(max(min(float(args.pose_init_train_adaptive_densify_error_percentile), 99.0), 1.0)), + "--preview-frames", + str(max(int(args.pose_init_train_preview_frames), 1)), + ] + run_child_command(command) + if args.pose_init_train_replace_scene: + scene_ply = package_dir / "3dgs" / "scene.ply" + backup = output_dir / "scene_before_pose_init_train.ply" + if scene_ply.is_file(): + shutil.copy2(scene_ply, backup) + shutil.copy2(output_ply, scene_ply) + attach_pose_init_training_to_manifest( + package_dir, + output_dir, + replace_scene=bool(args.pose_init_train_replace_scene), + ) + return metrics + + +def attach_pose_init_dataset_to_manifest(package_dir: Path, pose_init_dir: Path) -> None: + import_path = package_dir / "motrix_import.json" + summary_path = pose_init_dir / "pose_init_dataset.json" + if not import_path.is_file() or not summary_path.is_file(): + return + manifest = json.loads(import_path.read_text(encoding="utf-8")) + summary = json.loads(summary_path.read_text(encoding="utf-8")) + relative_root = pose_init_dir.relative_to(package_dir).as_posix() + manifest["pose_init_dataset"] = relative_root + manifest.setdefault("verification", {})["pose_init_dataset"] = { + "format": summary.get("format"), + "root": relative_root, + "transforms": f"{relative_root}/{summary.get('transforms', 'transforms.json')}", + "images": f"{relative_root}/{summary.get('images', 'images')}", + "image_count": summary.get("image_count"), + "sparse_points_ply": f"{relative_root}/{summary.get('sparse_points_ply')}" + if summary.get("sparse_points_ply") + else None, + "sparse_point_count": summary.get("sparse_point_count"), + "registered_ratio": summary.get("registered_ratio"), + "ready_for_training": summary.get("ready_for_training"), + } + import_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def attach_pose_init_training_to_manifest(package_dir: Path, training_dir: Path, *, replace_scene: bool) -> None: + import_path = package_dir / "motrix_import.json" + metrics_path = training_dir / "pose_init_training_metrics.json" + output_ply = training_dir / "pose_init_trained_scene.ply" + preview = training_dir / "pose_init_training_preview.png" + multi_preview = training_dir / "pose_init_training_contact_sheet.png" + if not import_path.is_file() or not metrics_path.is_file(): + return + metrics = json.loads(metrics_path.read_text(encoding="utf-8")) + manifest = json.loads(import_path.read_text(encoding="utf-8")) + relative_root = training_dir.relative_to(package_dir).as_posix() + manifest["pose_init_training"] = { + "root": relative_root, + "scene_ply": output_ply.relative_to(package_dir).as_posix() if output_ply.is_file() else None, + "preview": preview.relative_to(package_dir).as_posix() if preview.is_file() else None, + "multi_preview": multi_preview.relative_to(package_dir).as_posix() if multi_preview.is_file() else None, + "metrics": metrics_path.relative_to(package_dir).as_posix(), + "replaced_package_scene": replace_scene, + } + manifest.setdefault("verification", {})["pose_init_training"] = { + "format": metrics.get("format"), + "trained_gaussian_count": metrics.get("trained_gaussian_count"), + "frame_count": metrics.get("frame_count"), + "max_steps": metrics.get("max_steps"), + "final_loss": metrics.get("final_loss"), + "final_preview_psnr": metrics.get("final_preview_psnr"), + "final_preview_mae": metrics.get("final_preview_mae"), + "mean_step_psnr_last_10": metrics.get("mean_step_psnr_last_10"), + "mean_step_mae_last_10": metrics.get("mean_step_mae_last_10"), + "multi_preview_mean_psnr": metrics.get("multi_preview_mean_psnr"), + "multi_preview_mean_mae": metrics.get("multi_preview_mean_mae"), + "adaptive_densify": { + "enabled": (metrics.get("adaptive_densify") or {}).get("enabled"), + "event_count": (metrics.get("adaptive_densify") or {}).get("event_count"), + "added_point_count": (metrics.get("adaptive_densify") or {}).get("added_point_count"), + }, + } + import_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def copy_sfm_coverage_preview(report: dict[str, Any], artifact_dir: Path, package_dir: Path) -> Path | None: + coverage_value = report.get("coverage_preview") + if not coverage_value: + return None + source = Path(str(coverage_value)) + if not source.is_file(): + return None + dest = artifact_dir / "sfm_capture_coverage.png" + shutil.copy2(source, dest) + report["coverage_preview"] = dest.relative_to(package_dir).as_posix() + return dest + + +def _env_bool(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str) -> int | None: + value = os.environ.get(name) + if value is None or value.strip() == "": + return None + return int(value) + + +def extract_video_frames(video: Path, frames_dir: Path, *, fps: float) -> list[Path]: + if frames_dir.exists(): + shutil.rmtree(frames_dir) + frames_dir.mkdir(parents=True, exist_ok=True) + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + raise SystemExit("video workflow requires ffmpeg") + command = [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + video.as_posix(), + "-vf", + f"fps={fps:g}", + (frames_dir / "frame_%06d.png").as_posix(), + ] + completed = subprocess.run(command, check=False) + if completed.returncode != 0: + raise SystemExit(f"ffmpeg frame extraction failed with exit code {completed.returncode}") + frames = sorted(path for path in frames_dir.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES) + if not frames: + raise SystemExit(f"ffmpeg produced no frames in {frames_dir}") + return frames + + +def collect_existing_frames(frames_dir: Path) -> list[Path]: + if not frames_dir.is_dir(): + raise SystemExit(f"frames directory not found: {frames_dir}") + frames = sorted(path for path in frames_dir.iterdir() if is_video_frame_image(path)) + if not frames: + raise SystemExit(f"frames directory contains no supported images: {frames_dir}") + return frames + + +def normalize_existing_frames(frames: list[Path], output_dir: Path) -> list[Path]: + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + normalized: list[Path] = [] + for index, src in enumerate(frames, start=1): + dst = output_dir / f"frame_{index:06d}.png" + write_normalized_frame(src, dst) + normalized.append(dst) + return normalized + + +def is_video_frame_image(path: Path) -> bool: + if not path.is_file() or path.suffix.lower() not in IMAGE_SUFFIXES: + return False + name = path.name.lower() + return "contact_sheet" not in name and "preview" not in name + + +def select_video_keyframes( + frames: list[Path], + output_dir: Path, + *, + max_views: int, + quality_report: Path, + min_quality_score: float, +) -> list[Path]: + if not frames: + raise SystemExit("no video frames available for keyframe selection") + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + frame_metrics = {index: frame_quality(frames[index]) for index in range(len(frames))} + selected_indices = select_diverse_keyframe_indices(frames, frame_metrics, max_views=max_views) + selected_paths: list[Path] = [] + for output_index, source_index in enumerate(selected_indices, start=1): + src = frames[source_index] + dst = output_dir / f"frame_{output_index:06d}.png" + write_normalized_frame(src, dst) + selected_paths.append(dst) + contact_sheet = write_keyframe_contact_sheet(selected_paths, output_dir / "selected_keyframes_contact_sheet.png") + + selected_metrics = [frame_metrics[index] for index in selected_indices] + diversity = selected_frame_diversity(selected_paths) + mean_sharpness = float(np.mean([metric["sharpness"] for metric in selected_metrics])) + mean_exposure = float(np.mean([metric["exposure"] for metric in selected_metrics])) + frame_count_score = min(1.0, len(selected_paths) / max(float(max_views), 1.0)) + diversity_score = max(0.0, min(1.0, diversity / 0.10)) + score = 0.26 * frame_count_score + 0.28 * mean_sharpness + 0.22 * mean_exposure + 0.24 * diversity_score + warnings = video_quality_warnings( + frame_count=len(selected_paths), + requested_views=max_views, + mean_sharpness=mean_sharpness, + mean_exposure=mean_exposure, + diversity=diversity, + ) + report = { + "passed": float(score) >= min_quality_score, + "score": round(float(score), 4), + "min_quality_score": min_quality_score, + "source_frame_count": len(frames), + "selected_frame_count": len(selected_paths), + "requested_views": max_views, + "frame_fusion": "video_keyframes", + "selection_strategy": "coverage_anchored_quality_diversity_v1", + "mean_sharpness": round(mean_sharpness, 4), + "mean_exposure": round(mean_exposure, 4), + "mean_temporal_diversity": round(float(diversity), 4), + "selected_source_indices": selected_indices, + "selected_frames": [path.as_posix() for path in selected_paths], + "contact_sheet": contact_sheet.as_posix(), + "warnings": warnings, + } + quality_report.parent.mkdir(parents=True, exist_ok=True) + quality_report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return selected_paths + + +def select_sfm_registered_keyframes( + frames: list[Path], + output_dir: Path, + *, + sfm_report: Path, + sfm_report_data: dict[str, Any], + max_views: int, + orientation_weight: float, + quality_report: Path, + min_quality_score: float, +) -> list[Path]: + frame_by_name = {path.name: path for path in frames} + frame_index_by_name = {path.name: index for index, path in enumerate(frames)} + ordered_names = sfm_registered_pose_names(sfm_report_data, frame_by_name) + if not ordered_names: + return [] + target_count = min(max_views, len(ordered_names)) + if target_count <= 0: + return [] + + frame_metrics = { + frame_index_by_name[name]: frame_quality(frame_by_name[name]) + for name in ordered_names + if name in frame_index_by_name + } + pose_alignment = sfm_camera_up_alignment(sfm_report_data) + selected_names = select_pose_binned_registered_names( + ordered_names, + frame_index_by_name=frame_index_by_name, + frame_metrics=frame_metrics, + pose_alignment=pose_alignment, + orientation_weight=orientation_weight, + target_count=target_count, + ) + if not selected_names: + return [] + + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + selected_paths: list[Path] = [] + selected_indices: list[int] = [] + for output_index, name in enumerate(selected_names, start=1): + src = frame_by_name[name] + dst = output_dir / f"frame_{output_index:06d}.png" + write_normalized_frame(src, dst) + selected_paths.append(dst) + selected_indices.append(frame_index_by_name[name]) + contact_sheet = write_keyframe_contact_sheet(selected_paths, output_dir / "selected_keyframes_contact_sheet.png") + + selected_metrics = [frame_metrics[index] for index in selected_indices if index in frame_metrics] + diversity = selected_frame_diversity(selected_paths) + mean_sharpness = float(np.mean([metric["sharpness"] for metric in selected_metrics])) if selected_metrics else 0.0 + mean_exposure = float(np.mean([metric["exposure"] for metric in selected_metrics])) if selected_metrics else 0.0 + frame_count_score = min(1.0, len(selected_paths) / max(float(max_views), 1.0)) + diversity_score = max(0.0, min(1.0, diversity / 0.10)) + registered_ratio = float(sfm_report_data.get("registered_ratio") or 0.0) + selected_alignment = [pose_alignment.get(name, 0.0) for name in selected_names] + score = 0.22 * frame_count_score + 0.20 * mean_sharpness + 0.18 * mean_exposure + 0.20 * diversity_score + 0.20 * min(registered_ratio, 1.0) + warnings = video_quality_warnings( + frame_count=len(selected_paths), + requested_views=max_views, + mean_sharpness=mean_sharpness, + mean_exposure=mean_exposure, + diversity=diversity, + ) + report = { + "passed": float(score) >= min_quality_score, + "score": round(float(score), 4), + "min_quality_score": min_quality_score, + "source_frame_count": len(frames), + "registered_frame_count": len(ordered_names), + "selected_frame_count": len(selected_paths), + "requested_views": max_views, + "frame_fusion": "image_set_sfm_registered_keyframes", + "selection_strategy": "sfm_registered_pose_bins_v1", + "source_sfm_report": sfm_report.as_posix(), + "registered_ratio": round(registered_ratio, 4), + "orientation_weight": round(float(orientation_weight), 4), + "mean_pose_up_alignment": round(float(np.mean(selected_alignment)), 4) if selected_alignment else None, + "min_pose_up_alignment": round(float(np.min(selected_alignment)), 4) if selected_alignment else None, + "mean_sharpness": round(mean_sharpness, 4), + "mean_exposure": round(mean_exposure, 4), + "mean_temporal_diversity": round(float(diversity), 4), + "selected_source_indices": selected_indices, + "selected_registered_images": selected_names, + "selected_frames": [path.as_posix() for path in selected_paths], + "contact_sheet": contact_sheet.as_posix(), + "warnings": warnings, + } + quality_report.parent.mkdir(parents=True, exist_ok=True) + quality_report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return selected_paths + + +def sfm_registered_pose_names(sfm_report_data: dict[str, Any], frame_by_name: dict[str, Path]) -> list[str]: + names: list[str] = [] + seen: set[str] = set() + for pose in sfm_report_data.get("camera_poses", []): + if not isinstance(pose, dict): + continue + name = str(pose.get("image_name") or "") + if name in frame_by_name and name not in seen: + names.append(name) + seen.add(name) + if not names: + for name in sfm_report_data.get("registered_images", []): + name = str(name) + if name in frame_by_name and name not in seen: + names.append(name) + seen.add(name) + return sorted(names, key=frame_name_sort_key) + + +def select_pose_binned_registered_names( + ordered_names: list[str], + *, + frame_index_by_name: dict[str, int], + frame_metrics: dict[int, dict[str, float]], + pose_alignment: dict[str, float], + orientation_weight: float, + target_count: int, +) -> list[str]: + if target_count >= len(ordered_names): + return list(ordered_names) + selected: list[str] = [] + edges = np.linspace(0, len(ordered_names), target_count + 1, dtype=int) + for bin_index in range(target_count): + start = int(edges[bin_index]) + stop = int(edges[bin_index + 1]) + if stop <= start: + stop = min(start + 1, len(ordered_names)) + candidates = ordered_names[start:stop] + if not candidates: + continue + best = max( + candidates, + key=lambda name: registered_frame_score( + name, + frame_index_by_name, + frame_metrics, + pose_alignment=pose_alignment, + orientation_weight=orientation_weight, + ), + ) + if best not in selected: + selected.append(best) + if len(selected) < target_count: + selected_set = set(selected) + for name in sorted( + ordered_names, + key=lambda item: registered_frame_score( + item, + frame_index_by_name, + frame_metrics, + pose_alignment=pose_alignment, + orientation_weight=orientation_weight, + ), + reverse=True, + ): + if name not in selected_set: + selected.append(name) + selected_set.add(name) + if len(selected) >= target_count: + break + return sorted(selected, key=frame_name_sort_key) + + +def registered_frame_score( + name: str, + frame_index_by_name: dict[str, int], + frame_metrics: dict[int, dict[str, float]], + *, + pose_alignment: dict[str, float] | None = None, + orientation_weight: float = 0.0, +) -> float: + index = frame_index_by_name.get(name) + if index is None: + return 0.0 + metrics = frame_metrics.get(index, {}) + base = 0.62 * float(metrics.get("sharpness", 0.0)) + 0.38 * float(metrics.get("exposure", 0.0)) + alignment = float((pose_alignment or {}).get(name, 0.0)) + weight = max(0.0, min(float(orientation_weight), 1.0)) + return (1.0 - weight) * base + weight * alignment + + +def sfm_camera_up_alignment(sfm_report_data: dict[str, Any]) -> dict[str, float]: + up_by_name: dict[str, np.ndarray] = {} + for pose in sfm_report_data.get("camera_poses", []): + if not isinstance(pose, dict): + continue + name = str(pose.get("image_name") or "") + qvec = pose.get("qvec_wxyz") + if not name or not isinstance(qvec, list) or len(qvec) != 4: + continue + rotation = qvec_wxyz_to_rotation([float(value) for value in qvec]) + up = rotation.T @ np.array([0.0, -1.0, 0.0], dtype=np.float64) + norm = float(np.linalg.norm(up)) + if norm > 1e-9: + up_by_name[name] = up / norm + if not up_by_name: + return {} + dominant = np.mean(np.stack(list(up_by_name.values()), axis=0), axis=0) + dominant_norm = float(np.linalg.norm(dominant)) + if dominant_norm <= 1e-9: + return {name: 0.0 for name in up_by_name} + dominant = dominant / dominant_norm + return { + name: max(0.0, float(np.dot(up, dominant))) + for name, up in up_by_name.items() + } + + +def qvec_wxyz_to_rotation(qvec: list[float]) -> np.ndarray: + w, x, y, z = qvec + return np.array( + [ + [1.0 - 2.0 * y * y - 2.0 * z * z, 2.0 * x * y - 2.0 * z * w, 2.0 * x * z + 2.0 * y * w], + [2.0 * x * y + 2.0 * z * w, 1.0 - 2.0 * x * x - 2.0 * z * z, 2.0 * y * z - 2.0 * x * w], + [2.0 * x * z - 2.0 * y * w, 2.0 * y * z + 2.0 * x * w, 1.0 - 2.0 * x * x - 2.0 * y * y], + ], + dtype=np.float64, + ) + + +def frame_name_sort_key(name: str) -> tuple[int, str]: + stem = Path(name).stem + digits = "".join(ch for ch in stem if ch.isdigit()) + if digits: + return (int(digits), name) + return (0, name) + + +def select_diverse_keyframe_indices( + frames: list[Path], + frame_metrics: dict[int, dict[str, float]], + *, + max_views: int, +) -> list[int]: + target_count = min(max_views, len(frames)) + if target_count <= 0: + return [] + if target_count == len(frames): + return list(range(len(frames))) + + features = [frame_feature(path) for path in frames] + base_scores = { + index: 0.62 * frame_metrics[index]["sharpness"] + 0.38 * frame_metrics[index]["exposure"] + for index in range(len(frames)) + } + selected: list[int] = [] + + anchor_width = max(1, int(np.ceil(len(frames) / max(float(target_count), 1.0)))) + selected.append(best_frame_in_range(0, anchor_width, base_scores)) + if target_count > 1: + tail_start = max(0, len(frames) - anchor_width) + selected.append(best_frame_in_range(tail_start, len(frames), base_scores)) + + selected = sorted(dict.fromkeys(selected)) + target_spacing = max(1.0, (len(frames) - 1) / max(float(target_count - 1), 1.0)) + while len(selected) < target_count: + selected_set = set(selected) + best_index = None + best_score = -1.0 + for index in range(len(frames)): + if index in selected_set: + continue + visual_novelty = min( + frame_feature_distance(features[index], features[selected_index]) + for selected_index in selected + ) + visual_score = max(0.0, min(1.0, visual_novelty / 0.10)) + temporal_novelty = min(abs(index - selected_index) for selected_index in selected) / target_spacing + temporal_score = max(0.0, min(1.0, temporal_novelty)) + score = 0.48 * base_scores[index] + 0.34 * visual_score + 0.18 * temporal_score + if score > best_score: + best_score = score + best_index = index + if best_index is None: + break + selected.append(best_index) + selected.sort() + return selected + + +def best_frame_in_range(start: int, stop: int, base_scores: dict[int, float]) -> int: + return max(range(start, stop), key=lambda index: base_scores[index]) + + +def frame_feature(path: Path) -> np.ndarray: + return np.asarray( + open_rgb_image(path).resize((96, 54), PIL_BILINEAR), + dtype=np.float32, + ) / 255.0 + + +def frame_feature_distance(a: np.ndarray, b: np.ndarray) -> float: + return float(np.mean(np.abs(a - b))) + + +def write_keyframe_contact_sheet(frames: list[Path], output: Path) -> Path: + if not frames: + raise SystemExit("cannot write keyframe contact sheet without frames") + columns = min(4, len(frames)) + rows = int(np.ceil(len(frames) / columns)) + tile = (320, 180) + header_h = 26 + canvas = Image.new("RGB", (columns * tile[0], rows * (tile[1] + header_h)), "white") + for index, path in enumerate(frames): + image = open_rgb_image(path) + image.thumbnail(tile) + col = index % columns + row = index // columns + x = col * tile[0] + (tile[0] - image.width) // 2 + y = row * (tile[1] + header_h) + header_h + (tile[1] - image.height) // 2 + canvas.paste(image, (x, y)) + output.parent.mkdir(parents=True, exist_ok=True) + canvas.save(output) + return output + + +def frame_quality(path: Path) -> dict[str, float]: + image = open_rgb_image(path) + gray = np.asarray(image.convert("L").resize((192, 108), PIL_BILINEAR), dtype=np.float32) / 255.0 + gx = np.diff(gray, axis=1) + gy = np.diff(gray, axis=0) + sharpness = max(0.0, min(1.0, float(np.mean(np.abs(gx)) + np.mean(np.abs(gy))) / 0.10)) + clipped = float(np.mean((gray < 0.025) | (gray > 0.975))) + contrast = float(np.std(gray)) + exposure = max(0.0, min(1.0, (1.0 - clipped * 2.0) * min(contrast / 0.22, 1.0))) + return {"sharpness": sharpness, "exposure": exposure} + + +def selected_frame_diversity(frames: list[Path]) -> float: + if len(frames) < 2: + return 0.0 + arrays = [ + np.asarray(open_rgb_image(path).resize((128, 72), PIL_BILINEAR), dtype=np.float32) / 255.0 + for path in frames + ] + diffs = [float(np.mean(np.abs(arrays[index] - arrays[index - 1]))) for index in range(1, len(arrays))] + return float(np.mean(diffs)) + + +def open_rgb_image(path: Path) -> Image.Image: + return ImageOps.exif_transpose(Image.open(path)).convert("RGB") + + +def write_normalized_frame(src: Path, dst: Path) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + open_rgb_image(src).save(dst) + + +def video_quality_warnings( + *, + frame_count: int, + requested_views: int, + mean_sharpness: float, + mean_exposure: float, + diversity: float, +) -> list[str]: + warnings: list[str] = [] + if frame_count < min(4, requested_views): + warnings.append("too few selected video keyframes for scene reconstruction") + if mean_sharpness < 0.32: + warnings.append("selected video frames are blurry or low-detail") + if mean_exposure < 0.32: + warnings.append("selected video frames have weak contrast or clipped exposure") + if diversity < 0.025 and frame_count > 1: + warnings.append("video camera motion may be too small for scene reconstruction") + return warnings + + +def attach_video_quality_to_package(package_dir: Path, quality_report: Path) -> Path: + dest = package_dir / "video_frame_quality.json" + dest.write_text(quality_report.read_text(encoding="utf-8"), encoding="utf-8") + import_path = package_dir / "motrix_import.json" + if import_path.is_file(): + manifest = json.loads(import_path.read_text(encoding="utf-8")) + manifest["video_frame_quality"] = dest.relative_to(package_dir).as_posix() + manifest["recommended_camera"] = "source_3dgs_preview" + manifest["viewer_args"] = source_preview_viewer_args(manifest.get("viewer_args")) + quality = json.loads(dest.read_text(encoding="utf-8")) + manifest.setdefault("verification", {})["video_frame_quality"] = { + "score": quality.get("score"), + "selected_frame_count": quality.get("selected_frame_count"), + "source_frame_count": quality.get("source_frame_count"), + "mean_temporal_diversity": quality.get("mean_temporal_diversity"), + } + import_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return dest + + +def source_preview_viewer_args(value: Any) -> list[str]: + args = [str(item) for item in value] if isinstance(value, list) else [] + if not args: + args = ["--package", "/path/to/package", "--camera", "overview"] + args = set_arg_value(args, "--camera", "closeup") + for item in ("--static-gs-preview", "--show-source-gs", "--source-gs-only"): + if item not in args: + args.append(item) + return args + + +def set_arg_value(args: list[str], flag: str, value: str) -> list[str]: + updated = list(args) + if flag in updated: + index = updated.index(flag) + if index + 1 < len(updated): + updated[index + 1] = value + return updated + updated.extend([flag, value]) + return updated + + +def write_visual_report( + package_dir: Path, + output_dir: Path, + *, + min_scene_vertices: int, + min_source_preview_psnr: float, + max_source_preview_mae: float, + motrix_viewer_render: Path | None, +) -> dict[str, Any]: + verifier = load_verifier() + report = verifier.verify_package( + package_dir, + min_scene_vertices=min_scene_vertices, + min_source_preview_psnr=min_source_preview_psnr, + max_source_preview_mae=max_source_preview_mae, + ) + return verifier.write_visual_acceptance_report( + package_dir, + report, + output_dir, + motrix_viewer_render=motrix_viewer_render, + ) + + +def load_one_image_helpers(): + return load_module("real2sim_one_image_motrix_scene_helpers", REPO_ROOT / "scripts" / "real2sim_one_image_motrix_scene.py") + + +def load_verifier(): + return load_module("real2sim_verify_motrix_package_helpers", REPO_ROOT / "scripts" / "real2sim_verify_motrix_package.py") + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise SystemExit(f"failed to load helper module: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run_child_command(command: list[str], *, env: dict[str, str] | None = None) -> None: + result = subprocess.run(command, cwd=REPO_ROOT, env=env, text=True, stdout=sys.stderr, stderr=sys.stderr) + if result.returncode != 0: + raise SystemExit(result.returncode) + + +def run_child_command_allowing_exit(command: list[str], *, allowed: set[int], env: dict[str, str] | None = None) -> None: + result = subprocess.run(command, cwd=REPO_ROOT, env=env, text=True, stdout=sys.stderr, stderr=sys.stderr) + if result.returncode not in allowed: + raise SystemExit(result.returncode) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vendor/gs_playground/scripts/real2sim_wuying_launch_package.sh b/vendor/gs_playground/scripts/real2sim_wuying_launch_package.sh new file mode 100755 index 0000000..9fefc70 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_wuying_launch_package.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'USAGE' +Usage: + scripts/real2sim_wuying_launch_package.sh --archive /path/to/real2sim_scene.tar.gz [--dest /tmp/real2sim_import] [--with-gs] + +Extract a generated one-image Real2Sim Motrix package on the Wuying desktop and +launch the visible MotrixSim viewer. +USAGE +} + +archive="" +dest="/tmp/real2sim_import" +with_gs=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --archive) + archive="${2:-}" + shift 2 + ;; + --dest) + dest="${2:-}" + shift 2 + ;; + --with-gs) + with_gs=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage + exit 2 + ;; + esac +done + +if [[ -z "${archive}" || ! -f "${archive}" ]]; then + echo "--archive must point to an existing .tar.gz package" >&2 + exit 2 +fi + +mkdir -p "${dest}" +tar -xzf "${archive}" -C "${dest}" +package_dir="$(find "${dest}" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1)" +if [[ -z "${package_dir}" || ! -f "${package_dir}/configs/go2_real2sim.json" ]]; then + echo "Extracted package missing configs/go2_real2sim.json under ${dest}" >&2 + exit 1 +fi + +recommended_camera="" +if [[ -f "${package_dir}/motrix_import.json" ]]; then + recommended_camera="$(python3 - "${package_dir}/motrix_import.json" <<'PY' || true +import json +import sys +try: + data = json.load(open(sys.argv[1], encoding="utf-8")) +except Exception: + data = {} +print(data.get("recommended_camera", "")) +PY +)" +fi + +viewer_camera="overview" +if [[ "${recommended_camera}" == "source_3dgs_preview" ]]; then + viewer_camera="closeup" +fi + +args=(--config "${package_dir}/configs/go2_real2sim.json" --camera "${viewer_camera}" --gs-width 900 --gs-height 700 --static-gs-preview) +if [[ "${with_gs}" -eq 1 ]]; then + args+=(--show-source-gs) + if [[ "${recommended_camera}" == "source_3dgs_preview" ]]; then + args+=(--source-gs-only) + fi + args=(--with-gs "${args[@]}") +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +runtime_repo="${REAL2SIM_WUYING_RUNTIME_REPO:-$(cd "${script_dir}/.." && pwd)}" +if [[ ! -x "${runtime_repo}/scripts/real2sim_wuying_viewer.sh" || ! -x "${runtime_repo}/.venv/bin/python" ]]; then + for candidate in \ + /opt/gs_playground_real2sim_48q \ + /opt/gs_playground-real2sim_48q \ + /opt/gs_playground \ + /workspace/gs_playground \ + /root/gs_playground; do + if [[ -x "${candidate}/scripts/real2sim_wuying_viewer.sh" && -x "${candidate}/.venv/bin/python" ]]; then + runtime_repo="${candidate}" + break + fi + done +fi +if [[ ! -x "${runtime_repo}/scripts/real2sim_wuying_viewer.sh" || ! -x "${runtime_repo}/.venv/bin/python" ]]; then + echo "Could not find a Real2Sim runtime repo with scripts/real2sim_wuying_viewer.sh and .venv/bin/python." >&2 + echo "Set REAL2SIM_WUYING_RUNTIME_REPO=/path/to/gs_playground on the Wuying desktop." >&2 + exit 1 +fi + +echo "Launching MotrixSim Real2Sim package: ${package_dir}" +echo "Runtime repo: ${runtime_repo}" +cd "${runtime_repo}" +exec scripts/real2sim_wuying_viewer.sh "${args[@]}" diff --git a/vendor/gs_playground/scripts/real2sim_wuying_paper_env.sh b/vendor/gs_playground/scripts/real2sim_wuying_paper_env.sh new file mode 100644 index 0000000..f540d78 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_wuying_paper_env.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Runtime wiring for the 48 GiB Wuying Real2Sim paper-alignment machine. +# +# Usage: +# source scripts/real2sim_wuying_paper_env.sh +# python -m real2sim check --paper-strict --config configs/real2sim.external.example.json --backend external + +set -euo pipefail + +export REAL2SIM_PROJECT_ROOT="${REAL2SIM_PROJECT_ROOT:-/opt/gs_playground_real2sim_48q}" +export REAL2SIM_RUNTIME_ROOT="${REAL2SIM_RUNTIME_ROOT:-/opt/real2sim_paper_runtime}" + +export REAL2SIM_ANYSPLAT_ROOT="${REAL2SIM_ANYSPLAT_ROOT:-${REAL2SIM_RUNTIME_ROOT}/AnySplat}" +export REAL2SIM_SAM2_ROOT="${REAL2SIM_SAM2_ROOT:-${REAL2SIM_RUNTIME_ROOT}/sam2}" +export REAL2SIM_SAM3D_ROOT="${REAL2SIM_SAM3D_ROOT:-${REAL2SIM_RUNTIME_ROOT}/sam-3d-objects}" +export REAL2SIM_SPEEDY_SPLAT_ROOT="${REAL2SIM_SPEEDY_SPLAT_ROOT:-${REAL2SIM_RUNTIME_ROOT}/speedy-splat}" + +export REAL2SIM_ANY_SPLAT_PYTHON="${REAL2SIM_ANY_SPLAT_PYTHON:-${REAL2SIM_RUNTIME_ROOT}/envs/anysplat/bin/python}" +export REAL2SIM_SAM3D_PYTHON="${REAL2SIM_SAM3D_PYTHON:-${REAL2SIM_RUNTIME_ROOT}/envs/sam3d-objects/bin/python}" + +export CUDA_HOME="${CUDA_HOME:-/usr/local/cuda-12.1}" +export CONDA_PREFIX="${CONDA_PREFIX:-${CUDA_HOME}}" +export LIDRA_SKIP_INIT="${LIDRA_SKIP_INIT:-1}" +export PATH="${CUDA_HOME}/bin:${PATH}" +export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}" + +export REAL2SIM_SAM2_CHECKPOINT="${REAL2SIM_SAM2_CHECKPOINT:-${REAL2SIM_SAM2_ROOT}/sam2.1_hiera_large.pt}" +export REAL2SIM_SAM2_MODEL_CFG="${REAL2SIM_SAM2_MODEL_CFG:-configs/sam2.1/sam2.1_hiera_l.yaml}" +export REAL2SIM_SAM3D_CONFIG="${REAL2SIM_SAM3D_CONFIG:-${REAL2SIM_SAM3D_ROOT}/checkpoints/hf/pipeline.yaml}" + +_real2sim_pythonpath_parts=( + "${REAL2SIM_SAM2_ROOT}" + "${REAL2SIM_SAM3D_ROOT}/notebook" + "${REAL2SIM_SAM3D_ROOT}" + "${REAL2SIM_ANYSPLAT_ROOT}" + "${REAL2SIM_SPEEDY_SPLAT_ROOT}" +) +if [[ -n "${PYTHONPATH:-}" ]]; then + _real2sim_pythonpath_parts+=("${PYTHONPATH}") +fi +export PYTHONPATH="$(IFS=:; echo "${_real2sim_pythonpath_parts[*]}")" +unset _real2sim_pythonpath_parts + +export REAL2SIM_SYNTHETIC_MULTIVIEW_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/synthetic_multiview_expand.py --image {image} --output-dir {synthetic_multiview_dir} --frames-dir {synthetic_frames_dir} --prompt {synthetic_multiview_prompt} --mode {synthetic_multiview_mode} --size 1536x1024" +export REAL2SIM_GROUNDING_DINO_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/groundingdino_hf_detect.py --image {image} --prompt {prompt} --output {detections_json} --max-detections 12" +export REAL2SIM_SAM2_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/sam2_segment.py --image {image} --detections {detections_json} --masks-dir {masks_dir} --output {instances_json}" +export REAL2SIM_INSTANCE_REFINEMENT_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/instance_refine.py --instances {instances_json} --output {instances_json} --image {image} --merge-saturated-accessories --accessory-search-margin-ratio 0.25 --accessory-contact-passes 4 --accessory-saturation-threshold 0.45 --accessory-max-area-growth 0.15 --accessory-min-component-area-ratio 0.001 --accessory-max-components 8" +export REAL2SIM_LAMA_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/lama_inpaint.py --image {image} --instances {instances_json} --output {background_image} --prompt {prompt} --work-dir {work_dir} --redetect-command-env REAL2SIM_GROUNDING_DINO_COMMAND --segment-command-env REAL2SIM_SAM2_COMMAND --require-redetect-loop" +export REAL2SIM_DEPTH_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/depth_anything_depth.py --image {background_image} --output {depth_map} --model depth-anything/Depth-Anything-V2-Small-hf" +export REAL2SIM_ANYSPLAT_COMMAND="${REAL2SIM_ANY_SPLAT_PYTHON} scripts/real2sim_external/anysplat_scene.py --image {background_image} --frames-dir {frames_dir} --output-dir {scene_reconstruction_dir} --ply {scene_ply} --mesh {scene_mesh} --cameras {cameras_json} --depth {depth_map} --max-views 8" +export REAL2SIM_SAM3D_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/sam3d_objects_reconstruct.py --image {image} --mask {mask} --instance {instance_json} --output-dir {object_dir} --ply {object_ply} --mesh {object_mesh} --pose {object_pose} --cameras {cameras_json} --match-camera-preprocess --object-centric-preprocess --object-preprocess-size 896 --object-crop-margin 0.12 --canonicalize-gaussian-to-mesh --remove-video-overlay --texture-baking --texture-render-backend gsplat --texture-baking-engine pytorch3d" +export REAL2SIM_POSE_ALIGNMENT_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/depth_mask_pose_align.py --assets {aligned_assets_json} --instances {instances_json} --depth {depth_map} --cameras {cameras_json} --work-dir {work_dir} --output {aligned_assets_json}" +export REAL2SIM_SPEEDY_SPLAT_COMMAND="${REAL2SIM_ANY_SPLAT_PYTHON} scripts/real2sim_external/speedy_splat_prune.py --input {input} --output {output} --image {source_image} --cameras {cameras_json} --keep-ratio {keep_ratio}" diff --git a/vendor/gs_playground/scripts/real2sim_wuying_viewer.sh b/vendor/gs_playground/scripts/real2sim_wuying_viewer.sh new file mode 100755 index 0000000..273a766 --- /dev/null +++ b/vendor/gs_playground/scripts/real2sim_wuying_viewer.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# Launch the generated Real2Sim viewer on the visible Wuying desktop. + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +foreground=0 +with_gs=0 +force_render_app=0 +package_dir="${REAL2SIM_WUYING_PACKAGE:-}" +config_path="${REAL2SIM_WUYING_CONFIG:-}" +scene_path="${REAL2SIM_WUYING_SCENE:-}" +while [[ "$#" -gt 0 ]]; do + case "${1}" in + --foreground) + foreground=1 + shift + ;; + --with-gs) + with_gs=1 + force_render_app=1 + shift + ;; + --render-app) + force_render_app=1 + shift + ;; + --native) + force_render_app=0 + shift + ;; + --package) + package_dir="${2:?--package requires a package directory}" + shift 2 + ;; + --config) + config_path="${2:?--config requires a config JSON path}" + force_render_app=1 + shift 2 + ;; + --scene) + scene_path="${2:?--scene requires an MJCF scene path}" + shift 2 + ;; + --stop) + pkill -f "real2sim_scene_viewer[.]py" || true + pkill -f "motrixsim[.]interactive_viewer" || true + echo "Stopped Real2Sim viewer processes." + exit 0 + ;; + *) + break + ;; + esac +done + +unset PYTHONPATH +export SDL_AUDIODRIVER="${SDL_AUDIODRIVER:-dummy}" +export ALSOFT_DRIVERS="${ALSOFT_DRIVERS:-null}" +export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-8.9}" +if [[ -z "${XDG_RUNTIME_DIR:-}" && -d "/run/user/$(id -u)" ]]; then + export XDG_RUNTIME_DIR="/run/user/$(id -u)" +fi + +choose_display() { + local first_ok="" + local candidate + local seen=" " + for candidate in "${DISPLAY:-}" :1 :92; do + [[ -n "${candidate}" ]] || continue + [[ "${seen}" == *" ${candidate} "* ]] && continue + seen+="${candidate} " + DISPLAY="${candidate}" timeout 2s xdpyinfo >/dev/null 2>&1 || continue + if [[ -z "${first_ok}" ]]; then + first_ok="${candidate}" + fi + if DISPLAY="${candidate}" timeout 2s xwininfo -root -tree 2>/dev/null | grep -Eq 'gnome-shell|gnome-terminal|MotrixRender'; then + printf '%s\n' "${candidate}" + return 0 + fi + done + if [[ -n "${first_ok}" ]]; then + printf '%s\n' "${first_ok}" + return 0 + fi + printf '%s\n' "${DISPLAY:-:1}" +} + +export DISPLAY="$(choose_display)" +echo "Launching Real2Sim viewer on DISPLAY=${DISPLAY}" + +resolve_latest_package() { + find outputs -path "*/mjcf/scene.xml" -type f -printf '%T@ %h\n' 2>/dev/null \ + | sort -nr \ + | head -1 \ + | awk '{print $2}' \ + | sed 's#/mjcf$##' +} + +if [[ -z "${package_dir}" && ( -z "${scene_path}" || -z "${config_path}" ) ]]; then + package_dir="$(resolve_latest_package || true)" +fi +if [[ -n "${package_dir}" ]]; then + if [[ -z "${scene_path}" ]]; then + scene_path="${package_dir%/}/mjcf/scene.xml" + fi + if [[ -z "${config_path}" ]]; then + config_path="$(find "${package_dir%/}/configs" -maxdepth 1 -name '*_real2sim.json' -type f 2>/dev/null | sort | head -1 || true)" + fi +fi + +if [[ "$#" -eq 0 ]]; then + if [[ "${force_render_app}" -eq 0 ]]; then + if [[ -z "${scene_path}" ]]; then + echo "No scene path found. Pass --scene or --package." >&2 + exit 1 + fi + set -- \ + --file="${scene_path}" + elif [[ "${with_gs}" -eq 1 ]]; then + if [[ -z "${config_path}" ]]; then + echo "No Real2Sim config found. Pass --config or --package." >&2 + exit 1 + fi + set -- \ + --config "${config_path}" \ + --camera overview \ + --gs-width 320 \ + --gs-height 210 \ + --gs-widget-left 930 \ + --gs-widget-top 10 \ + --gs-render-fps 8 \ + --static-gs-preview \ + --viewer-fps 20 \ + --frame-sleep-ms 30 \ + --show-source-gs \ + --show-system-gs + else + if [[ -n "${config_path}" ]]; then + set -- \ + --config "${config_path}" \ + --camera system \ + --no-gs \ + --viewer-fps 12 \ + --frame-sleep-ms 55 + elif [[ -n "${scene_path}" ]]; then + set -- \ + --scene "${scene_path}" \ + --camera system \ + --no-gs \ + --viewer-fps 12 \ + --frame-sleep-ms 55 + else + echo "No Real2Sim config or scene found. Pass --config, --scene, or --package." >&2 + exit 1 + fi + fi +else + force_render_app=1 + parsed_args=() + if [[ -n "${config_path}" ]]; then + parsed_args+=(--config "${config_path}") + elif [[ -n "${scene_path}" ]]; then + parsed_args+=(--scene "${scene_path}") + fi + set -- "${parsed_args[@]}" "$@" +fi + +pkill -f "real2sim_scene_viewer[.]py" || true +pkill -f "motrixsim[.]interactive_viewer" || true + +if [[ "${foreground}" -eq 1 ]]; then + if [[ "${force_render_app}" -eq 0 ]]; then + exec .venv/bin/python -m motrixsim.interactive_viewer "$@" + fi + exec .venv/bin/python scripts/real2sim_scene_viewer.py "$@" +fi + +log_path="${REAL2SIM_WUYING_VIEWER_LOG:-/tmp/real2sim_wuying_viewer.log}" +rm -f "${log_path}" +if [[ "${force_render_app}" -eq 0 ]]; then + cmd=(.venv/bin/python -m motrixsim.interactive_viewer "$@") +else + cmd=(.venv/bin/python scripts/real2sim_scene_viewer.py "$@") +fi +printf 'Command:' >"${log_path}" +printf ' %q' "${cmd[@]}" >>"${log_path}" +printf '\n' >>"${log_path}" +setsid "${cmd[@]}" >>"${log_path}" 2>&1 < /dev/null & +pid=$! +echo "Started Real2Sim viewer PID=${pid}" +echo "Log: ${log_path}" +sleep 2 +if ! ps -p "${pid}" >/dev/null 2>&1; then + echo "Viewer exited early; last log lines:" >&2 + tail -80 "${log_path}" >&2 || true + exit 1 +fi