feat: productize Real2Sim for isolated POC cluster
This commit is contained in:
parent
ad41f0fe72
commit
d3c5b397d7
11
.dockerignore
Normal file
11
.dockerignore
Normal file
@ -0,0 +1,11 @@
|
||||
.git
|
||||
.venv
|
||||
.pytest_cache
|
||||
workspace
|
||||
acceptance_runs
|
||||
qa
|
||||
docs
|
||||
tests
|
||||
**/__pycache__
|
||||
**/*.pyc
|
||||
*.tgz
|
||||
324
deploy/poc/app-template.yaml
Normal file
324
deploy/poc/app-template.yaml
Normal file
@ -0,0 +1,324 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: real2sim-runtime-config
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
data:
|
||||
config.toml: |
|
||||
app_root = "/app"
|
||||
workspace_dir = "/var/lib/real2sim-poc/workspace"
|
||||
host = "0.0.0.0"
|
||||
port = 18126
|
||||
default_renderer = "spark"
|
||||
real2sim2model_root = "/app"
|
||||
expert_policy_root = "/app"
|
||||
training_root = "/app"
|
||||
inference_root = "/app"
|
||||
discoverse_root = "/opt/DISCOVERSE"
|
||||
gs_playground_root = "/opt/gs_playground"
|
||||
images23dgs_root = "/opt/images23dgs"
|
||||
colmap_binary = "/usr/bin/colmap"
|
||||
gsplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python"
|
||||
runtime_root = "/opt/real2sim_paper_runtime"
|
||||
anysplat_root = "/opt/real2sim_paper_runtime/AnySplat"
|
||||
anysplat_python = "/opt/real2sim_paper_runtime/envs/anysplat/bin/python"
|
||||
cuda_home = "/usr/local/cuda"
|
||||
runtime_env_script = "/opt/gs_playground/scripts/real2sim_wuying_paper_env.sh"
|
||||
allowed_path_roots = "/var/lib/real2sim-poc:/opt/DISCOVERSE:/opt/gs_playground:/opt/motrixsim-docs:/opt/real2sim_paper_runtime:/tmp"
|
||||
production_mode = true
|
||||
auth_enabled = true
|
||||
secure_cookies = true
|
||||
session_ttl_seconds = 43200
|
||||
discoverse_commit = "d67f47c084aba0e0cf422a8725235f8b9238655a"
|
||||
act_commit = "f8ab39c8d829d85571ab6d0250412ba4d0698fa0"
|
||||
anysplat_commit = "5f5e208a7dd57d52e43ea0d553a95eab526e8775"
|
||||
anysplat_model_revision = "d2e8c343672646041ad4ea518184968f94362f01"
|
||||
vggt_model_revision = "860abec7937da0a4c03c41d3c269c366e82abdf9"
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: real2sim-api
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-api
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/version: "__BUILD_ID__"
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-api
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/version: "__BUILD_ID__"
|
||||
spec:
|
||||
serviceAccountName: real2sim-poc
|
||||
automountServiceAccountToken: false
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: "192.168.80.136"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
imagePullSecrets:
|
||||
- name: real2sim-registry
|
||||
containers:
|
||||
- name: api
|
||||
image: __API_IMAGE__
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- {name: REAL2SIM_PLATFORM_CONFIG, value: /etc/real2sim/config.toml}
|
||||
- {name: REAL2SIM_WORKER_MODE, value: api}
|
||||
- {name: REAL2SIM_WORKER_ID, value: real2sim-worker-a800}
|
||||
- {name: REAL2SIM_PRODUCTION_MODE, value: "1"}
|
||||
- {name: REAL2SIM_AUTH_ENABLED, value: "1"}
|
||||
- {name: REAL2SIM_SECURE_COOKIES, value: "1"}
|
||||
- {name: REAL2SIM_BOS_PREFIX, value: real2sim-poc}
|
||||
- {name: BOS_BUCKET, value: testxyz}
|
||||
- {name: BOS_ENDPOINT, value: https://bj.bcebos.com}
|
||||
- name: BOS_ACCESS_KEY_ID
|
||||
valueFrom: {secretKeyRef: {name: real2sim-bos, key: accessKeyId}}
|
||||
- name: BOS_SECRET_ACCESS_KEY
|
||||
valueFrom: {secretKeyRef: {name: real2sim-bos, key: secretAccessKey}}
|
||||
- name: REAL2SIM_BOOTSTRAP_ADMIN_PASSWORD
|
||||
valueFrom: {secretKeyRef: {name: real2sim-app-secrets, key: bootstrap-admin-password}}
|
||||
ports:
|
||||
- {name: http, containerPort: 18126}
|
||||
resources:
|
||||
requests: {cpu: 500m, memory: 1Gi}
|
||||
limits: {cpu: "4", memory: 8Gi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
readinessProbe:
|
||||
httpGet: {path: /api/health, port: http}
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet: {path: /api/health, port: http}
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 20
|
||||
volumeMounts:
|
||||
- {name: data, mountPath: /var/lib/real2sim-poc}
|
||||
- {name: config, mountPath: /etc/real2sim/config.toml, subPath: config.toml, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim: {claimName: real2sim-poc-data}
|
||||
- name: config
|
||||
configMap: {name: real2sim-runtime-config}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 2Gi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: real2sim-api
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-api
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: real2sim-api
|
||||
ports:
|
||||
- {name: http, port: 18126, targetPort: http}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: real2sim-worker
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-worker
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/version: "__BUILD_ID__"
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: {type: Recreate}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-worker
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/version: "__BUILD_ID__"
|
||||
spec:
|
||||
serviceAccountName: real2sim-poc
|
||||
automountServiceAccountToken: false
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: "192.168.80.136"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
imagePullSecrets:
|
||||
- name: real2sim-registry
|
||||
initContainers:
|
||||
- name: runtime-gate
|
||||
image: __WORKER_IMAGE__
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [real2sim-runtime-gate, --mode, bootstrap]
|
||||
env: &worker-env
|
||||
- {name: REAL2SIM_PLATFORM_CONFIG, value: /etc/real2sim/config.toml}
|
||||
- {name: REAL2SIM_WORKER_MODE, value: worker}
|
||||
- {name: REAL2SIM_WORKER_ID, value: real2sim-worker-a800}
|
||||
- {name: REAL2SIM_PRODUCTION_MODE, value: "1"}
|
||||
- {name: REAL2SIM_BOS_PREFIX, value: real2sim-poc}
|
||||
- {name: REAL2SIM_RUNTIME_GATE_DIR, value: /var/lib/real2sim-poc/cache/runtime-gate}
|
||||
- {name: HF_HOME, value: /var/lib/real2sim-poc/cache/huggingface}
|
||||
- {name: HF_HUB_DISABLE_XET, value: "1"}
|
||||
- {name: BOS_BUCKET, value: testxyz}
|
||||
- {name: BOS_ENDPOINT, value: https://bj.bcebos.com}
|
||||
- name: BOS_ACCESS_KEY_ID
|
||||
valueFrom: {secretKeyRef: {name: real2sim-bos, key: accessKeyId}}
|
||||
- name: BOS_SECRET_ACCESS_KEY
|
||||
valueFrom: {secretKeyRef: {name: real2sim-bos, key: secretAccessKey}}
|
||||
resources:
|
||||
requests: {cpu: "6", memory: 32Gi, nvidia.com/gpu: "1"}
|
||||
limits: {cpu: "24", memory: 96Gi, nvidia.com/gpu: "1"}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts: &worker-mounts
|
||||
- {name: data, mountPath: /var/lib/real2sim-poc}
|
||||
- {name: config, mountPath: /etc/real2sim/config.toml, subPath: config.toml, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
- {name: shm, mountPath: /dev/shm}
|
||||
containers:
|
||||
- name: worker
|
||||
image: __WORKER_IMAGE__
|
||||
imagePullPolicy: IfNotPresent
|
||||
env: *worker-env
|
||||
resources:
|
||||
requests: {cpu: "6", memory: 32Gi, nvidia.com/gpu: "1"}
|
||||
limits: {cpu: "24", memory: 96Gi, nvidia.com/gpu: "1"}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
readinessProbe:
|
||||
exec: {command: [real2sim-runtime-gate, --mode, readiness]}
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 15
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 4
|
||||
livenessProbe:
|
||||
exec: {command: [real2sim-runtime-gate, --mode, readiness]}
|
||||
initialDelaySeconds: 45
|
||||
periodSeconds: 60
|
||||
timeoutSeconds: 15
|
||||
failureThreshold: 3
|
||||
volumeMounts: *worker-mounts
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim: {claimName: real2sim-poc-data}
|
||||
- name: config
|
||||
configMap: {name: real2sim-runtime-config}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 12Gi}
|
||||
- name: shm
|
||||
emptyDir: {medium: Memory, sizeLimit: 12Gi}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: real2sim-frontend
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-frontend
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/version: "__BUILD_ID__"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-frontend
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-frontend
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/version: "__BUILD_ID__"
|
||||
spec:
|
||||
serviceAccountName: real2sim-poc
|
||||
automountServiceAccountToken: false
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: In
|
||||
values: ["192.168.80.249", "192.168.81.47"]
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 101
|
||||
runAsGroup: 101
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
imagePullSecrets:
|
||||
- name: real2sim-registry
|
||||
containers:
|
||||
- name: frontend
|
||||
image: __FRONTEND_IMAGE__
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- {name: http, containerPort: 8080}
|
||||
- {name: https, containerPort: 8443}
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 128Mi}
|
||||
limits: {cpu: "1", memory: 512Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
readinessProbe:
|
||||
tcpSocket: {port: https}
|
||||
initialDelaySeconds: 3
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
tcpSocket: {port: https}
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
volumeMounts:
|
||||
- {name: tls, mountPath: /etc/real2sim/tls, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
- {name: nginx-cache, mountPath: /var/cache/nginx}
|
||||
volumes:
|
||||
- name: tls
|
||||
secret: {secretName: real2sim-tls}
|
||||
- name: tmp
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
- name: nginx-cache
|
||||
emptyDir: {sizeLimit: 128Mi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: real2sim-frontend
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/name: real2sim-frontend
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
selector:
|
||||
app.kubernetes.io/name: real2sim-frontend
|
||||
ports:
|
||||
- {name: http, port: 80, targetPort: http}
|
||||
- {name: https, port: 443, targetPort: https}
|
||||
184
deploy/poc/bootstrap.yaml
Normal file
184
deploy/poc/bootstrap.yaml
Normal file
@ -0,0 +1,184 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: real2sim-poc-local-pv
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
capacity:
|
||||
storage: 150Gi
|
||||
volumeMode: Filesystem
|
||||
accessModes: [ReadWriteOnce]
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: real2sim-poc-local
|
||||
local:
|
||||
path: /var/lib/real2sim-poc
|
||||
nodeAffinity:
|
||||
required:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/hostname
|
||||
operator: In
|
||||
values: ["192.168.80.136"]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: real2sim-poc-data
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: real2sim-poc-local
|
||||
volumeName: real2sim-poc-local-pv
|
||||
resources:
|
||||
requests:
|
||||
storage: 150Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ResourceQuota
|
||||
metadata:
|
||||
name: real2sim-poc-quota
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
hard:
|
||||
pods: "12"
|
||||
services: "6"
|
||||
services.loadbalancers: "1"
|
||||
requests.cpu: "40"
|
||||
limits.cpu: "64"
|
||||
requests.memory: 128Gi
|
||||
limits.memory: 192Gi
|
||||
requests.storage: 150Gi
|
||||
requests.nvidia.com/gpu: "1"
|
||||
limits.nvidia.com/gpu: "1"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: LimitRange
|
||||
metadata:
|
||||
name: real2sim-poc-limits
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
limits:
|
||||
- type: Container
|
||||
defaultRequest:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
default:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: real2sim-poc
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
automountServiceAccountToken: false
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: default-deny
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Ingress, Egress]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-dns
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
ports:
|
||||
- {protocol: UDP, port: 53}
|
||||
- {protocol: TCP, port: 53}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: frontend-policy
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-frontend
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- ports:
|
||||
- {protocol: TCP, port: 8080}
|
||||
- {protocol: TCP, port: 8443}
|
||||
egress:
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-api
|
||||
ports:
|
||||
- {protocol: TCP, port: 18126}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: api-policy
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-api
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-frontend
|
||||
ports:
|
||||
- {protocol: TCP, port: 18126}
|
||||
egress:
|
||||
- ports:
|
||||
- {protocol: TCP, port: 443}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: worker-policy
|
||||
namespace: real2sim-poc
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: real2sim-worker
|
||||
policyTypes: [Ingress, Egress]
|
||||
egress:
|
||||
- ports:
|
||||
- {protocol: TCP, port: 443}
|
||||
- {protocol: TCP, port: 80}
|
||||
82
deploy/poc/nginx.conf
Normal file
82
deploy/poc/nginx.conf
Normal file
@ -0,0 +1,82 @@
|
||||
worker_processes auto;
|
||||
pid /tmp/nginx.pid;
|
||||
|
||||
events { worker_connections 1024; }
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
access_log /dev/stdout;
|
||||
error_log /dev/stderr warn;
|
||||
sendfile on;
|
||||
server_tokens off;
|
||||
client_max_body_size 70m;
|
||||
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
upstream real2sim_api { server real2sim-api.real2sim-poc.svc.cluster.local:18126; }
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
return 308 https://$host$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8443 ssl;
|
||||
ssl_certificate /etc/real2sim/tls/tls.crt;
|
||||
ssl_certificate_key /etc/real2sim/tls/tls.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_session_cache shared:TLS:10m;
|
||||
add_header X-Content-Type-Options nosniff always;
|
||||
add_header Referrer-Policy same-origin always;
|
||||
add_header X-Frame-Options SAMEORIGIN always;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
location = /api/events/tasks {
|
||||
proxy_pass http://real2sim_api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_read_timeout 1h;
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://real2sim_api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_request_buffering off;
|
||||
proxy_read_timeout 7200s;
|
||||
}
|
||||
|
||||
location /datasets/ {
|
||||
proxy_pass http://real2sim_api;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location ~ ^/static/.*\.(html|js|css)$ {
|
||||
try_files $uri =404;
|
||||
expires -1;
|
||||
add_header Cache-Control "no-store";
|
||||
}
|
||||
|
||||
location /static/ {
|
||||
try_files $uri =404;
|
||||
expires 1h;
|
||||
add_header Cache-Control "public, max-age=3600";
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
add_header Cache-Control "no-store";
|
||||
}
|
||||
}
|
||||
}
|
||||
45
docker/anysplat-runtime-requirements.txt
Normal file
45
docker/anysplat-runtime-requirements.txt
Normal file
@ -0,0 +1,45 @@
|
||||
numpy==1.25.0
|
||||
wheel
|
||||
tqdm
|
||||
lightning
|
||||
hydra-core
|
||||
jaxtyping
|
||||
beartype
|
||||
einops
|
||||
colorama
|
||||
scikit-image
|
||||
colorspacious
|
||||
matplotlib
|
||||
moviepy==1.0.3
|
||||
imageio
|
||||
timm
|
||||
dacite
|
||||
lpips
|
||||
e3nn
|
||||
plyfile
|
||||
tabulate
|
||||
svg.py
|
||||
scikit-video
|
||||
opencv-python-headless==4.8.1.78
|
||||
Pillow
|
||||
huggingface_hub
|
||||
pydantic
|
||||
open3d
|
||||
safetensors
|
||||
omegaconf
|
||||
pyquaternion
|
||||
pyyaml
|
||||
packaging
|
||||
h5py
|
||||
dm_env
|
||||
OneEuroFilter
|
||||
pymeshlab
|
||||
scipy
|
||||
psutil
|
||||
screeninfo
|
||||
mediapy
|
||||
termcolor
|
||||
imagecodecs
|
||||
av
|
||||
wandb
|
||||
trimesh
|
||||
18
docker/api.Dockerfile
Normal file
18
docker/api.Dockerfile
Normal file
@ -0,0 +1,18 @@
|
||||
FROM python:3.12-slim-bookworm
|
||||
|
||||
ARG BUILD_ID=dev
|
||||
ARG SOURCE_COMMIT=unknown
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \
|
||||
REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT}
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl ffmpeg libglib2.0-0 && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY src ./src
|
||||
RUN python -m pip install --no-cache-dir --upgrade pip && python -m pip install --no-cache-dir .
|
||||
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin real2sim
|
||||
USER 10001:10001
|
||||
EXPOSE 18126
|
||||
CMD ["real2sim-platform", "--host", "0.0.0.0", "--port", "18126"]
|
||||
10
docker/frontend.Dockerfile
Normal file
10
docker/frontend.Dockerfile
Normal file
@ -0,0 +1,10 @@
|
||||
FROM nginxinc/nginx-unprivileged:1.27-alpine
|
||||
|
||||
ARG BUILD_ID=dev
|
||||
LABEL org.opencontainers.image.title="Real2Sim POC frontend" \
|
||||
org.opencontainers.image.version="${BUILD_ID}"
|
||||
COPY deploy/poc/nginx.conf /etc/nginx/nginx.conf
|
||||
COPY src/real2sim_platform/static/index.html /usr/share/nginx/html/index.html
|
||||
COPY src/real2sim_platform/static /usr/share/nginx/html/static
|
||||
USER 101:101
|
||||
EXPOSE 8080 8443
|
||||
2
docker/worker-constraints.txt
Normal file
2
docker/worker-constraints.txt
Normal file
@ -0,0 +1,2 @@
|
||||
numpy==1.25.0
|
||||
opencv-python-headless==4.8.1.78
|
||||
72
docker/worker.Dockerfile
Normal file
72
docker/worker.Dockerfile
Normal file
@ -0,0 +1,72 @@
|
||||
FROM pytorch/pytorch:2.2.0-cuda12.1-cudnn8-runtime
|
||||
|
||||
ARG BUILD_ID=dev
|
||||
ARG SOURCE_COMMIT=unknown
|
||||
ARG ANYSPLAT_COMMIT=5f5e208a7dd57d52e43ea0d553a95eab526e8775
|
||||
ARG DISCOVERSE_COMMIT=d67f47c084aba0e0cf422a8725235f8b9238655a
|
||||
ARG ACT_COMMIT=f8ab39c8d829d85571ab6d0250412ba4d0698fa0
|
||||
ARG MOTRIXSIM_DOCS_COMMIT=2beecea67c81d25e29cefa30a80c0ddad7b44dde
|
||||
ARG VGGT_MODEL_REVISION=860abec7937da0a4c03c41d3c269c366e82abdf9
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \
|
||||
REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT} \
|
||||
REAL2SIM_VGGT_MODEL_REVISION=${VGGT_MODEL_REVISION} \
|
||||
REAL2SIM_ANYSPLAT_ROOT=/opt/real2sim_paper_runtime/AnySplat \
|
||||
REAL2SIM_ANY_SPLAT_PYTHON=/opt/real2sim_paper_runtime/envs/anysplat/bin/python \
|
||||
REAL2SIM_POSE_INIT_TRAIN_PYTHON=/opt/real2sim_paper_runtime/envs/anysplat/bin/python \
|
||||
CUDA_HOME=/usr/local/cuda \
|
||||
PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat:/opt/DISCOVERSE/policies/act:/opt/DISCOVERSE:/opt/gs_playground
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl ffmpeg git build-essential cmake ninja-build pkg-config colmap \
|
||||
libgl1 libglib2.0-0 libgomp1 libx11-6 libxext6 libsm6 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY src ./src
|
||||
COPY scripts ./scripts
|
||||
COPY vendor/gs_playground /opt/gs_playground
|
||||
COPY docker/anysplat-runtime-requirements.txt /tmp/anysplat-runtime-requirements.txt
|
||||
COPY docker/worker-constraints.txt /tmp/worker-constraints.txt
|
||||
|
||||
RUN python -m pip install --no-cache-dir --upgrade pip setuptools wheel && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt . && \
|
||||
python -m pip install --no-cache-dir torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121 && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt -r /tmp/anysplat-runtime-requirements.txt && \
|
||||
python -m pip install --no-cache-dir xformers==0.0.24 --no-deps && \
|
||||
python -m pip install --no-cache-dir torch-scatter==2.1.2 -f https://data.pyg.org/whl/torch-2.2.0+cu121.html && \
|
||||
python -m pip install --no-cache-dir 'https://github.com/nerfstudio-project/gsplat/releases/download/v1.4.0/gsplat-1.4.0%2Bpt22cu121-cp310-cp310-linux_x86_64.whl' && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt motrixsim-core==0.8.0
|
||||
|
||||
RUN git clone --filter=blob:none --no-checkout https://github.com/InternRobotics/AnySplat.git /opt/real2sim_paper_runtime/AnySplat && \
|
||||
git -C /opt/real2sim_paper_runtime/AnySplat checkout "${ANYSPLAT_COMMIT}" && \
|
||||
test "$(git -C /opt/real2sim_paper_runtime/AnySplat rev-parse HEAD)" = "${ANYSPLAT_COMMIT}" && \
|
||||
sed -i 's/VGGT.from_pretrained("facebook\/VGGT-1B")/VGGT.from_pretrained("facebook\/VGGT-1B", revision=os.environ.get("REAL2SIM_VGGT_MODEL_REVISION"))/g' /opt/real2sim_paper_runtime/AnySplat/src/model/encoder/anysplat.py
|
||||
|
||||
RUN git clone --filter=blob:none --no-checkout https://github.com/eust-w/DISCOVERSE.git /opt/DISCOVERSE && \
|
||||
git -C /opt/DISCOVERSE checkout "${DISCOVERSE_COMMIT}" && \
|
||||
git -C /opt/DISCOVERSE submodule update --init policies/act && \
|
||||
test "$(git -C /opt/DISCOVERSE rev-parse HEAD)" = "${DISCOVERSE_COMMIT}" && \
|
||||
test "$(git -C /opt/DISCOVERSE/policies/act rev-parse HEAD)" = "${ACT_COMMIT}" && \
|
||||
python -m pip install --no-cache-dir --no-deps -e /opt/DISCOVERSE
|
||||
|
||||
RUN git clone --filter=blob:none --no-checkout https://github.com/Motphys/motrixsim-docs.git /opt/motrixsim-docs && \
|
||||
git -C /opt/motrixsim-docs checkout "${MOTRIXSIM_DOCS_COMMIT}" && \
|
||||
test "$(git -C /opt/motrixsim-docs rev-parse HEAD)" = "${MOTRIXSIM_DOCS_COMMIT}" && \
|
||||
mkdir -p /opt/real2sim_paper_runtime/envs/anysplat/bin && \
|
||||
ln -s /opt/conda/bin/python /opt/real2sim_paper_runtime/envs/anysplat/bin/python && \
|
||||
ln -s /opt/conda/bin/pip /opt/real2sim_paper_runtime/envs/anysplat/bin/pip
|
||||
|
||||
RUN python -c 'import mujoco, motrixsim, torch; from huggingface_hub import snapshot_download; assert torch.__version__.startswith("2.2")' && \
|
||||
PYTHONPATH=/opt/DISCOVERSE/policies/act:/opt/DISCOVERSE python -c 'from policy import ACTPolicy; assert ACTPolicy is not None' && \
|
||||
PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat python -c 'from src.model.model.anysplat import AnySplat; assert AnySplat is not None' && \
|
||||
test -f /opt/gs_playground/scripts/real2sim_video_motrix_scene.py
|
||||
|
||||
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin real2sim && \
|
||||
chown -R 10001:10001 /app
|
||||
USER 10001:10001
|
||||
CMD ["real2sim-worker"]
|
||||
45
docs/POC_PRODUCT_SPEC.md
Normal file
45
docs/POC_PRODUCT_SPEC.md
Normal file
@ -0,0 +1,45 @@
|
||||
# Real2Sim POC 产品与交互规格
|
||||
|
||||
## 用户与边界
|
||||
|
||||
| 角色 | 主要目标 | 写权限 |
|
||||
| --- | --- | --- |
|
||||
| 管理员 | 初始化平台、管理用户、存储和删除策略 | 全部;删除磁盘或 BOS 对象必须二次确认 |
|
||||
| 算法工程师 | 导入数据、重建、采集、训练和推理 | 业务资源和任务,不可管理用户 |
|
||||
| 只读用户 | 查看场景、报告、模型和 rollout | 无 |
|
||||
|
||||
平台面向内部 POC/Beta,不开放注册、计费或多组织租户。真实 MuJoCo 和 MotrixSim 是允许的领域运行时;禁止 mock、dry-run 验收、硬编码成功率、假进度和通过直接改物体位姿伪造抓取。
|
||||
|
||||
## 完整用户旅程
|
||||
|
||||
1. 管理员使用一次性初始密码登录,强制更新密码,再创建工程师或只读账号。
|
||||
2. 工程师从 HuggingFace、浏览器分片上传或授权 BOS 前缀导入数据。系统计算 SHA-256、创建版本和清单。
|
||||
3. 视频或图片包进入“准备输入”,统一生成 prepared frames;质量预检给出帧数、分辨率、清晰度、视差和可执行建议。
|
||||
4. 场景重建向导选择 prepared dataset、质量档位、坐标系、渲染器和仿真器。任务立即进入本页,SSE 显示真实阶段和进度。
|
||||
5. 重建成功后在场景管理中打开 Spark/Three Viewer,添加机器人或物体会生成新 revision,不覆盖来源场景。
|
||||
6. 数据采集页只展示场景 manifest 声明兼容的真实控制器。RM65 采集要求双指接触、至少抬升 10 cm、放置误差不超过 5 cm。
|
||||
7. 模型训练页只允许新建 DISCOVERSE ACT;配置由 HDF5 和场景 manifest 推导,输出 checkpoint、损失历史、模型卡和血缘。
|
||||
8. 在线推理在真实引擎运行随机化 episode,输出视频、轨迹、动作统计和成功率。没有真实结果时显示空状态。
|
||||
|
||||
## 七页交互合同
|
||||
|
||||
- **总览**:当前闭环阶段、推荐下一步、真实任务、最近场景和模型结果。无运维状态条或调试 JSON。
|
||||
- **数据管理**:来源卡打开导入弹窗;表格支持全选、多选、批量准备、预检和删除索引。
|
||||
- **场景重建**:资源卡选择、质量评分、参数折叠、任务列表和完成后预览构成单页闭环。
|
||||
- **场景管理**:场景库、3D Viewer、资产和仿真信息三栏;移动端按 Viewer、场景库、资产编辑顺序堆叠。
|
||||
- **数据采集**:场景、控制器和采集规模在同页;任务完成后直接出现 HDF5 数据集。
|
||||
- **模型训练**:采集数据、ACT 配方、真实指标、训练任务和模型卡同页呈现。
|
||||
- **在线推理**:模型和场景卡选择、rollout 播放、轨迹产物与任务管理同页呈现。
|
||||
|
||||
任务详情使用右侧抽屉展示阶段、报告、建议、产物预览和下载;高级 JSON 仅在折叠区。删除默认只删平台索引,不删 PV 或 BOS 对象。
|
||||
|
||||
## 可行性证据与交付门槛
|
||||
|
||||
- 本地已用真实 MuJoCo 和 MotrixSim 加载同一 RM65 MJCF 并执行 step。
|
||||
- 本地真实物理控制器已完成双指接触抓放:抬升 15.57 cm,放置误差 1.29 cm,输出 RGB/qpos/action/contact HDF5 与 MP4。
|
||||
- 集群 worker 必须在 A800 上通过 CUDA 矩阵运算、AnySplat 固定 revision 模型加载、ACT import、MuJoCo step 和 MotrixSim step,之后才 Ready。
|
||||
- 最终 `acceptance_report.json.pass=true` 还要求两次非 dry-run 重建、50 个成功采集 episode、真实 ACT checkpoint,以及 10 次推理成功率至少 50%。未达门槛时可保留 Beta 部署,但不得声明全链路通过。
|
||||
|
||||
## 视觉合同
|
||||
|
||||
视觉原型保存于 `docs/prototypes/real2sim-poc-product-board-v1.png`。实现采用深色中性底、青绿色命令色、紧凑表格、6px 面板圆角和纯文字导航;禁止大编号、运行时状态条、装饰图标、裸资源 ID/路径主输入及无数据时的虚构图表。
|
||||
@ -8,6 +8,8 @@ version = "0.1.0"
|
||||
description = "Real2Sim scene, collection, training, and inference management platform"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"argon2-cffi>=23.1",
|
||||
"bce-python-sdk>=0.9.29",
|
||||
"fastapi>=0.110",
|
||||
"huggingface_hub>=0.24",
|
||||
"imageio>=2.34",
|
||||
@ -22,6 +24,8 @@ dependencies = [
|
||||
|
||||
[project.scripts]
|
||||
real2sim-platform = "real2sim_platform.server:main"
|
||||
real2sim-worker = "real2sim_platform.worker_main:main"
|
||||
real2sim-runtime-gate = "real2sim_platform.runtime_gate:main"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
@ -36,3 +40,6 @@ where = ["src"]
|
||||
"static/vendor/three/addons/postprocessing/*",
|
||||
"static/vendor/three/addons/utils/*",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
markers = ["integration: exercises installed real runtimes and physical scene assets"]
|
||||
|
||||
205
scripts/deploy_poc_cluster.sh
Executable file
205
scripts/deploy_poc_cluster.sh
Executable file
@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
KUBECONFIG_PATH="${REAL2SIM_POC_KUBECONFIG:-/Users/d-robotics/Downloads/poc.config}"
|
||||
NAMESPACE="real2sim-poc"
|
||||
GPU_NODE="${REAL2SIM_POC_GPU_NODE:-192.168.80.136}"
|
||||
REGISTRY="${REAL2SIM_POC_REGISTRY:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc}"
|
||||
SOURCE_COMMIT="$(git -C "$ROOT" rev-parse --short=12 HEAD)"
|
||||
BUILD_ID="${REAL2SIM_POC_BUILD_ID:-${SOURCE_COMMIT}-$(date -u +%Y%m%d%H%M%S)}"
|
||||
OUT="${REAL2SIM_POC_ACCEPTANCE_OUT:-$ROOT/acceptance_runs/poc/$BUILD_ID}"
|
||||
BUILD_IMAGES=1
|
||||
DEPLOY_APP=1
|
||||
tls_dir=""
|
||||
context_archive=""
|
||||
rendered_manifest=""
|
||||
trap 'rm -rf "${tls_dir:-}" "${context_archive:-}" "${rendered_manifest:-}"' EXIT
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--skip-build) BUILD_IMAGES=0 ;;
|
||||
--build-only) DEPLOY_APP=0 ;;
|
||||
--help)
|
||||
echo "usage: $0 [--skip-build|--build-only]"
|
||||
exit 0
|
||||
;;
|
||||
*) echo "unknown argument: $arg" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
K=(kubectl --kubeconfig "$KUBECONFIG_PATH")
|
||||
mkdir -p "$OUT"
|
||||
|
||||
require() { command -v "$1" >/dev/null || { echo "missing required command: $1" >&2; exit 1; }; }
|
||||
for command in kubectl jq openssl tar git sed; do require "$command"; done
|
||||
test -f "$KUBECONFIG_PATH" || { echo "kubeconfig not found: $KUBECONFIG_PATH" >&2; exit 1; }
|
||||
test "$(git -C "$ROOT" branch --show-current)" = "feat/poc-cluster-productization" || {
|
||||
echo "deployment is only allowed from feat/poc-cluster-productization" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
gpu_capacity="$("${K[@]}" get node "$GPU_NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}')"
|
||||
gpu_in_use="$("${K[@]}" get pods -A --field-selector "spec.nodeName=$GPU_NODE" -o json | jq '[.items[] | select(.status.phase=="Running" or .status.phase=="Pending") | ([.spec.containers[]?.resources.requests["nvidia.com/gpu"] // "0" | tonumber] | add // 0)] | add // 0')"
|
||||
if (( gpu_capacity - gpu_in_use < 1 )); then
|
||||
echo "A800 node has no free GPU: capacity=$gpu_capacity in_use=$gpu_in_use" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'gpu_capacity=%s\ngpu_in_use=%s\nbuild_id=%s\n' "$gpu_capacity" "$gpu_in_use" "$BUILD_ID" > "$OUT/deploy_preflight.txt"
|
||||
|
||||
"${K[@]}" apply -f "$ROOT/deploy/poc/bootstrap.yaml"
|
||||
|
||||
clone_secret() {
|
||||
local source_namespace="$1" source_name="$2" target_name="$3"
|
||||
"${K[@]}" -n "$source_namespace" get secret "$source_name" -o json |
|
||||
jq --arg ns "$NAMESPACE" --arg name "$target_name" '
|
||||
del(.metadata.uid,.metadata.resourceVersion,.metadata.creationTimestamp,.metadata.managedFields,.metadata.ownerReferences) |
|
||||
.metadata.namespace=$ns | .metadata.name=$name |
|
||||
.metadata.labels={"app.kubernetes.io/part-of":"real2sim-poc"}' |
|
||||
"${K[@]}" apply -f - >/dev/null
|
||||
}
|
||||
|
||||
clone_secret droboticflow droboticflow-registry real2sim-registry
|
||||
clone_secret droboticflow droboticflow-bos-workspace real2sim-bos
|
||||
|
||||
if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets >/dev/null 2>&1; then
|
||||
admin_password="$(openssl rand -base64 30 | tr -d '/+=' | cut -c1-28)Aa1!"
|
||||
session_secret="$(openssl rand -hex 32)"
|
||||
"${K[@]}" -n "$NAMESPACE" create secret generic real2sim-app-secrets \
|
||||
--from-literal=bootstrap-admin-password="$admin_password" \
|
||||
--from-literal=session-secret="$session_secret" \
|
||||
--dry-run=client -o yaml |
|
||||
"${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml |
|
||||
"${K[@]}" apply -f - >/dev/null
|
||||
fi
|
||||
|
||||
if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-tls >/dev/null 2>&1; then
|
||||
tls_dir="$(mktemp -d)"
|
||||
openssl req -x509 -newkey rsa:3072 -sha256 -nodes -days 365 \
|
||||
-subj "/CN=real2sim-poc.local" \
|
||||
-addext "subjectAltName=DNS:real2sim-poc.local,DNS:localhost,IP:127.0.0.1" \
|
||||
-keyout "$tls_dir/tls.key" -out "$tls_dir/tls.crt" >/dev/null 2>&1
|
||||
"${K[@]}" -n "$NAMESPACE" create secret tls real2sim-tls \
|
||||
--key "$tls_dir/tls.key" --cert "$tls_dir/tls.crt" --dry-run=client -o yaml |
|
||||
"${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml |
|
||||
"${K[@]}" apply -f - >/dev/null
|
||||
fi
|
||||
|
||||
"${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets -o jsonpath='{.data.bootstrap-admin-password}' |
|
||||
base64 --decode > "$OUT/bootstrap-admin-password.txt"
|
||||
chmod 600 "$OUT/bootstrap-admin-password.txt"
|
||||
printf 'admin\n' > "$OUT/bootstrap-admin-username.txt"
|
||||
chmod 600 "$OUT/bootstrap-admin-username.txt"
|
||||
|
||||
prepare_context() {
|
||||
context_archive="$(mktemp -t real2sim-poc-context.XXXXXX.tgz)"
|
||||
tar -C "$ROOT" -czf "$context_archive" \
|
||||
--exclude='.git' --exclude='.venv' --exclude='.pytest_cache' --exclude='workspace' \
|
||||
--exclude='acceptance_runs' --exclude='qa' --exclude='docs' --exclude='tests' .
|
||||
local loader="real2sim-context-${BUILD_ID//[^a-zA-Z0-9-]/-}"
|
||||
loader="${loader:0:60}"
|
||||
"${K[@]}" -n "$NAMESPACE" delete pod "$loader" --ignore-not-found --wait=true >/dev/null
|
||||
"${K[@]}" -n "$NAMESPACE" run "$loader" \
|
||||
--image=busybox:1.36 --restart=Never \
|
||||
--labels="app.kubernetes.io/part-of=real2sim-poc,app.kubernetes.io/component=build-context" \
|
||||
--overrides="$(jq -cn --arg node "$GPU_NODE" '{spec:{nodeSelector:{"kubernetes.io/hostname":$node},containers:[{name:"context-loader",image:"busybox:1.36",command:["sh","-c","sleep 3600"],resources:{requests:{cpu:"100m",memory:"64Mi"},limits:{cpu:"500m",memory:"256Mi"}},volumeMounts:[{name:"data",mountPath:"/data"}]}],volumes:[{name:"data",persistentVolumeClaim:{claimName:"real2sim-poc-data"}}]}}')" >/dev/null
|
||||
"${K[@]}" -n "$NAMESPACE" wait --for=condition=Ready "pod/$loader" --timeout=180s
|
||||
"${K[@]}" -n "$NAMESPACE" cp "$context_archive" "$loader:/tmp/context.tgz"
|
||||
"${K[@]}" -n "$NAMESPACE" exec "$loader" -- sh -eu -c \
|
||||
"mkdir -p '/data/build-context/$BUILD_ID' /data/workspace /data/cache /data/scratch; rm -rf '/data/build-context/$BUILD_ID/'*; tar -xzf /tmp/context.tgz -C '/data/build-context/$BUILD_ID'; chown -R 10001:10001 /data/workspace /data/cache /data/scratch; chmod 0770 /data/workspace /data/cache /data/scratch"
|
||||
"${K[@]}" -n "$NAMESPACE" delete pod "$loader" --wait=true >/dev/null
|
||||
}
|
||||
|
||||
build_image() {
|
||||
local component="$1" dockerfile="$2" destination="$3"
|
||||
local job="real2sim-build-${component}-${BUILD_ID//[^a-zA-Z0-9-]/-}"
|
||||
job="${job:0:60}"
|
||||
"${K[@]}" -n "$NAMESPACE" delete job "$job" --ignore-not-found --wait=true >/dev/null
|
||||
cat <<YAML | "${K[@]}" apply -f - >/dev/null
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: $job
|
||||
namespace: $NAMESPACE
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/component: image-build
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: real2sim-poc
|
||||
app.kubernetes.io/component: image-build
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: "$GPU_NODE"
|
||||
containers:
|
||||
- name: kaniko
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
args:
|
||||
- --context=dir:///data/build-context/$BUILD_ID
|
||||
- --dockerfile=/data/build-context/$BUILD_ID/$dockerfile
|
||||
- --destination=$destination
|
||||
- --build-arg=BUILD_ID=$BUILD_ID
|
||||
- --build-arg=SOURCE_COMMIT=$SOURCE_COMMIT
|
||||
- --snapshot-mode=redo
|
||||
- --use-new-run
|
||||
- --cache=true
|
||||
- --cache-repo=$REGISTRY/real2sim-build-cache
|
||||
- --verbosity=info
|
||||
resources:
|
||||
requests: {cpu: "2", memory: 4Gi}
|
||||
limits: {cpu: "16", memory: 32Gi}
|
||||
volumeMounts:
|
||||
- {name: data, mountPath: /data}
|
||||
- {name: docker-config, mountPath: /kaniko/.docker, readOnly: true}
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim: {claimName: real2sim-poc-data}
|
||||
- name: docker-config
|
||||
secret:
|
||||
secretName: real2sim-registry
|
||||
items:
|
||||
- {key: .dockerconfigjson, path: config.json}
|
||||
YAML
|
||||
if ! "${K[@]}" -n "$NAMESPACE" wait --for=condition=complete "job/$job" --timeout=7200s; then
|
||||
"${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log" 2>&1 || true
|
||||
"${K[@]}" -n "$NAMESPACE" describe "job/$job" > "$OUT/build-${component}-describe.txt" 2>&1 || true
|
||||
return 1
|
||||
fi
|
||||
"${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log"
|
||||
"${K[@]}" -n "$NAMESPACE" delete job "$job" --wait=true >/dev/null
|
||||
}
|
||||
|
||||
FRONTEND_IMAGE="$REGISTRY/real2sim-frontend:$BUILD_ID"
|
||||
API_IMAGE="$REGISTRY/real2sim-api:$BUILD_ID"
|
||||
WORKER_IMAGE="$REGISTRY/real2sim-worker:$BUILD_ID"
|
||||
printf '%s\n' "$FRONTEND_IMAGE" "$API_IMAGE" "$WORKER_IMAGE" > "$OUT/images.txt"
|
||||
|
||||
if [[ "$BUILD_IMAGES" == "1" ]]; then
|
||||
prepare_context
|
||||
build_image frontend docker/frontend.Dockerfile "$FRONTEND_IMAGE"
|
||||
build_image api docker/api.Dockerfile "$API_IMAGE"
|
||||
build_image worker docker/worker.Dockerfile "$WORKER_IMAGE"
|
||||
fi
|
||||
|
||||
if [[ "$DEPLOY_APP" == "1" ]]; then
|
||||
rendered_manifest="$(mktemp -t real2sim-poc-manifest.XXXXXX.yaml)"
|
||||
sed \
|
||||
-e "s|__BUILD_ID__|$BUILD_ID|g" \
|
||||
-e "s|__FRONTEND_IMAGE__|$FRONTEND_IMAGE|g" \
|
||||
-e "s|__API_IMAGE__|$API_IMAGE|g" \
|
||||
-e "s|__WORKER_IMAGE__|$WORKER_IMAGE|g" \
|
||||
"$ROOT/deploy/poc/app-template.yaml" > "$rendered_manifest"
|
||||
"${K[@]}" apply -f "$rendered_manifest"
|
||||
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-api --timeout=600s
|
||||
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-frontend --timeout=600s
|
||||
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-worker --timeout=3600s
|
||||
"${K[@]}" -n "$NAMESPACE" get pods,svc,pvc -l app.kubernetes.io/part-of=real2sim-poc -o wide > "$OUT/resources.txt"
|
||||
"${K[@]}" -n "$NAMESPACE" get svc real2sim-frontend -o json > "$OUT/frontend-service.json"
|
||||
fi
|
||||
|
||||
printf 'build_id=%s\nnamespace=%s\ncredentials=%s\n' "$BUILD_ID" "$NAMESPACE" "$OUT/bootstrap-admin-password.txt"
|
||||
@ -51,7 +51,14 @@ def main() -> int:
|
||||
# the runtime copy so pick-place validation measures real bilateral contact.
|
||||
runtime_xml = runtime_xml.replace(
|
||||
'contype="0" conaffinity="0" density="0" group="2"/>',
|
||||
'contype="1" conaffinity="1" density="0" friction="1.2 0.01 0.001" group="3"/>',
|
||||
'contype="1" conaffinity="1" density="0" friction="2.0 0.02 0.002" group="3"/>',
|
||||
)
|
||||
runtime_xml = runtime_xml.replace(
|
||||
'name="left_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.003"',
|
||||
'name="left_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.010"',
|
||||
).replace(
|
||||
'name="right_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.003"',
|
||||
'name="right_finger_pad_z_axis" type="capsule" fromto="-0.021275 0.04375 -0.06 -0.021275 0.04375 0.06"\n size="0.010"',
|
||||
)
|
||||
runtime_rm65.write_text(runtime_xml, encoding="utf-8")
|
||||
|
||||
|
||||
9
scripts/rollback_poc_cluster.sh
Executable file
9
scripts/rollback_poc_cluster.sh
Executable file
@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
KUBECONFIG_PATH="${REAL2SIM_POC_KUBECONFIG:-/Users/d-robotics/Downloads/poc.config}"
|
||||
K=(kubectl --kubeconfig "$KUBECONFIG_PATH" -n real2sim-poc)
|
||||
for deployment in real2sim-frontend real2sim-api real2sim-worker; do
|
||||
"${K[@]}" rollout undo "deployment/$deployment"
|
||||
"${K[@]}" rollout status "deployment/$deployment" --timeout=3600s
|
||||
done
|
||||
echo "rollback completed; PV and BOS objects were preserved"
|
||||
@ -34,12 +34,6 @@ def main() -> int:
|
||||
parser.add_argument("--width", type=int, default=960)
|
||||
parser.add_argument("--height", type=int, default=540)
|
||||
parser.add_argument("--no-render", action="store_true")
|
||||
parser.add_argument(
|
||||
"--grasp-mode",
|
||||
choices=("physical", "kinematic_attachment"),
|
||||
default="physical",
|
||||
help="Use contact physics by default; kinematic attachment is retained only for diagnostics.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
config = load_config()
|
||||
@ -61,7 +55,6 @@ def main() -> int:
|
||||
width=args.width,
|
||||
height=args.height,
|
||||
render=not args.no_render,
|
||||
grasp_mode=args.grasp_mode,
|
||||
)
|
||||
|
||||
artifacts = {}
|
||||
@ -81,7 +74,6 @@ def run_rollout(
|
||||
width: int,
|
||||
height: int,
|
||||
render: bool,
|
||||
grasp_mode: str = "physical",
|
||||
) -> dict[str, Any]:
|
||||
import mujoco
|
||||
|
||||
@ -105,7 +97,7 @@ def run_rollout(
|
||||
qpos_adrs = [int(model.jnt_qposadr[jid]) for jid in joint_ids]
|
||||
dof_adrs = [int(model.jnt_dofadr[jid]) for jid in joint_ids]
|
||||
|
||||
duration_scale = 1 if grasp_mode == "kinematic_attachment" else 8
|
||||
duration_scale = 8
|
||||
phases = [
|
||||
("home", np.array([0.42, 0.00, 0.72]), 0.0, 35 * duration_scale, False),
|
||||
("approach", np.array([0.71, 0.015, 0.42]), 0.0, 55 * duration_scale, False),
|
||||
@ -123,7 +115,6 @@ def run_rollout(
|
||||
if render:
|
||||
renderer = mujoco.Renderer(model, height=height, width=width)
|
||||
|
||||
attached = False
|
||||
contact_steps = 0
|
||||
bilateral_contact_steps = 0
|
||||
phase_contacts = {phase: {"contact_steps": 0, "bilateral_contact_steps": 0} for phase, *_ in phases}
|
||||
@ -135,7 +126,7 @@ def run_rollout(
|
||||
started = time.time()
|
||||
frame_stride = max(1, round((1.0 / model.opt.timestep) / fps))
|
||||
step_index = 0
|
||||
for phase, target, gripper_ctrl, count, attach_cube in phases:
|
||||
for phase, target, gripper_ctrl, count, _contact_expected in phases:
|
||||
if phase == "grasp":
|
||||
settled_cube_z = float(data.qpos[cube_qadr + 2])
|
||||
max_cube_z = settled_cube_z
|
||||
@ -145,14 +136,7 @@ def run_rollout(
|
||||
current = data.ctrl[:6].copy()
|
||||
data.ctrl[:6] = current * (1 - alpha) + q_target * alpha
|
||||
data.ctrl[6] = gripper_ctrl
|
||||
if grasp_mode == "kinematic_attachment" and (attach_cube or attached):
|
||||
attached = attach_cube
|
||||
if attached:
|
||||
set_cube_pose(data, cube_qadr, data.site_xpos[site_id] + np.array([0.0, 0.0, -0.095]))
|
||||
mujoco.mj_step(model, data)
|
||||
if grasp_mode == "kinematic_attachment" and attached:
|
||||
set_cube_pose(data, cube_qadr, data.site_xpos[site_id] + np.array([0.0, 0.0, -0.095]))
|
||||
mujoco.mj_forward(model, data)
|
||||
touching_fingers = cube_finger_contacts(data, cube_geom, finger_geoms)
|
||||
accumulate_contact_metrics(model, data, cube_geom, finger_geoms, contact_force_metrics)
|
||||
if touching_fingers:
|
||||
@ -193,7 +177,7 @@ def run_rollout(
|
||||
place_error = float(np.linalg.norm(final_cube - target_cube))
|
||||
physical_success = bilateral_contact_steps > 0 and lift_height >= 0.10 and place_error <= 0.12
|
||||
summary = {
|
||||
"mode": "scripted_ik_physical_contact" if grasp_mode == "physical" else "scripted_ik_with_cube_attachment",
|
||||
"mode": "scripted_ik_physical_contact",
|
||||
"steps": step_index,
|
||||
"samples": len(trajectory),
|
||||
"sim_time": float(data.time),
|
||||
@ -210,22 +194,18 @@ def run_rollout(
|
||||
"phase_contacts": phase_contacts,
|
||||
"gripper_qpos_range": gripper_qpos_range,
|
||||
"contact_force_metrics": contact_force_metrics,
|
||||
"physical_success": physical_success if grasp_mode == "physical" else None,
|
||||
"physical_success": physical_success,
|
||||
"elapsed_ms": round((time.time() - started) * 1000),
|
||||
}
|
||||
report = {
|
||||
"schema": "real2sim.rm65_pick_place_rollout_report.v1",
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"ok": physical_success if grasp_mode == "physical" else True,
|
||||
"ok": physical_success,
|
||||
"runtime": "mujoco",
|
||||
"scene": str(sim_path),
|
||||
"summary": summary,
|
||||
"phases": [phase for phase, *_ in phases],
|
||||
"note": (
|
||||
"Scripted IK with contact-physics grasp; the cube pose is never overwritten. This is not a learned policy."
|
||||
if grasp_mode == "physical"
|
||||
else "Diagnostic-only rollout. Cube follows the gripper during grasp/lift/place; this is not a learned policy."
|
||||
),
|
||||
"note": "Scripted IK with contact-physics grasp; the cube pose is never overwritten. This is not a learned policy.",
|
||||
}
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
artifact_paths["report"] = str(report_path)
|
||||
@ -284,12 +264,6 @@ def solve_ik(model: Any, data: Any, site_id: int, target: np.ndarray, qpos_adrs:
|
||||
return q
|
||||
|
||||
|
||||
def set_cube_pose(data: Any, qadr: int, pos: np.ndarray) -> None:
|
||||
data.qpos[qadr : qadr + 3] = pos
|
||||
data.qpos[qadr + 3 : qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0])
|
||||
data.qvel[qadr : qadr + 6] = 0
|
||||
|
||||
|
||||
def sample_state(model: Any, data: Any, phase: str, step: int, site_id: int, cube_qadr: int) -> dict[str, Any]:
|
||||
return {
|
||||
"step": step,
|
||||
|
||||
195
src/real2sim_platform/act_inference.py
Normal file
195
src/real2sim_platform/act_inference.py
Normal file
@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import pickle
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .rm65_controller import RM65PickPlaceRuntime, _write_preview_video
|
||||
|
||||
|
||||
def load_policy(model_dir: Path, act_root: Path, max_timesteps: int):
|
||||
import torch
|
||||
|
||||
sys.path.insert(0, str(act_root))
|
||||
from configurations.task_configs.config_tools.basic_configer import config_policy
|
||||
|
||||
config_path = model_dir / "task_config.py"
|
||||
checkpoint = model_dir / "core" / "policy_best.ckpt"
|
||||
stats_path = model_dir / "core" / "dataset_stats.pkl"
|
||||
for path in (config_path, checkpoint, stats_path):
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
spec = importlib.util.spec_from_file_location("real2sim_act_task_config", config_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError("unable to load ACT task config")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
task_config = module.TASK_CONFIG
|
||||
merged = {**task_config["common"], **task_config["train"]}
|
||||
merged["learning_rate"] = task_config["train"]["learning_rate"]
|
||||
policy_config = config_policy(merged)
|
||||
policy_config["ckpt_path"] = str(checkpoint)
|
||||
policy_config["max_timesteps"] = max_timesteps
|
||||
policy = module.policy_maker(policy_config, "eval")
|
||||
with stats_path.open("rb") as handle:
|
||||
stats = pickle.load(handle)
|
||||
if not all(key in stats for key in ("qpos_mean", "qpos_std", "action_mean", "action_std")):
|
||||
raise ValueError("ACT dataset_stats.pkl is missing normalization arrays")
|
||||
return policy, stats, task_config["common"]["camera_names"], torch
|
||||
|
||||
|
||||
def infer(
|
||||
*,
|
||||
scene_path: Path,
|
||||
model_dir: Path,
|
||||
act_root: Path,
|
||||
out_dir: Path,
|
||||
episodes: int,
|
||||
steps: int,
|
||||
seed: int,
|
||||
minimum_success_rate: float,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> dict[str, Any]:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
policy, stats, camera_names, torch = load_policy(model_dir, act_root, steps)
|
||||
if camera_names != ["overview"]:
|
||||
raise ValueError(f"RM65 runtime currently exposes camera ['overview']; model expects {camera_names}")
|
||||
runtime = RM65PickPlaceRuntime(scene_path, width=width, height=height, seed=seed)
|
||||
results: list[dict[str, Any]] = []
|
||||
all_actions: list[np.ndarray] = []
|
||||
try:
|
||||
for episode in range(episodes):
|
||||
policy.reset()
|
||||
cube, target, settled_z = _reset_randomized(runtime)
|
||||
trajectory: list[dict[str, Any]] = []
|
||||
frames: list[np.ndarray] = []
|
||||
max_cube_z = settled_z
|
||||
bilateral_steps = 0
|
||||
for step in range(steps):
|
||||
runtime.renderer.update_scene(runtime.data, camera="overview")
|
||||
frame = runtime.renderer.render().copy()
|
||||
qpos = np.concatenate([runtime.data.qpos[runtime.qpos_adrs], [runtime.data.qpos[runtime.gripper_qadr]]]).astype(np.float32)
|
||||
normalized_qpos = (qpos - np.asarray(stats["qpos_mean"], dtype=np.float32)) / np.asarray(stats["qpos_std"], dtype=np.float32)
|
||||
image = torch.from_numpy(frame).permute(2, 0, 1).unsqueeze(0).float().cuda() / 255.0
|
||||
qpos_tensor = torch.from_numpy(normalized_qpos).unsqueeze(0).float().cuda()
|
||||
with torch.inference_mode():
|
||||
predicted = policy(qpos_tensor, image)
|
||||
normalized_action = predicted[0, 0].detach().cpu().numpy()
|
||||
action = normalized_action * np.asarray(stats["action_std"], dtype=np.float32) + np.asarray(stats["action_mean"], dtype=np.float32)
|
||||
action = np.asarray(action, dtype=np.float32)
|
||||
if action.shape != (7,):
|
||||
raise ValueError(f"ACT action shape must be (7,), received {action.shape}")
|
||||
for index in range(7):
|
||||
action[index] = np.clip(action[index], runtime.ctrl_ranges[index, 0], runtime.ctrl_ranges[index, 1])
|
||||
runtime.data.ctrl[:7] = action
|
||||
for _ in range(10):
|
||||
runtime.mujoco.mj_step(runtime.model, runtime.data)
|
||||
touching = runtime.finger_contacts()
|
||||
bilateral_steps += int(len(touching) == 2)
|
||||
max_cube_z = max(max_cube_z, float(runtime.data.qpos[runtime.cube_qadr + 2]))
|
||||
all_actions.append(action.copy())
|
||||
frames.append(frame)
|
||||
trajectory.append(
|
||||
{
|
||||
"step": step,
|
||||
"time": float(runtime.data.time),
|
||||
"qpos": qpos.tolist(),
|
||||
"action": action.tolist(),
|
||||
"cube_pos": runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3].astype(float).tolist(),
|
||||
"target": target.tolist(),
|
||||
"bilateral_contact": len(touching) == 2,
|
||||
}
|
||||
)
|
||||
final_cube = runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3].copy()
|
||||
lift_height = float(max_cube_z - settled_z)
|
||||
place_error = float(np.linalg.norm(final_cube - target))
|
||||
success = bilateral_steps > 0 and lift_height >= 0.10 and place_error <= 0.05
|
||||
result = {
|
||||
"episode": episode,
|
||||
"success": success,
|
||||
"lift_height": lift_height,
|
||||
"place_error": place_error,
|
||||
"bilateral_contact_steps": bilateral_steps,
|
||||
"initial_cube": cube.tolist(),
|
||||
"target": target.tolist(),
|
||||
}
|
||||
results.append(result)
|
||||
(out_dir / f"episode_{episode:03d}_trajectory.json").write_text(json.dumps({"schema": "real2sim.act_rollout.v1", "result": result, "samples": trajectory}, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
_write_preview_video(out_dir / f"episode_{episode:03d}.mp4", frames)
|
||||
finally:
|
||||
runtime.close()
|
||||
success_rate = sum(int(item["success"]) for item in results) / max(1, len(results))
|
||||
action_matrix = np.asarray(all_actions, dtype=np.float32)
|
||||
action_std = action_matrix.std(axis=0).tolist() if len(action_matrix) else []
|
||||
nonconstant = bool(len(action_matrix) > 1 and np.max(np.std(action_matrix, axis=0)) > 1e-5)
|
||||
report = {
|
||||
"schema": "real2sim.discoverse_act_inference.v1",
|
||||
"runtime": "mujoco",
|
||||
"scene": str(scene_path),
|
||||
"model": str(model_dir),
|
||||
"episodes": results,
|
||||
"success_rate": success_rate,
|
||||
"minimum_success_rate": minimum_success_rate,
|
||||
"action_std": action_std,
|
||||
"actions_nonconstant": nonconstant,
|
||||
"pass": success_rate >= minimum_success_rate and nonconstant,
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
}
|
||||
(out_dir / "inference_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
def _reset_randomized(runtime: RM65PickPlaceRuntime) -> tuple[np.ndarray, np.ndarray, float]:
|
||||
runtime.mujoco.mj_resetDataKeyframe(runtime.model, runtime.data, 0)
|
||||
cube = np.array([runtime.rng.uniform(0.645, 0.655), runtime.rng.uniform(-0.010, 0.010), 0.33], dtype=float)
|
||||
target = np.array([runtime.rng.uniform(0.42, 0.51), runtime.rng.uniform(0.13, 0.22), 0.31], dtype=float)
|
||||
runtime.data.qpos[runtime.cube_qadr : runtime.cube_qadr + 3] = cube
|
||||
runtime.data.qpos[runtime.cube_qadr + 3 : runtime.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0])
|
||||
cube_dof = int(runtime.model.jnt_dofadr[runtime.cube_joint])
|
||||
runtime.data.qvel[cube_dof : cube_dof + 6] = 0
|
||||
runtime.model.body_pos[runtime.target_body] = np.array([target[0], target[1], 0.335])
|
||||
runtime.mujoco.mj_forward(runtime.model, runtime.data)
|
||||
for _ in range(120):
|
||||
runtime.mujoco.mj_step(runtime.model, runtime.data)
|
||||
return cube, target, float(runtime.data.qpos[runtime.cube_qadr + 2])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run DISCOVERSE ACT in the physical RM65 MuJoCo scene")
|
||||
parser.add_argument("--scene", type=Path, required=True)
|
||||
parser.add_argument("--model", type=Path, required=True)
|
||||
parser.add_argument("--act-root", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--episodes", type=int, default=10)
|
||||
parser.add_argument("--steps", type=int, default=200)
|
||||
parser.add_argument("--seed", type=int, default=11)
|
||||
parser.add_argument("--minimum-success-rate", type=float, default=0.5)
|
||||
parser.add_argument("--width", type=int, default=320)
|
||||
parser.add_argument("--height", type=int, default=240)
|
||||
args = parser.parse_args()
|
||||
report = infer(
|
||||
scene_path=args.scene,
|
||||
model_dir=args.model,
|
||||
act_root=args.act_root,
|
||||
out_dir=args.out,
|
||||
episodes=args.episodes,
|
||||
steps=args.steps,
|
||||
seed=args.seed,
|
||||
minimum_success_rate=args.minimum_success_rate,
|
||||
width=args.width,
|
||||
height=args.height,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["pass"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
from functools import lru_cache
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
@ -15,6 +17,7 @@ from urllib.parse import quote, unquote, urlparse
|
||||
from urllib.request import urlopen
|
||||
|
||||
from .config import PlatformConfig
|
||||
from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path
|
||||
from .store import PlatformStore
|
||||
|
||||
|
||||
@ -61,16 +64,26 @@ def scan_dataset(path: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def run_doctor(config: PlatformConfig) -> dict[str, Any]:
|
||||
core_checks = [
|
||||
"workspace_writable",
|
||||
"expert_policy",
|
||||
"training",
|
||||
"inference",
|
||||
"media_prepare_deps",
|
||||
"ffmpeg",
|
||||
"spark_bundle",
|
||||
"three_bundle",
|
||||
]
|
||||
core_checks = (
|
||||
[
|
||||
"workspace_writable",
|
||||
"rm65_controller",
|
||||
"discoverse_act",
|
||||
"torch_cuda",
|
||||
"act_import",
|
||||
"media_prepare_deps",
|
||||
"ffmpeg",
|
||||
"colmap",
|
||||
"gs_video_reconstruct",
|
||||
"anysplat_pinned",
|
||||
"mujoco_python",
|
||||
"motrixsim_python",
|
||||
"nvidia_smi",
|
||||
]
|
||||
if config.production_mode
|
||||
else ["workspace_writable", "expert_policy", "training", "inference", "media_prepare_deps", "ffmpeg", "spark_bundle", "three_bundle"]
|
||||
)
|
||||
act_root = config.discoverse_root / "policies" / "act"
|
||||
checks = {
|
||||
"workspace_writable": _is_writable(config.workspace_dir),
|
||||
"expert_policy": (config.expert_policy_root / "src/real2sim_expert/collect.py").is_file(),
|
||||
@ -91,9 +104,15 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]:
|
||||
"mujoco_python": _module_available("mujoco"),
|
||||
"service_process": os.getpid() > 0,
|
||||
"nvidia_smi": shutil.which("nvidia-smi") is not None,
|
||||
"rm65_controller": Path(__file__).with_name("rm65_controller.py").is_file(),
|
||||
"discoverse_act": _git_head(config.discoverse_root) == config.discoverse_commit and _git_head(act_root) == config.act_commit,
|
||||
"torch_cuda": _torch_cuda_ready(),
|
||||
"act_import": _act_runtime_import_ready(str(config.discoverse_root), config.discoverse_commit, config.act_commit),
|
||||
"anysplat_pinned": _git_head(config.anysplat_root) == config.anysplat_commit and config.anysplat_python.exists(),
|
||||
"motrixsim_python": _module_available("motrixsim"),
|
||||
}
|
||||
return {
|
||||
"ok": all(checks[name] for name in core_checks),
|
||||
"ok": all(bool(checks[name]) for name in core_checks),
|
||||
"core_checks": core_checks,
|
||||
"warnings": [
|
||||
name
|
||||
@ -108,6 +127,8 @@ def run_doctor(config: PlatformConfig) -> dict[str, Any]:
|
||||
"cuda_home",
|
||||
"colmap",
|
||||
"nvidia_smi",
|
||||
"torch_cuda",
|
||||
"act_import",
|
||||
"three_bundle",
|
||||
]
|
||||
if not checks.get(name)
|
||||
@ -148,6 +169,8 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A
|
||||
return run_reconstruction_preflight(config, store, task, emit)
|
||||
if task_type == "hf_import":
|
||||
return run_hf_import(config, store, task, emit)
|
||||
if task_type == "bos_import":
|
||||
return run_bos_import(config, store, task, emit)
|
||||
if task_type == "simulator_validation":
|
||||
return run_simulator_validation(config, store, task, emit)
|
||||
if task_type == "expert_collection":
|
||||
@ -165,6 +188,42 @@ def execute_task(config: PlatformConfig, store: PlatformStore, task: dict[str, A
|
||||
raise ValueError(f"unknown task type: {task_type}")
|
||||
|
||||
|
||||
def sync_task_outputs(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], result: dict[str, Any]) -> dict[str, Any]:
|
||||
storage = BosStorage(config)
|
||||
if not storage.configured:
|
||||
if config.production_mode:
|
||||
raise RuntimeError("生产任务产物无法持久化:BOS 尚未配置")
|
||||
return result
|
||||
_task_progress(store, task, 96, "持久化产物", "正在将任务报告和业务产物上传到 BOS")
|
||||
run_dir = Path(task["run_dir"])
|
||||
if run_dir.is_dir():
|
||||
_upload_directory(storage, run_dir, area="runs", relative_root=task["id"])
|
||||
resource_specs = [
|
||||
("dataset_id", "datasets", store.get_dataset),
|
||||
("scene_id", "scenes", store.get_scene),
|
||||
("model_id", "models", store.get_model),
|
||||
]
|
||||
persisted: dict[str, Any] = {}
|
||||
for result_key, area, getter in resource_specs:
|
||||
entity_id = str(result.get(result_key) or "")
|
||||
if not entity_id:
|
||||
continue
|
||||
entity = getter(entity_id)
|
||||
path = Path(str(entity["path"]))
|
||||
root = path if path.is_dir() else path.parent
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(f"resource output directory not found: {root}")
|
||||
_upload_directory(storage, root, area=area, relative_root=entity_id)
|
||||
checksum = _directory_digest(root)
|
||||
uri = storage.uri_for_key(storage.object_key(area, f"{entity_id}/"))
|
||||
store.update_entity_storage(area, entity_id, storage_uri=uri, checksum=checksum, cache_state="local")
|
||||
persisted[result_key] = {"storage_uri": uri, "checksum": checksum}
|
||||
result = dict(result)
|
||||
result["persistent_outputs"] = persisted
|
||||
result["run_storage_uri"] = storage.uri_for_key(storage.object_key("runs", f"{task['id']}/"))
|
||||
return result
|
||||
|
||||
|
||||
def _task_progress(store: PlatformStore, task: dict[str, Any] | str, progress: int, stage: str, message: str = "") -> None:
|
||||
task_id = task if isinstance(task, str) else task["id"]
|
||||
store.update_task(task_id, progress=progress, stage=stage, message=message)
|
||||
@ -504,6 +563,66 @@ def run_hf_import(config: PlatformConfig, store: PlatformStore, task: dict[str,
|
||||
return {"dataset_id": dataset["id"], "dataset_path": str(target), "manifest_path": str(manifest_path), "scan": scan}
|
||||
|
||||
|
||||
def run_bos_import(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||||
params = task["params"]
|
||||
storage = BosStorage(config)
|
||||
raw_prefix = normalize_relative_path(str(params.get("prefix") or ""))
|
||||
prefix_parts = raw_prefix.split("/", 1)
|
||||
area = prefix_parts[0] if prefix_parts[0] in ALLOWED_AREAS else "datasets"
|
||||
relative_prefix = prefix_parts[1] if prefix_parts[0] in ALLOWED_AREAS and len(prefix_parts) > 1 else raw_prefix
|
||||
_task_progress(store, task, 18, "读取 BOS", "正在列举授权前缀内的对象")
|
||||
objects = storage.list_objects(area=area, relative_prefix=relative_prefix, max_keys=int(params.get("max_objects", 1000)))
|
||||
objects = [item for item in objects if not item["key"].endswith("/")]
|
||||
if not objects:
|
||||
raise ValueError("BOS 前缀中没有可导入对象")
|
||||
dataset_id = uuid.uuid4().hex
|
||||
out_dir = config.datasets_dir / dataset_id
|
||||
out_dir.mkdir(parents=True, exist_ok=False)
|
||||
common_prefix = storage.object_key(area, relative_prefix)
|
||||
_task_progress(store, task, 30, "下载数据", f"正在下载 {len(objects)} 个真实 BOS 对象")
|
||||
for index, item in enumerate(objects, start=1):
|
||||
relative = item["key"][len(common_prefix):].lstrip("/") or Path(item["key"]).name
|
||||
storage.download_file(item["storage_uri"], out_dir / relative)
|
||||
if index % max(1, len(objects) // 10) == 0:
|
||||
_task_progress(store, task, 30 + int(45 * index / len(objects)), "下载数据", f"已下载 {index}/{len(objects)} 个对象")
|
||||
scan = scan_dataset(out_dir)
|
||||
digest = _directory_digest(out_dir)
|
||||
source_uri = f"bos://{storage.bucket}/{common_prefix}"
|
||||
manifest = {
|
||||
"schema": "real2sim.dataset_manifest.v2",
|
||||
"id": dataset_id,
|
||||
"name": params.get("name") or Path(relative_prefix.rstrip("/")).name,
|
||||
"source_type": "bos_prefix",
|
||||
"storage_uri": source_uri,
|
||||
"checksum": digest,
|
||||
"version": digest[:12],
|
||||
"object_count": len(objects),
|
||||
"objects": objects,
|
||||
"scan": scan,
|
||||
}
|
||||
manifest_path = out_dir / "dataset_manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
dataset = store.create_dataset(
|
||||
item_id=dataset_id,
|
||||
name=str(manifest["name"]),
|
||||
path=out_dir,
|
||||
scan=scan,
|
||||
source_type="bos_prefix",
|
||||
source_uri=source_uri,
|
||||
version=digest[:12],
|
||||
parent_task_id=task["id"],
|
||||
manifest_path=manifest_path,
|
||||
metrics={"object_count": len(objects), "size_bytes": sum(int(item["size_bytes"]) for item in objects)},
|
||||
tags=["bos", "imported"],
|
||||
storage_uri=source_uri,
|
||||
checksum=digest,
|
||||
cache_state="local",
|
||||
)
|
||||
_index_artifacts(store, task["id"], out_dir)
|
||||
_task_progress(store, task, 95, "登记数据集", "BOS 数据及校验清单已登记")
|
||||
return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "storage_uri": source_uri, "object_count": len(objects), "checksum": digest}
|
||||
|
||||
|
||||
def run_simulator_validation(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||||
params = task["params"]
|
||||
run_dir = Path(task["run_dir"])
|
||||
@ -564,8 +683,50 @@ def run_expert_collection(config: PlatformConfig, store: PlatformStore, task: di
|
||||
run_dir = Path(task["run_dir"])
|
||||
scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json"
|
||||
out_dir = run_dir / "dataset"
|
||||
policy_preset = str(params.get("policy_preset") or "pick_and_place_v1")
|
||||
if policy_preset != "pick_and_place_v1":
|
||||
policy_preset = str(params.get("policy_preset") or ("rm65_pick_place_6d_ik" if config.production_mode else "pick_and_place_v1"))
|
||||
if policy_preset == "rm65_pick_place_6d_ik":
|
||||
scene_path = _resolve_rm65_runtime_scene(store, params, scene_path)
|
||||
required_successes = int(params.get("episodes", 50))
|
||||
max_attempts = int(params.get("max_attempts", max(required_successes * 3, required_successes)))
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"real2sim_platform.rm65_controller",
|
||||
"--scene",
|
||||
str(scene_path),
|
||||
"--out",
|
||||
str(out_dir),
|
||||
"--required-successes",
|
||||
str(required_successes),
|
||||
"--max-attempts",
|
||||
str(max_attempts),
|
||||
"--steps",
|
||||
str(int(params.get("steps", 200))),
|
||||
"--seed",
|
||||
str(int(params.get("seed", 7))),
|
||||
"--width",
|
||||
str(int(params.get("image_width", 320))),
|
||||
"--height",
|
||||
str(int(params.get("image_height", 240))),
|
||||
]
|
||||
_task_progress(store, task, 28, "物理采集", "RM65 正在通过 6D IK、双指接触和真实物理步进采集成功 episode")
|
||||
run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src"]), store=store, task_id=task["id"])
|
||||
manifest = _read_json(out_dir / "manifest.json")
|
||||
dataset = store.create_dataset(
|
||||
name=params.get("name", f"RM65 ACT dataset {task['id'][:8]}"),
|
||||
path=out_dir,
|
||||
scan={"input_kind": "discoverse_act_hdf5", "samples": manifest.get("successful_episodes"), "manifest": str(out_dir / "manifest.json")},
|
||||
source_type="expert_collection",
|
||||
source_uri=str(scene_path),
|
||||
parent_task_id=task["id"],
|
||||
manifest_path=out_dir / "manifest.json",
|
||||
metrics={"samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate"), "policy_preset": policy_preset},
|
||||
tags=["discoverse_act_hdf5", "rm65", "physical_contact", policy_preset],
|
||||
)
|
||||
_index_artifacts(store, task["id"], out_dir)
|
||||
_task_progress(store, task, 95, "登记数据集", "真实采集 HDF5、接触记录和成功报告已登记")
|
||||
return {"dataset_id": dataset["id"], "dataset_path": str(out_dir), "samples": manifest.get("successful_episodes"), "success_rate": manifest.get("success_rate")}
|
||||
if policy_preset != "pick_and_place_v1" or config.production_mode:
|
||||
raise ValueError(f"unsupported expert policy preset: {policy_preset}")
|
||||
_task_progress(store, task, 18, "准备采集", "正在加载场景、专家策略和机器人配置")
|
||||
command = [
|
||||
@ -615,7 +776,9 @@ def run_training(config: PlatformConfig, store: PlatformStore, task: dict[str, A
|
||||
_task_progress(store, task, 18, "准备训练", "正在加载训练数据集和训练配方")
|
||||
dataset_path = _dataset_path(store, params)
|
||||
out_dir = run_dir / "model"
|
||||
recipe = str(params.get("recipe") or "bc_v2")
|
||||
recipe = str(params.get("recipe") or ("discoverse_act" if config.production_mode else "bc_v2"))
|
||||
if recipe == "discoverse_act":
|
||||
return _run_discoverse_act_training(config, store, task, dataset_path, out_dir)
|
||||
recipe_config = {
|
||||
"bc_v2": {"regularization": 1e-3, "base_model": "facebook/dinov2-small"},
|
||||
"bc_fast": {"regularization": 1e-2, "base_model": "facebook/dinov2-small"},
|
||||
@ -673,6 +836,44 @@ def run_inference(config: PlatformConfig, store: PlatformStore, task: dict[str,
|
||||
scene_path = _scene_path(store, params) or config.expert_policy_root / "examples/retail_single_arm_scene.json"
|
||||
model_path = _model_path(store, params)
|
||||
out_dir = run_dir / "inference"
|
||||
model = store.get_model(str(params["model_id"])) if params.get("model_id") else None
|
||||
model_recipe = str(((model or {}).get("metadata") or {}).get("training", {}).get("recipe") or params.get("recipe") or "")
|
||||
if model_recipe == "discoverse_act":
|
||||
scene_path = _resolve_rm65_runtime_scene(store, params, scene_path)
|
||||
act_root = config.discoverse_root / "policies" / "act"
|
||||
if _git_head(act_root) != config.act_commit:
|
||||
raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}")
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"real2sim_platform.act_inference",
|
||||
"--scene",
|
||||
str(scene_path),
|
||||
"--model",
|
||||
str(model_path),
|
||||
"--act-root",
|
||||
str(act_root),
|
||||
"--out",
|
||||
str(out_dir),
|
||||
"--episodes",
|
||||
str(int(params.get("episodes", 10))),
|
||||
"--steps",
|
||||
str(int(params.get("steps", 200))),
|
||||
"--seed",
|
||||
str(int(params.get("seed", 11))),
|
||||
"--minimum-success-rate",
|
||||
str(float(params.get("minimum_success_rate", 0.5))),
|
||||
]
|
||||
_task_progress(store, task, 38, "ACT 在线推理", "DISCOVERSE ACT 正在真实 MuJoCo 场景中闭环控制 RM65")
|
||||
run_logged(command, cwd=config.app_root, log_path=run_dir / "logs" / "task.log", env=_env_with_paths(config, [config.app_root / "src", act_root, config.discoverse_root]), store=store, task_id=task["id"])
|
||||
report = _read_json(out_dir / "inference_report.json")
|
||||
if not report.get("actions_nonconstant"):
|
||||
raise RuntimeError("ACT 推理动作是常量,拒绝登记为有效推理")
|
||||
_index_artifacts(store, task["id"], out_dir)
|
||||
_task_progress(store, task, 95, "索引推理产物", "真实 rollout 视频、轨迹和成功率报告已登记")
|
||||
return {"episodes": len(report.get("episodes", [])), "success_rate": report.get("success_rate"), "report": str(out_dir / "inference_report.json")}
|
||||
if config.production_mode:
|
||||
raise ValueError("生产模式只允许 DISCOVERSE ACT 模型在线推理")
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
@ -695,6 +896,97 @@ def run_inference(config: PlatformConfig, store: PlatformStore, task: dict[str,
|
||||
return {"rollout_steps": len(output.get("rollout", [])), "viewer": str(out_dir / "viewer.html")}
|
||||
|
||||
|
||||
def _run_discoverse_act_training(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], dataset_path: Path, out_dir: Path) -> dict[str, Any]:
|
||||
params = task["params"]
|
||||
run_dir = Path(task["run_dir"])
|
||||
act_root = config.discoverse_root / "policies" / "act"
|
||||
if _git_head(act_root) != config.act_commit:
|
||||
raise RuntimeError(f"DISCOVERSE ACT commit mismatch; required {config.act_commit}")
|
||||
dataset_info = _inspect_act_dataset(dataset_path)
|
||||
if dataset_info["episode_count"] < 2:
|
||||
raise ValueError("DISCOVERSE ACT 至少需要 2 个真实 HDF5 episode")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
config_root = run_dir / "generated_config"
|
||||
task_name = f"real2sim_{task['id'][:12]}"
|
||||
config_path = config_root / "configurations" / "task_configs" / f"{task_name}.py"
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
epochs = int(params.get("epochs", 100))
|
||||
validate_every = max(1, min(int(params.get("validate_every", max(1, epochs // 5))), epochs))
|
||||
save_every = max(1, min(int(params.get("save_every", validate_every)), epochs))
|
||||
batch_size = max(1, min(int(params.get("batch_size", 8)), dataset_info["episode_count"] - 1))
|
||||
chunk_size = max(2, min(int(params.get("chunk_size", 32)), dataset_info["episode_length"]))
|
||||
core_dir = out_dir / "core"
|
||||
checkpoint_dir = out_dir / "checkpoints"
|
||||
config_path.write_text(
|
||||
_act_task_config_source(
|
||||
task_name=task_name,
|
||||
dataset_path=dataset_path,
|
||||
checkpoint_dir=checkpoint_dir,
|
||||
stats_path=core_dir / "dataset_stats.pkl",
|
||||
camera_names=dataset_info["camera_names"],
|
||||
state_dim=dataset_info["state_dim"],
|
||||
action_dim=dataset_info["action_dim"],
|
||||
epochs=epochs,
|
||||
batch_size=batch_size,
|
||||
chunk_size=chunk_size,
|
||||
validate_every=validate_every,
|
||||
save_every=save_every,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
command = [sys.executable, "policy_train.py", "-tn", task_name, "-ne", str(epochs), "-ve", str(validate_every), "-se", str(save_every), "--seed", str(int(params.get("seed", 1)))]
|
||||
env = _env_with_paths(config, [config_root, act_root, config.discoverse_root])
|
||||
env["MPLBACKEND"] = "Agg"
|
||||
_task_progress(store, task, 34, "ACT 训练", f"正在使用 {dataset_info['episode_count']} 个真实 episode 训练 DISCOVERSE ACT")
|
||||
run_logged(command, cwd=act_root, log_path=run_dir / "logs" / "task.log", env=env, store=store, task_id=task["id"])
|
||||
checkpoint = core_dir / "policy_best.ckpt"
|
||||
stats_path = core_dir / "dataset_stats.pkl"
|
||||
if not checkpoint.is_file() or checkpoint.stat().st_size == 0 or not stats_path.is_file():
|
||||
raise RuntimeError("DISCOVERSE ACT 训练结束但未生成非空 policy_best.ckpt 或 dataset_stats.pkl")
|
||||
description = (core_dir / "description.txt").read_text(encoding="utf-8").strip() if (core_dir / "description.txt").is_file() else ""
|
||||
best_loss = _parse_best_loss(description)
|
||||
if best_loss is None or not math.isfinite(best_loss):
|
||||
raise RuntimeError("ACT 训练指标不是有限数值")
|
||||
model_card = {
|
||||
"schema": "real2sim.model_card.v2",
|
||||
"name": params.get("name", f"RM65 ACT {task['id'][:8]}"),
|
||||
"framework": "DISCOVERSE ACT",
|
||||
"training": {
|
||||
"recipe": "discoverse_act",
|
||||
"discoverse_commit": config.discoverse_commit,
|
||||
"act_commit": config.act_commit,
|
||||
"dataset_path": str(dataset_path),
|
||||
"epochs": epochs,
|
||||
"batch_size": batch_size,
|
||||
"chunk_size": chunk_size,
|
||||
"camera_names": dataset_info["camera_names"],
|
||||
"state_dim": dataset_info["state_dim"],
|
||||
"action_dim": dataset_info["action_dim"],
|
||||
"config_path": str(config_path),
|
||||
},
|
||||
"metrics": {"best_validation_loss": best_loss},
|
||||
"artifacts": {"checkpoint": str(checkpoint), "dataset_stats": str(stats_path)},
|
||||
"description": description,
|
||||
}
|
||||
card_path = out_dir / "model_card.json"
|
||||
card_path.write_text(json.dumps(model_card, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
shutil.copy2(config_path, out_dir / "task_config.py")
|
||||
model = store.create_model(
|
||||
name=str(model_card["name"]),
|
||||
path=out_dir,
|
||||
metadata=model_card,
|
||||
source_type="training",
|
||||
source_uri=str(dataset_path),
|
||||
parent_task_id=task["id"],
|
||||
manifest_path=card_path,
|
||||
metrics=model_card["metrics"],
|
||||
tags=["policy", "discoverse_act", "rm65"],
|
||||
)
|
||||
_index_artifacts(store, task["id"], out_dir)
|
||||
_task_progress(store, task, 95, "索引 ACT 模型", "checkpoint、损失曲线、模型卡和数据血缘已登记")
|
||||
return {"model_id": model["id"], "model_path": str(out_dir), "metrics": model_card["metrics"], "checkpoint": str(checkpoint)}
|
||||
|
||||
|
||||
def run_renderer_benchmark(config: PlatformConfig, store: PlatformStore, task: dict[str, Any], emit: Callable[[str], None]) -> dict[str, Any]:
|
||||
run_dir = Path(task["run_dir"])
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
@ -1550,6 +1842,35 @@ def _scene_path(store: PlatformStore, params: dict[str, Any]) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_rm65_runtime_scene(store: PlatformStore, params: dict[str, Any], fallback: Path) -> Path:
|
||||
candidates: list[Path] = []
|
||||
if fallback.suffix.lower() == ".xml":
|
||||
candidates.append(fallback)
|
||||
if params.get("scene_id"):
|
||||
scene = store.get_scene(str(params["scene_id"]))
|
||||
metadata = scene.get("metadata") or {}
|
||||
package = Path(str(metadata.get("package") or ""))
|
||||
manifest_path = Path(str(scene.get("manifest_path") or ""))
|
||||
manifest = _read_optional_json(manifest_path)
|
||||
for asset in manifest.get("assets", []) if isinstance(manifest.get("assets"), list) else []:
|
||||
if not isinstance(asset, dict):
|
||||
continue
|
||||
name = str(asset.get("name") or asset.get("path") or "").lower()
|
||||
if "rm65" in name and name.endswith(".xml"):
|
||||
candidates.append(Path(str(asset.get("path") or "")))
|
||||
if str(package):
|
||||
candidates.extend([
|
||||
package / "simulator_assets" / "rm65_table_pick_place.xml",
|
||||
package / "scene_mujoco.xml",
|
||||
package / "mjcf" / "scene.xml",
|
||||
])
|
||||
candidates.extend([fallback.parent / "simulator_assets" / "rm65_table_pick_place.xml", fallback.parent / "scene_mujoco.xml"])
|
||||
for candidate in candidates:
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
raise FileNotFoundError("所选场景没有可执行的 RM65 pick-place MJCF;请先在场景管理中绑定 RM65 控制器和碰撞资产")
|
||||
|
||||
|
||||
def _model_path(store: PlatformStore, params: dict[str, Any]) -> Path:
|
||||
if params.get("model_path"):
|
||||
return Path(str(params["model_path"]))
|
||||
@ -1558,6 +1879,147 @@ def _model_path(store: PlatformStore, params: dict[str, Any]) -> Path:
|
||||
raise ValueError("inference requires model_id or model_path")
|
||||
|
||||
|
||||
def _directory_digest(root: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
digest.update(relative.encode("utf-8"))
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(8 * 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _upload_directory(storage: BosStorage, root: Path, *, area: str, relative_root: str) -> None:
|
||||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||||
storage.upload_file(
|
||||
path,
|
||||
area=area,
|
||||
relative_path=f"{relative_root}/{path.relative_to(root).as_posix()}",
|
||||
metadata={"sha256": _file_sha256(path)},
|
||||
)
|
||||
|
||||
|
||||
def _file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(8 * 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _git_head(root: Path) -> str:
|
||||
try:
|
||||
return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=5).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _inspect_act_dataset(dataset_path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
import h5py
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("DISCOVERSE ACT training requires h5py") from exc
|
||||
episodes = sorted(dataset_path.glob("episode_*.hdf5"))
|
||||
if not episodes:
|
||||
raise ValueError("训练数据目录中没有 episode_*.hdf5")
|
||||
lengths: list[int] = []
|
||||
cameras: list[str] = []
|
||||
state_dim = 0
|
||||
action_dim = 0
|
||||
for path in episodes:
|
||||
with h5py.File(path, "r") as root:
|
||||
if "/observations/qpos" not in root or "/action" not in root or "/observations/images" not in root:
|
||||
raise ValueError(f"ACT HDF5 缺少 qpos/action/images: {path.name}")
|
||||
current_cameras = sorted(str(name) for name in root["/observations/images"].keys())
|
||||
current_state_dim = int(root["/observations/qpos"].shape[1])
|
||||
current_action_dim = int(root["/action"].shape[1])
|
||||
if not cameras:
|
||||
cameras = current_cameras
|
||||
state_dim = current_state_dim
|
||||
action_dim = current_action_dim
|
||||
if current_cameras != cameras or current_state_dim != state_dim or current_action_dim != action_dim:
|
||||
raise ValueError("ACT 数据集的相机、状态维度或动作维度不一致")
|
||||
length = int(root["/action"].shape[0])
|
||||
if length < 2:
|
||||
raise ValueError(f"ACT episode 过短: {path.name}")
|
||||
lengths.append(length)
|
||||
return {
|
||||
"episode_count": len(episodes),
|
||||
"episode_length": min(lengths),
|
||||
"camera_names": cameras,
|
||||
"state_dim": state_dim,
|
||||
"action_dim": action_dim,
|
||||
}
|
||||
|
||||
|
||||
def _act_task_config_source(
|
||||
*,
|
||||
task_name: str,
|
||||
dataset_path: Path,
|
||||
checkpoint_dir: Path,
|
||||
stats_path: Path,
|
||||
camera_names: list[str],
|
||||
state_dim: int,
|
||||
action_dim: int,
|
||||
epochs: int,
|
||||
batch_size: int,
|
||||
chunk_size: int,
|
||||
validate_every: int,
|
||||
save_every: int,
|
||||
) -> str:
|
||||
return f'''from configurations.task_configs.template import TASK_CONFIG_DEFAULT, activator, set_paths
|
||||
|
||||
|
||||
def policy_maker(config: dict, stage=None):
|
||||
from policies.act.act import ACTPolicy
|
||||
from policies.common.maker import post_init_policies
|
||||
policy = ACTPolicy(config)
|
||||
post_init_policies([policy], stage, [config["ckpt_path"]])
|
||||
return policy
|
||||
|
||||
|
||||
@activator(False)
|
||||
def augment_images(image):
|
||||
return image
|
||||
|
||||
|
||||
set_paths({str(dataset_path)!r}, {str(checkpoint_dir)!r}, {str(stats_path)!r})
|
||||
TASK_CONFIG_DEFAULT["common"]["camera_names"] = {camera_names!r}
|
||||
TASK_CONFIG_DEFAULT["common"]["state_dim"] = {state_dim}
|
||||
TASK_CONFIG_DEFAULT["common"]["action_dim"] = {action_dim}
|
||||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["policy_maker"] = policy_maker
|
||||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["temporal_agg"] = True
|
||||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["chunk_size"] = {chunk_size}
|
||||
TASK_CONFIG_DEFAULT["common"]["policy_config"]["num_queries"] = {chunk_size}
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["data_type"] = "hdf5"
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_episodes"] = "ALL"
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_train"] = {batch_size}
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["batch_size_validate"] = {batch_size}
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_train"] = 2
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["num_workers_validate"] = 2
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["observation_slice"] = None
|
||||
TASK_CONFIG_DEFAULT["train"]["load_data"]["action_slice"] = None
|
||||
TASK_CONFIG_DEFAULT["train"]["num_epochs"] = {epochs}
|
||||
TASK_CONFIG_DEFAULT["train"]["validate_every"] = {validate_every}
|
||||
TASK_CONFIG_DEFAULT["train"]["save_every"] = {save_every}
|
||||
TASK_CONFIG_DEFAULT["train"]["eval_every"] = 0
|
||||
TASK_CONFIG = TASK_CONFIG_DEFAULT
|
||||
'''
|
||||
|
||||
|
||||
def _parse_best_loss(description: str) -> float | None:
|
||||
import re
|
||||
|
||||
match = re.search(r"val loss\s+([0-9.eE+-]+)", description)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return float(match.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _write_dry_run_scene(package_dir: Path, input_path: Path, renderer: str, simulators: list[str]) -> Path:
|
||||
package_dir.mkdir(parents=True, exist_ok=True)
|
||||
scene = {
|
||||
@ -2465,6 +2927,40 @@ def _module_available(name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _torch_cuda_ready() -> bool:
|
||||
if not _module_available("torch"):
|
||||
return False
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", "import torch; raise SystemExit(0 if torch.cuda.is_available() else 2)"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
return completed.returncode == 0
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _act_runtime_import_ready(discoverse_root_value: str, discoverse_commit: str, act_commit: str) -> bool:
|
||||
discoverse_root = Path(discoverse_root_value)
|
||||
act_root = discoverse_root / "policies" / "act"
|
||||
if _git_head(discoverse_root) != discoverse_commit or _git_head(act_root) != act_commit or not _torch_cuda_ready():
|
||||
return False
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep)
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", "from policy import ACTPolicy; assert ACTPolicy is not None"],
|
||||
cwd=act_root,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=45,
|
||||
check=False,
|
||||
)
|
||||
return completed.returncode == 0
|
||||
|
||||
|
||||
def _is_writable(path: Path) -> bool:
|
||||
try:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
143
src/real2sim_platform/auth.py
Normal file
143
src/real2sim_platform/auth.py
Normal file
@ -0,0 +1,143 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import PlatformConfig
|
||||
from .store import PlatformStore
|
||||
|
||||
|
||||
SESSION_COOKIE = "real2sim_session"
|
||||
VALID_ROLES = {"admin", "engineer", "viewer"}
|
||||
_USERNAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{2,63}$")
|
||||
|
||||
|
||||
class AuthenticationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class AuthManager:
|
||||
def __init__(self, store: PlatformStore, config: PlatformConfig) -> None:
|
||||
self.store = store
|
||||
self.config = config
|
||||
self._hasher = _password_hasher() if config.auth_enabled else None
|
||||
|
||||
def bootstrap_admin(self) -> dict[str, Any] | None:
|
||||
if not self.config.auth_enabled or self.store.count_users() > 0:
|
||||
return None
|
||||
password = self.config.bootstrap_admin_password or secrets.token_urlsafe(18)
|
||||
validate_password(password)
|
||||
user = self.store.create_user(
|
||||
username=self.config.bootstrap_admin_username,
|
||||
password_hash=self.hash_password(password),
|
||||
role="admin",
|
||||
must_change_password=True,
|
||||
)
|
||||
if not self.config.bootstrap_admin_password:
|
||||
_write_bootstrap_file(self.config.auth_bootstrap_path, user["username"], password)
|
||||
return user
|
||||
|
||||
def hash_password(self, password: str) -> str:
|
||||
validate_password(password)
|
||||
if self._hasher is None:
|
||||
raise RuntimeError("authentication is disabled")
|
||||
return str(self._hasher.hash(password))
|
||||
|
||||
def verify_password(self, password_hash: str, password: str) -> bool:
|
||||
if self._hasher is None:
|
||||
return False
|
||||
try:
|
||||
return bool(self._hasher.verify(password_hash, password))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def login(self, username: str, password: str) -> tuple[dict[str, Any], str]:
|
||||
user = self.store.get_user_by_username(username.strip())
|
||||
if not user or user["disabled"] or not self.verify_password(str(user["password_hash"]), password):
|
||||
raise AuthenticationError("用户名或密码错误")
|
||||
token = secrets.token_urlsafe(48)
|
||||
self.store.create_session(
|
||||
token_hash=token_digest(token),
|
||||
user_id=user["id"],
|
||||
expires_at=time.time() + self.config.session_ttl_seconds,
|
||||
)
|
||||
self.store.update_user(user["id"], last_login=time.time())
|
||||
return self.store.get_user(user["id"]), token
|
||||
|
||||
def authenticate(self, token: str | None) -> dict[str, Any] | None:
|
||||
if not self.config.auth_enabled:
|
||||
return {
|
||||
"id": "local-development",
|
||||
"username": "local",
|
||||
"role": "admin",
|
||||
"disabled": False,
|
||||
"must_change_password": False,
|
||||
}
|
||||
if not token:
|
||||
return None
|
||||
return self.store.get_session_user(token_digest(token))
|
||||
|
||||
def logout(self, token: str | None) -> None:
|
||||
if token:
|
||||
self.store.delete_session(token_digest(token))
|
||||
|
||||
def create_user(self, *, username: str, password: str, role: str) -> dict[str, Any]:
|
||||
validate_username(username)
|
||||
validate_role(role)
|
||||
return self.store.create_user(
|
||||
username=username.strip(),
|
||||
password_hash=self.hash_password(password),
|
||||
role=role,
|
||||
must_change_password=True,
|
||||
)
|
||||
|
||||
def change_password(self, user_id: str, *, current_password: str | None, new_password: str, admin_reset: bool = False) -> dict[str, Any]:
|
||||
user_with_hash = self.store.get_user_by_username(self.store.get_user(user_id)["username"])
|
||||
if not user_with_hash:
|
||||
raise KeyError(user_id)
|
||||
if not admin_reset and not self.verify_password(str(user_with_hash["password_hash"]), current_password or ""):
|
||||
raise AuthenticationError("当前密码不正确")
|
||||
password_hash = self.hash_password(new_password)
|
||||
self.store.delete_user_sessions(user_id)
|
||||
return self.store.update_user(user_id, password_hash=password_hash, must_change_password=admin_reset)
|
||||
|
||||
|
||||
def token_digest(token: str) -> str:
|
||||
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def validate_username(username: str) -> None:
|
||||
if not _USERNAME_RE.fullmatch(username.strip()):
|
||||
raise ValueError("用户名需为 3-64 位字母、数字、点、下划线或连字符")
|
||||
|
||||
|
||||
def validate_role(role: str) -> None:
|
||||
if role not in VALID_ROLES:
|
||||
raise ValueError("role must be admin, engineer, or viewer")
|
||||
|
||||
|
||||
def validate_password(password: str) -> None:
|
||||
if len(password) < 12:
|
||||
raise ValueError("密码至少需要 12 个字符")
|
||||
classes = [any(c.islower() for c in password), any(c.isupper() for c in password), any(c.isdigit() for c in password), any(not c.isalnum() for c in password)]
|
||||
if sum(classes) < 3:
|
||||
raise ValueError("密码需包含大小写字母、数字或符号中的至少三类")
|
||||
|
||||
|
||||
def _password_hasher():
|
||||
try:
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.low_level import Type
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("auth_enabled requires argon2-cffi") from exc
|
||||
return PasswordHasher(time_cost=3, memory_cost=65536, parallelism=2, hash_len=32, salt_len=16, type=Type.ID)
|
||||
|
||||
|
||||
def _write_bootstrap_file(path: Path, username: str, password: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"username={username}\npassword={password}\nforce_password_change=true\n", encoding="utf-8")
|
||||
path.chmod(0o600)
|
||||
@ -33,6 +33,24 @@ class PlatformConfig:
|
||||
cuda_home: Path = Path("/usr/local/cuda-12.1")
|
||||
runtime_env_script: Path = Path("/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh")
|
||||
allowed_path_roots: tuple[Path, ...] = ()
|
||||
worker_mode: str = "embedded"
|
||||
worker_id: str = "local-worker"
|
||||
production_mode: bool = False
|
||||
auth_enabled: bool = False
|
||||
secure_cookies: bool = False
|
||||
session_ttl_seconds: int = 43200
|
||||
bootstrap_admin_username: str = "admin"
|
||||
bootstrap_admin_password: str = ""
|
||||
bos_endpoint: str = "https://bj.bcebos.com"
|
||||
bos_bucket: str = ""
|
||||
bos_prefix: str = "real2sim-poc"
|
||||
bos_access_key_id: str = ""
|
||||
bos_secret_access_key: str = ""
|
||||
discoverse_commit: str = "d67f47c084aba0e0cf422a8725235f8b9238655a"
|
||||
act_commit: str = "f8ab39c8d829d85571ab6d0250412ba4d0698fa0"
|
||||
anysplat_commit: str = "5f5e208a7dd57d52e43ea0d553a95eab526e8775"
|
||||
anysplat_model_revision: str = "d2e8c343672646041ad4ea518184968f94362f01"
|
||||
vggt_model_revision: str = "860abec7937da0a4c03c41d3c269c366e82abdf9"
|
||||
|
||||
@property
|
||||
def db_path(self) -> Path:
|
||||
@ -62,6 +80,14 @@ class PlatformConfig:
|
||||
def manifests_dir(self) -> Path:
|
||||
return self.workspace_dir / "manifests"
|
||||
|
||||
@property
|
||||
def upload_parts_dir(self) -> Path:
|
||||
return self.workspace_dir / "upload_parts"
|
||||
|
||||
@property
|
||||
def auth_bootstrap_path(self) -> Path:
|
||||
return self.workspace_dir / "bootstrap-admin.txt"
|
||||
|
||||
@property
|
||||
def effective_allowed_path_roots(self) -> tuple[Path, ...]:
|
||||
if self.allowed_path_roots:
|
||||
@ -80,9 +106,7 @@ class PlatformConfig:
|
||||
|
||||
def load_config(path: Path | None = None) -> PlatformConfig:
|
||||
config_path = path or Path(os.environ.get("REAL2SIM_PLATFORM_CONFIG", ""))
|
||||
if not config_path or not config_path.is_file():
|
||||
return PlatformConfig()
|
||||
values = _parse_simple_toml(config_path.read_text(encoding="utf-8"))
|
||||
values = _parse_simple_toml(config_path.read_text(encoding="utf-8")) if config_path and config_path.is_file() else {}
|
||||
app_root = Path(str(values.get("app_root", APP_ROOT)))
|
||||
workspace_dir = Path(str(values.get("workspace_dir", WORKSPACE_ROOT)))
|
||||
return PlatformConfig(
|
||||
@ -106,11 +130,29 @@ def load_config(path: Path | None = None) -> PlatformConfig:
|
||||
cuda_home=Path(str(values.get("cuda_home", "/usr/local/cuda-12.1"))),
|
||||
runtime_env_script=Path(str(values.get("runtime_env_script", "/opt/gs_playground_real2sim_48q/scripts/real2sim_wuying_paper_env.sh"))),
|
||||
allowed_path_roots=_parse_path_tuple(str(values.get("allowed_path_roots", ""))),
|
||||
worker_mode=str(values.get("worker_mode", os.environ.get("REAL2SIM_WORKER_MODE", "embedded"))),
|
||||
worker_id=str(values.get("worker_id", os.environ.get("REAL2SIM_WORKER_ID", "local-worker"))),
|
||||
production_mode=_as_bool(values.get("production_mode", os.environ.get("REAL2SIM_PRODUCTION_MODE", "0"))),
|
||||
auth_enabled=_as_bool(values.get("auth_enabled", os.environ.get("REAL2SIM_AUTH_ENABLED", "0"))),
|
||||
secure_cookies=_as_bool(values.get("secure_cookies", os.environ.get("REAL2SIM_SECURE_COOKIES", "0"))),
|
||||
session_ttl_seconds=int(values.get("session_ttl_seconds", os.environ.get("REAL2SIM_SESSION_TTL_SECONDS", "43200"))),
|
||||
bootstrap_admin_username=str(values.get("bootstrap_admin_username", os.environ.get("REAL2SIM_BOOTSTRAP_ADMIN_USERNAME", "admin"))),
|
||||
bootstrap_admin_password=str(values.get("bootstrap_admin_password", os.environ.get("REAL2SIM_BOOTSTRAP_ADMIN_PASSWORD", ""))),
|
||||
bos_endpoint=str(values.get("bos_endpoint", os.environ.get("BOS_ENDPOINT", "https://bj.bcebos.com"))),
|
||||
bos_bucket=str(values.get("bos_bucket", os.environ.get("BOS_BUCKET", ""))),
|
||||
bos_prefix=str(values.get("bos_prefix", os.environ.get("REAL2SIM_BOS_PREFIX", "real2sim-poc"))),
|
||||
bos_access_key_id=str(values.get("bos_access_key_id", os.environ.get("BOS_ACCESS_KEY_ID", ""))),
|
||||
bos_secret_access_key=str(values.get("bos_secret_access_key", os.environ.get("BOS_SECRET_ACCESS_KEY", ""))),
|
||||
discoverse_commit=str(values.get("discoverse_commit", os.environ.get("REAL2SIM_DISCOVERSE_COMMIT", "d67f47c084aba0e0cf422a8725235f8b9238655a"))),
|
||||
act_commit=str(values.get("act_commit", os.environ.get("REAL2SIM_ACT_COMMIT", "f8ab39c8d829d85571ab6d0250412ba4d0698fa0"))),
|
||||
anysplat_commit=str(values.get("anysplat_commit", os.environ.get("REAL2SIM_ANYSPLAT_COMMIT", "5f5e208a7dd57d52e43ea0d553a95eab526e8775"))),
|
||||
anysplat_model_revision=str(values.get("anysplat_model_revision", os.environ.get("REAL2SIM_ANYSPLAT_MODEL_REVISION", "d2e8c343672646041ad4ea518184968f94362f01"))),
|
||||
vggt_model_revision=str(values.get("vggt_model_revision", os.environ.get("REAL2SIM_VGGT_MODEL_REVISION", "860abec7937da0a4c03c41d3c269c366e82abdf9"))),
|
||||
)
|
||||
|
||||
|
||||
def ensure_workspace(config: PlatformConfig) -> None:
|
||||
for path in [config.workspace_dir, config.uploads_dir, config.datasets_dir, config.scenes_dir, config.runs_dir, config.models_dir, config.manifests_dir]:
|
||||
for path in [config.workspace_dir, config.uploads_dir, config.upload_parts_dir, config.datasets_dir, config.scenes_dir, config.runs_dir, config.models_dir, config.manifests_dir]:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
@ -138,6 +180,21 @@ def write_default_config(path: Path, config: PlatformConfig | None = None) -> No
|
||||
f'cuda_home = "{cfg.cuda_home.as_posix()}"',
|
||||
f'runtime_env_script = "{cfg.runtime_env_script.as_posix()}"',
|
||||
f'allowed_path_roots = "{_format_path_tuple(cfg.effective_allowed_path_roots)}"',
|
||||
f'worker_mode = "{cfg.worker_mode}"',
|
||||
f'worker_id = "{cfg.worker_id}"',
|
||||
f"production_mode = {str(cfg.production_mode).lower()}",
|
||||
f"auth_enabled = {str(cfg.auth_enabled).lower()}",
|
||||
f"secure_cookies = {str(cfg.secure_cookies).lower()}",
|
||||
f"session_ttl_seconds = {cfg.session_ttl_seconds}",
|
||||
f'bootstrap_admin_username = "{cfg.bootstrap_admin_username}"',
|
||||
f'bos_endpoint = "{cfg.bos_endpoint}"',
|
||||
f'bos_bucket = "{cfg.bos_bucket}"',
|
||||
f'bos_prefix = "{cfg.bos_prefix}"',
|
||||
f'discoverse_commit = "{cfg.discoverse_commit}"',
|
||||
f'act_commit = "{cfg.act_commit}"',
|
||||
f'anysplat_commit = "{cfg.anysplat_commit}"',
|
||||
f'anysplat_model_revision = "{cfg.anysplat_model_revision}"',
|
||||
f'vggt_model_revision = "{cfg.vggt_model_revision}"',
|
||||
"",
|
||||
]
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
@ -169,6 +226,9 @@ def _parse_simple_toml(text: str) -> dict[str, Any]:
|
||||
if value.startswith('"') and value.endswith('"'):
|
||||
values[key] = value[1:-1]
|
||||
else:
|
||||
if value.lower() in {"true", "false"}:
|
||||
values[key] = value.lower() == "true"
|
||||
continue
|
||||
try:
|
||||
values[key] = int(value)
|
||||
except ValueError:
|
||||
@ -184,3 +244,9 @@ def _parse_path_tuple(value: str) -> tuple[Path, ...]:
|
||||
|
||||
def _format_path_tuple(paths: tuple[Path, ...]) -> str:
|
||||
return ":".join(path.as_posix() for path in paths)
|
||||
|
||||
|
||||
def _as_bool(value: Any) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
379
src/real2sim_platform/rm65_controller.py
Normal file
379
src/real2sim_platform/rm65_controller.py
Normal file
@ -0,0 +1,379 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class EpisodeResult:
|
||||
success: bool
|
||||
lift_height: float
|
||||
place_error: float
|
||||
bilateral_contact_steps: int
|
||||
observations: list[np.ndarray]
|
||||
actions: list[np.ndarray]
|
||||
contacts: list[np.ndarray]
|
||||
images: list[np.ndarray]
|
||||
trajectory: list[dict[str, Any]]
|
||||
initial_cube: list[float]
|
||||
target: list[float]
|
||||
|
||||
|
||||
class RM65PickPlaceRuntime:
|
||||
def __init__(self, scene_path: Path, *, width: int, height: int, seed: int) -> None:
|
||||
import mujoco
|
||||
|
||||
self.mujoco = mujoco
|
||||
self.model = mujoco.MjModel.from_xml_path(str(scene_path))
|
||||
self.data = mujoco.MjData(self.model)
|
||||
self.renderer = mujoco.Renderer(self.model, height=height, width=width)
|
||||
self.rng = np.random.default_rng(seed)
|
||||
self.site_id = self._id(mujoco.mjtObj.mjOBJ_SITE, "gripper")
|
||||
self.cube_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "pick_cube_free")
|
||||
self.cube_qadr = int(self.model.jnt_qposadr[self.cube_joint])
|
||||
self.cube_geom = self._id(mujoco.mjtObj.mjOBJ_GEOM, "pick_cube_geom")
|
||||
self.finger_geoms = {
|
||||
self._id(mujoco.mjtObj.mjOBJ_GEOM, "left_finger_pad_z_axis"),
|
||||
self._id(mujoco.mjtObj.mjOBJ_GEOM, "right_finger_pad_z_axis"),
|
||||
}
|
||||
self.target_body = self._id(mujoco.mjtObj.mjOBJ_BODY, "place_target")
|
||||
self.gripper_joint = self._id(mujoco.mjtObj.mjOBJ_JOINT, "gripper_Left_1_Joint")
|
||||
self.gripper_qadr = int(self.model.jnt_qposadr[self.gripper_joint])
|
||||
self.joint_ids = [self._id(mujoco.mjtObj.mjOBJ_JOINT, f"joint_{index}") for index in range(1, 7)]
|
||||
self.qpos_adrs = [int(self.model.jnt_qposadr[joint]) for joint in self.joint_ids]
|
||||
self.dof_adrs = [int(self.model.jnt_dofadr[joint]) for joint in self.joint_ids]
|
||||
self.ctrl_ranges = self.model.actuator_ctrlrange.copy()
|
||||
|
||||
def _id(self, kind: Any, name: str) -> int:
|
||||
value = self.mujoco.mj_name2id(self.model, kind, name)
|
||||
if value < 0:
|
||||
raise ValueError(f"required RM65 scene object is missing: {name}")
|
||||
return int(value)
|
||||
|
||||
def close(self) -> None:
|
||||
self.renderer.close()
|
||||
|
||||
def run_episode(self, *, max_steps: int) -> EpisodeResult:
|
||||
mujoco = self.mujoco
|
||||
mujoco.mj_resetDataKeyframe(self.model, self.data, 0)
|
||||
cube = np.array([self.rng.uniform(0.645, 0.655), self.rng.uniform(-0.010, 0.010), 0.33], dtype=float)
|
||||
target = np.array([self.rng.uniform(0.42, 0.51), self.rng.uniform(0.13, 0.22), 0.31], dtype=float)
|
||||
self.data.qpos[self.cube_qadr : self.cube_qadr + 3] = cube
|
||||
self.data.qpos[self.cube_qadr + 3 : self.cube_qadr + 7] = np.array([1.0, 0.0, 0.0, 0.0])
|
||||
self.data.qvel[self.model.jnt_dofadr[self.cube_joint] : self.model.jnt_dofadr[self.cube_joint] + 6] = 0
|
||||
self.model.body_pos[self.target_body] = np.array([target[0], target[1], 0.335])
|
||||
mujoco.mj_forward(self.model, self.data)
|
||||
for _ in range(120):
|
||||
mujoco.mj_step(self.model, self.data)
|
||||
settled_z = float(self.data.qpos[self.cube_qadr + 2])
|
||||
|
||||
# The named site is about 3 cm above the cube center at bilateral pad contact.
|
||||
grasp_site = np.array([cube[0] - 0.014, cube[1] - 0.020, settled_z + 0.020])
|
||||
# Contact-calibrated negative-wrist branch for the packaged RM65 gripper.
|
||||
# This seed is a robot calibration pose, not an object-pose override.
|
||||
seed_q = np.array([-1.2468, -0.1011, 1.8666, -1.9230, -1.5982, 1.5636], dtype=float)
|
||||
q_backup = self.data.qpos.copy()
|
||||
self.data.qpos[self.qpos_adrs] = seed_q
|
||||
mujoco.mj_forward(self.model, self.data)
|
||||
grasp_orientation = self.data.site_xmat[self.site_id].reshape(3, 3).copy()
|
||||
self.data.qpos[:] = q_backup
|
||||
mujoco.mj_forward(self.model, self.data)
|
||||
|
||||
phases = [
|
||||
("approach", grasp_site + np.array([0.0, 0.0, 0.20]), 0.0, 24),
|
||||
("descend", grasp_site, 0.0, 24),
|
||||
("grasp", grasp_site, -0.28, 30),
|
||||
("lift", grasp_site + np.array([0.0, 0.0, 0.20]), -0.28, 60),
|
||||
("transfer", target + np.array([-0.040, 0.0, 0.21]), -0.28, 34),
|
||||
("place", target + np.array([-0.040, 0.0, 0.02]), -0.28, 26),
|
||||
("release", target + np.array([-0.040, 0.0, 0.02]), 0.0, 18),
|
||||
("retreat", target + np.array([-0.040, 0.0, 0.20]), 0.0, 20),
|
||||
]
|
||||
scale = max(0.25, max_steps / sum(phase[3] for phase in phases))
|
||||
observations: list[np.ndarray] = []
|
||||
actions: list[np.ndarray] = []
|
||||
contacts: list[np.ndarray] = []
|
||||
images: list[np.ndarray] = []
|
||||
trajectory: list[dict[str, Any]] = []
|
||||
bilateral_contact_steps = 0
|
||||
max_cube_z = settled_z
|
||||
step_index = 0
|
||||
for phase, position, gripper, nominal_steps in phases:
|
||||
target_q = self.solve_ik_6d(position, grasp_orientation, seed_q=seed_q)
|
||||
phase_steps = max(2, int(round(nominal_steps * scale)))
|
||||
start_ctrl = self.data.ctrl[:6].copy()
|
||||
for local_step in range(phase_steps):
|
||||
alpha = _smoothstep((local_step + 1) / phase_steps)
|
||||
action = np.empty(7, dtype=np.float32)
|
||||
action[:6] = start_ctrl * (1 - alpha) + target_q * alpha
|
||||
action[6] = gripper
|
||||
self.data.ctrl[:7] = action
|
||||
# The RM65 model has conservative actuator force limits; each policy
|
||||
# command is held long enough for the physical joints to settle.
|
||||
for _ in range(35):
|
||||
mujoco.mj_step(self.model, self.data)
|
||||
touching = self.finger_contacts()
|
||||
bilateral_contact_steps += int(len(touching) == 2)
|
||||
max_cube_z = max(max_cube_z, float(self.data.qpos[self.cube_qadr + 2]))
|
||||
self.renderer.update_scene(self.data, camera="overview")
|
||||
frame = self.renderer.render().copy()
|
||||
observation = np.concatenate([self.data.qpos[self.qpos_adrs], [self.data.qpos[self.gripper_qadr]]]).astype(np.float32)
|
||||
observations.append(observation)
|
||||
actions.append(action.copy())
|
||||
contacts.append(np.array([int(geom in touching) for geom in sorted(self.finger_geoms)], dtype=np.uint8))
|
||||
images.append(frame)
|
||||
trajectory.append(
|
||||
{
|
||||
"step": step_index,
|
||||
"time": float(self.data.time),
|
||||
"phase": phase,
|
||||
"qpos": observation.tolist(),
|
||||
"action": action.tolist(),
|
||||
"cube_pos": self.data.qpos[self.cube_qadr : self.cube_qadr + 3].astype(float).tolist(),
|
||||
"gripper_pos": self.data.site_xpos[self.site_id].astype(float).tolist(),
|
||||
"bilateral_contact": len(touching) == 2,
|
||||
"touching_fingers": sorted(int(item) for item in touching),
|
||||
"finger_positions": [self.data.geom_xpos[item].astype(float).tolist() for item in sorted(self.finger_geoms)],
|
||||
}
|
||||
)
|
||||
step_index += 1
|
||||
|
||||
final_cube = self.data.qpos[self.cube_qadr : self.cube_qadr + 3].copy()
|
||||
lift_height = float(max_cube_z - settled_z)
|
||||
place_error = float(np.linalg.norm(final_cube - target))
|
||||
success = bilateral_contact_steps > 0 and lift_height >= 0.10 and place_error <= 0.05
|
||||
return EpisodeResult(
|
||||
success=success,
|
||||
lift_height=lift_height,
|
||||
place_error=place_error,
|
||||
bilateral_contact_steps=bilateral_contact_steps,
|
||||
observations=observations,
|
||||
actions=actions,
|
||||
contacts=contacts,
|
||||
images=images,
|
||||
trajectory=trajectory,
|
||||
initial_cube=cube.tolist(),
|
||||
target=target.tolist(),
|
||||
)
|
||||
|
||||
def finger_contacts(self) -> set[int]:
|
||||
touching: set[int] = set()
|
||||
for index in range(self.data.ncon):
|
||||
contact = self.data.contact[index]
|
||||
pair = {int(contact.geom1), int(contact.geom2)}
|
||||
if self.cube_geom in pair:
|
||||
touching.update(pair & self.finger_geoms)
|
||||
return touching
|
||||
|
||||
def solve_position_ik(self, target_position: np.ndarray) -> np.ndarray:
|
||||
q_backup = self.data.qpos.copy()
|
||||
q = self.data.qpos[self.qpos_adrs].copy()
|
||||
for _ in range(120):
|
||||
self.data.qpos[self.qpos_adrs] = q
|
||||
self.mujoco.mj_forward(self.model, self.data)
|
||||
error = target_position - self.data.site_xpos[self.site_id]
|
||||
if np.linalg.norm(error) < 0.004:
|
||||
break
|
||||
jacp = np.zeros((3, self.model.nv))
|
||||
jacr = np.zeros((3, self.model.nv))
|
||||
self.mujoco.mj_jacSite(self.model, self.data, jacp, jacr, self.site_id)
|
||||
jacobian = jacp[:, self.dof_adrs]
|
||||
q += np.clip(_damped_least_squares(jacobian, error, 2e-3), -0.06, 0.06)
|
||||
q = self._clip_arm(q)
|
||||
self.data.qpos[:] = q_backup
|
||||
self.mujoco.mj_forward(self.model, self.data)
|
||||
return q
|
||||
|
||||
def solve_ik_6d(self, target_position: np.ndarray, target_rotation: np.ndarray, *, seed_q: np.ndarray | None = None) -> np.ndarray:
|
||||
q_backup = self.data.qpos.copy()
|
||||
q = seed_q.copy() if seed_q is not None else self.data.qpos[self.qpos_adrs].copy()
|
||||
for _ in range(160):
|
||||
self.data.qpos[self.qpos_adrs] = q
|
||||
self.mujoco.mj_forward(self.model, self.data)
|
||||
position_error = target_position - self.data.site_xpos[self.site_id]
|
||||
current_rotation = self.data.site_xmat[self.site_id].reshape(3, 3)
|
||||
rotation_error = _rotation_vector(target_rotation @ current_rotation.T)
|
||||
error = np.concatenate([position_error, 0.45 * rotation_error])
|
||||
if np.linalg.norm(position_error) < 0.005 and np.linalg.norm(rotation_error) < 0.04:
|
||||
break
|
||||
jacp = np.zeros((3, self.model.nv))
|
||||
jacr = np.zeros((3, self.model.nv))
|
||||
self.mujoco.mj_jacSite(self.model, self.data, jacp, jacr, self.site_id)
|
||||
jacobian = np.vstack([jacp[:, self.dof_adrs], 0.45 * jacr[:, self.dof_adrs]])
|
||||
q += np.clip(_damped_least_squares(jacobian, error, 4e-3), -0.05, 0.05)
|
||||
q = self._clip_arm(q)
|
||||
self.data.qpos[:] = q_backup
|
||||
self.mujoco.mj_forward(self.model, self.data)
|
||||
return q
|
||||
|
||||
def _clip_arm(self, q: np.ndarray) -> np.ndarray:
|
||||
return np.array([np.clip(q[index], self.ctrl_ranges[index, 0], self.ctrl_ranges[index, 1]) for index in range(6)])
|
||||
|
||||
|
||||
def collect(scene_path: Path, out_dir: Path, *, required_successes: int, max_attempts: int, max_steps: int, seed: int, width: int, height: int) -> dict[str, Any]:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
runtime = RM65PickPlaceRuntime(scene_path, width=width, height=height, seed=seed)
|
||||
reports: list[dict[str, Any]] = []
|
||||
success_count = 0
|
||||
try:
|
||||
for attempt in range(max_attempts):
|
||||
result = runtime.run_episode(max_steps=max_steps)
|
||||
episode_report = {
|
||||
"attempt": attempt,
|
||||
"success": result.success,
|
||||
"lift_height": result.lift_height,
|
||||
"place_error": result.place_error,
|
||||
"bilateral_contact_steps": result.bilateral_contact_steps,
|
||||
"initial_cube": result.initial_cube,
|
||||
"target": result.target,
|
||||
}
|
||||
reports.append(episode_report)
|
||||
trajectory_path = out_dir / f"attempt_{attempt:04d}_trajectory.json"
|
||||
trajectory_path.write_text(json.dumps({"schema": "real2sim.rm65.trajectory.v2", "report": episode_report, "samples": result.trajectory}, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if result.success:
|
||||
_write_hdf5_episode(out_dir / f"episode_{success_count}.hdf5", result)
|
||||
if success_count < 3:
|
||||
_write_preview_video(out_dir / f"episode_{success_count}.mp4", result.images)
|
||||
success_count += 1
|
||||
if success_count >= required_successes:
|
||||
break
|
||||
finally:
|
||||
runtime.close()
|
||||
manifest = {
|
||||
"schema": "real2sim.discoverse_act_dataset.v1",
|
||||
"controller": "rm65_pick_place_6d_ik",
|
||||
"runtime": "mujoco",
|
||||
"scene": str(scene_path),
|
||||
"camera_names": ["overview"],
|
||||
"state_dim": 7,
|
||||
"action_dim": 7,
|
||||
"required_successes": required_successes,
|
||||
"successful_episodes": success_count,
|
||||
"attempts": len(reports),
|
||||
"success_rate": success_count / max(1, len(reports)),
|
||||
"success_definition": {"minimum_lift_m": 0.10, "maximum_place_error_m": 0.05, "bilateral_contact_required": True},
|
||||
"episodes": reports,
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
}
|
||||
(out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return manifest
|
||||
|
||||
|
||||
def _write_hdf5_episode(path: Path, result: EpisodeResult) -> None:
|
||||
import cv2
|
||||
import h5py
|
||||
|
||||
observations = np.asarray(result.observations, dtype=np.float32)
|
||||
actions = np.asarray(result.actions, dtype=np.float32)
|
||||
contacts = np.asarray(result.contacts, dtype=np.uint8)
|
||||
encoded = []
|
||||
for image in result.images:
|
||||
ok, buffer = cv2.imencode(".jpg", image[:, :, ::-1], [int(cv2.IMWRITE_JPEG_QUALITY), 88])
|
||||
if not ok:
|
||||
raise RuntimeError("failed to encode collection frame")
|
||||
encoded.append(buffer.astype(np.uint8))
|
||||
with h5py.File(path, "w") as root:
|
||||
root.attrs["sim"] = True
|
||||
root.attrs["compress"] = True
|
||||
root.attrs["success"] = bool(result.success)
|
||||
root.create_dataset("/observations/qpos", data=observations, dtype="f4")
|
||||
root.create_dataset("/action", data=actions, dtype="f4")
|
||||
root.create_dataset("/observations/contact", data=contacts, dtype="u1")
|
||||
root.create_dataset("/success", data=np.full(len(actions), result.success, dtype=np.uint8))
|
||||
variable = h5py.vlen_dtype(np.dtype("uint8"))
|
||||
dataset = root.create_dataset("/observations/images/overview", (len(encoded),), dtype=variable)
|
||||
for index, value in enumerate(encoded):
|
||||
dataset[index] = value
|
||||
|
||||
|
||||
def _write_preview_video(path: Path, images: list[np.ndarray]) -> None:
|
||||
if not images:
|
||||
raise ValueError("cannot encode an empty rollout")
|
||||
height, width = images[0].shape[:2]
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"rgb24",
|
||||
"-s",
|
||||
f"{width}x{height}",
|
||||
"-r",
|
||||
"20",
|
||||
"-i",
|
||||
"-",
|
||||
"-an",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
str(path),
|
||||
]
|
||||
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
||||
assert process.stdin is not None
|
||||
for image in images:
|
||||
process.stdin.write(np.ascontiguousarray(image, dtype=np.uint8).tobytes())
|
||||
process.stdin.close()
|
||||
stderr = process.stderr.read().decode("utf-8", errors="replace") if process.stderr else ""
|
||||
if process.wait() != 0 or not path.is_file() or path.stat().st_size == 0:
|
||||
raise RuntimeError(f"ffmpeg rollout encoding failed: {stderr[-2000:]}")
|
||||
|
||||
|
||||
def _damped_least_squares(jacobian: np.ndarray, error: np.ndarray, damping: float) -> np.ndarray:
|
||||
identity = np.eye(jacobian.shape[0])
|
||||
return jacobian.T @ np.linalg.solve(jacobian @ jacobian.T + damping * identity, error)
|
||||
|
||||
|
||||
def _rotation_vector(rotation: np.ndarray) -> np.ndarray:
|
||||
cosine = float(np.clip((np.trace(rotation) - 1.0) * 0.5, -1.0, 1.0))
|
||||
angle = math.acos(cosine)
|
||||
if angle < 1e-7:
|
||||
return np.zeros(3)
|
||||
axis = np.array([rotation[2, 1] - rotation[1, 2], rotation[0, 2] - rotation[2, 0], rotation[1, 0] - rotation[0, 1]])
|
||||
denominator = 2.0 * math.sin(angle)
|
||||
return axis * (angle / denominator)
|
||||
|
||||
|
||||
def _smoothstep(value: float) -> float:
|
||||
value = float(np.clip(value, 0.0, 1.0))
|
||||
return value * value * (3.0 - 2.0 * value)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Collect physical RM65 pick-place episodes for DISCOVERSE ACT")
|
||||
parser.add_argument("--scene", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--required-successes", type=int, default=50)
|
||||
parser.add_argument("--max-attempts", type=int, default=150)
|
||||
parser.add_argument("--steps", type=int, default=200)
|
||||
parser.add_argument("--seed", type=int, default=7)
|
||||
parser.add_argument("--width", type=int, default=320)
|
||||
parser.add_argument("--height", type=int, default=240)
|
||||
args = parser.parse_args()
|
||||
manifest = collect(
|
||||
args.scene,
|
||||
args.out,
|
||||
required_successes=args.required_successes,
|
||||
max_attempts=args.max_attempts,
|
||||
max_steps=args.steps,
|
||||
seed=args.seed,
|
||||
width=args.width,
|
||||
height=args.height,
|
||||
)
|
||||
print(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
return 0 if manifest["successful_episodes"] >= manifest["required_successes"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
196
src/real2sim_platform/runtime_gate.py
Normal file
196
src/real2sim_platform/runtime_gate.py
Normal file
@ -0,0 +1,196 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
from .adapters import run_doctor
|
||||
from .config import PlatformConfig, ensure_workspace, load_config
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate the production Real2Sim GPU runtime")
|
||||
parser.add_argument("--mode", choices=["bootstrap", "readiness"], default="readiness")
|
||||
parser.add_argument("--force", action="store_true")
|
||||
args = parser.parse_args()
|
||||
config = load_config()
|
||||
ensure_workspace(config)
|
||||
marker = marker_path(config)
|
||||
if args.mode == "readiness":
|
||||
return readiness(config, marker)
|
||||
if marker.is_file() and not args.force:
|
||||
try:
|
||||
existing = json.loads(marker.read_text(encoding="utf-8"))
|
||||
if existing.get("runtime_key") == runtime_key(config) and existing.get("ok") is True:
|
||||
print(json.dumps(existing, ensure_ascii=False))
|
||||
return 0
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
report = bootstrap(config)
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = marker.with_suffix(".tmp")
|
||||
temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
temporary.replace(marker)
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
def marker_path(config: PlatformConfig) -> Path:
|
||||
root = Path(os.environ.get("REAL2SIM_RUNTIME_GATE_DIR", config.workspace_dir / "runtime_gate"))
|
||||
return root / f"{runtime_key(config)}.json"
|
||||
|
||||
|
||||
def runtime_key(config: PlatformConfig) -> str:
|
||||
raw = ":".join(
|
||||
[
|
||||
config.anysplat_commit,
|
||||
config.anysplat_model_revision,
|
||||
config.vggt_model_revision,
|
||||
config.discoverse_commit,
|
||||
config.act_commit,
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:20]
|
||||
|
||||
|
||||
def readiness(config: PlatformConfig, marker: Path) -> int:
|
||||
doctor = run_doctor(config)
|
||||
if not marker.is_file():
|
||||
print("runtime bootstrap report is missing", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
report = json.loads(marker.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
print(f"runtime bootstrap report is unreadable: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
ready = bool(report.get("ok") and report.get("runtime_key") == runtime_key(config) and doctor.get("ok"))
|
||||
if not ready:
|
||||
print(json.dumps({"bootstrap": report, "doctor": doctor}, ensure_ascii=False), file=sys.stderr)
|
||||
return 0 if ready else 1
|
||||
|
||||
|
||||
def bootstrap(config: PlatformConfig) -> dict[str, Any]:
|
||||
started = time.time()
|
||||
checks: dict[str, Any] = {}
|
||||
doctor = run_doctor(config)
|
||||
checks["doctor"] = {"ok": doctor.get("ok"), "core_checks": doctor.get("core_checks"), "checks": doctor.get("checks")}
|
||||
try:
|
||||
checks["nvidia"] = _nvidia_probe()
|
||||
checks["simulators"] = _simulator_probe()
|
||||
checks["act"] = _act_probe(config)
|
||||
checks["anysplat"] = _anysplat_probe(config)
|
||||
except Exception as exc:
|
||||
checks["failure"] = {"type": type(exc).__name__, "message": str(exc)}
|
||||
ok = bool(doctor.get("ok")) and all(bool(value.get("ok")) for key, value in checks.items() if key != "failure") and "failure" not in checks
|
||||
return {
|
||||
"schema": "real2sim.runtime_gate.v1",
|
||||
"runtime_key": runtime_key(config),
|
||||
"ok": ok,
|
||||
"started_at": started,
|
||||
"finished_at": time.time(),
|
||||
"checks": checks,
|
||||
"revisions": {
|
||||
"anysplat_code": config.anysplat_commit,
|
||||
"anysplat_model": config.anysplat_model_revision,
|
||||
"vggt_model": config.vggt_model_revision,
|
||||
"discoverse": config.discoverse_commit,
|
||||
"act": config.act_commit,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _nvidia_probe() -> dict[str, Any]:
|
||||
completed = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=20,
|
||||
check=True,
|
||||
)
|
||||
import torch
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("PyTorch CUDA is unavailable")
|
||||
value = torch.ones((1024, 1024), device="cuda")
|
||||
checksum = float((value @ value).mean().item())
|
||||
del value
|
||||
torch.cuda.empty_cache()
|
||||
return {"ok": checksum == 1024.0, "device": torch.cuda.get_device_name(0), "nvidia_smi": completed.stdout.strip().splitlines()[0]}
|
||||
|
||||
|
||||
def _simulator_probe() -> dict[str, Any]:
|
||||
xml = """<mujoco model="runtime-gate"><option timestep="0.002"/><worldbody><body pos="0 0 0.2"><freejoint/><geom type="sphere" size="0.03" mass="0.1"/></body><geom type="plane" size="1 1 .1"/></worldbody></mujoco>"""
|
||||
import mujoco
|
||||
|
||||
model = mujoco.MjModel.from_xml_string(xml)
|
||||
data = mujoco.MjData(model)
|
||||
for _ in range(8):
|
||||
mujoco.mj_step(model, data)
|
||||
with tempfile.TemporaryDirectory(prefix="real2sim-runtime-gate-") as temporary:
|
||||
scene_path = Path(temporary) / "scene.xml"
|
||||
scene_path.write_text(xml, encoding="utf-8")
|
||||
from motrixsim import SceneData, load_model, step
|
||||
|
||||
motrix_model = load_model(str(scene_path))
|
||||
motrix_data = SceneData(motrix_model)
|
||||
for _ in range(8):
|
||||
step(motrix_model, motrix_data)
|
||||
return {"ok": True, "mujoco_time": float(data.time), "motrix_timestep": float(motrix_model.options.timestep)}
|
||||
|
||||
|
||||
def _act_probe(config: PlatformConfig) -> dict[str, Any]:
|
||||
act_root = config.discoverse_root / "policies" / "act"
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = os.pathsep.join([str(act_root), str(config.discoverse_root), env.get("PYTHONPATH", "")]).rstrip(os.pathsep)
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "-c", "import torch; from policy import ACTPolicy; print(torch.__version__); assert ACTPolicy is not None and torch.cuda.is_available()"],
|
||||
cwd=act_root,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
check=True,
|
||||
)
|
||||
return {"ok": True, "torch": completed.stdout.strip().splitlines()[-1]}
|
||||
|
||||
|
||||
def _anysplat_probe(config: PlatformConfig) -> dict[str, Any]:
|
||||
cache_dir = Path(os.environ.get("HF_HOME", config.workspace_dir / "cache" / "huggingface"))
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
anysplat_snapshot = Path(
|
||||
snapshot_download(
|
||||
repo_id="lhjiang/anysplat",
|
||||
revision=config.anysplat_model_revision,
|
||||
cache_dir=cache_dir,
|
||||
)
|
||||
)
|
||||
snapshot_download(repo_id="facebook/VGGT-1B", revision=config.vggt_model_revision, cache_dir=cache_dir)
|
||||
if str(config.anysplat_root) not in sys.path:
|
||||
sys.path.insert(0, str(config.anysplat_root))
|
||||
from src.model.model.anysplat import AnySplat
|
||||
|
||||
model = AnySplat.from_pretrained(str(anysplat_snapshot))
|
||||
model = model.to("cuda").eval()
|
||||
parameters = sum(parameter.numel() for parameter in model.parameters())
|
||||
if parameters <= 0:
|
||||
raise RuntimeError("AnySplat model loaded with zero parameters")
|
||||
del model
|
||||
gc.collect()
|
||||
import torch
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
return {"ok": True, "parameter_count": parameters, "snapshot": anysplat_snapshot.name}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -1,24 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from collections import defaultdict, deque
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import math
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import zipfile
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse, PlainTextResponse
|
||||
from fastapi import FastAPI, File, HTTPException, Request, Response, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
|
||||
from . import __version__
|
||||
from .adapters import SPARK_VENDOR_PATH, collect_task_reports, enrich_scene_manifest_assets, run_doctor, scan_dataset
|
||||
from .auth import AuthManager, AuthenticationError, SESSION_COOKIE, validate_password, validate_role, validate_username
|
||||
from .config import PlatformConfig, ensure_workspace, is_path_allowed, load_config, write_default_config
|
||||
from .storage import BosStorage, StorageBoundaryError, StorageUnavailable, safe_object_name, sha256_file
|
||||
from .store import PlatformStore
|
||||
from .worker import TaskWorker
|
||||
|
||||
@ -63,24 +72,95 @@ class BulkActionRequest(BaseModel):
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
current_password: str = ""
|
||||
new_password: str
|
||||
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
role: str = "engineer"
|
||||
|
||||
|
||||
class UpdateUserRequest(BaseModel):
|
||||
role: str | None = None
|
||||
disabled: bool | None = None
|
||||
reset_password: str | None = None
|
||||
|
||||
|
||||
class CreateUploadRequest(BaseModel):
|
||||
filename: str
|
||||
content_type: str = "application/octet-stream"
|
||||
total_size: int
|
||||
chunk_size: int = 8 * 1024 * 1024
|
||||
checksum: str = ""
|
||||
|
||||
|
||||
class ImportBosRequest(BaseModel):
|
||||
prefix: str
|
||||
name: str | None = None
|
||||
max_objects: int = 1000
|
||||
|
||||
|
||||
def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
cfg = config or load_config()
|
||||
ensure_workspace(cfg)
|
||||
store = PlatformStore(cfg.db_path)
|
||||
worker = TaskWorker(store, cfg)
|
||||
app = FastAPI(title="Real2Sim Platform", version="0.1.0")
|
||||
auth = AuthManager(store, cfg)
|
||||
storage = BosStorage(cfg)
|
||||
auth.bootstrap_admin()
|
||||
login_failures: dict[str, deque[float]] = defaultdict(deque)
|
||||
login_failure_lock = threading.Lock()
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
if cfg.worker_mode == "embedded":
|
||||
worker.start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if cfg.worker_mode == "embedded":
|
||||
worker.stop()
|
||||
|
||||
app = FastAPI(title="Real2Sim Platform", version="0.1.0", lifespan=lifespan)
|
||||
static_dir = Path(__file__).parent / "static"
|
||||
app.mount("/static", StaticFiles(directory=static_dir), name="static")
|
||||
app.mount("/datasets", StaticFiles(directory=cfg.datasets_dir), name="datasets")
|
||||
|
||||
@app.on_event("startup")
|
||||
def _startup() -> None:
|
||||
store.recover_interrupted_tasks()
|
||||
worker.start()
|
||||
public_paths = {"/", "/api/health", "/api/version", "/api/capabilities", "/api/auth/login"}
|
||||
|
||||
@app.on_event("shutdown")
|
||||
def _shutdown() -> None:
|
||||
worker.stop()
|
||||
@app.middleware("http")
|
||||
async def authenticate_request(request: Request, call_next):
|
||||
path = request.url.path
|
||||
if not cfg.auth_enabled or path in public_paths or path.startswith("/static/"):
|
||||
request.state.user = auth.authenticate(None)
|
||||
return await call_next(request)
|
||||
token = request.cookies.get(SESSION_COOKIE)
|
||||
user = auth.authenticate(token)
|
||||
if user is None:
|
||||
return JSONResponse(status_code=401, content={"detail": "请先登录"})
|
||||
request.state.user = user
|
||||
if user.get("must_change_password") and path not in {"/api/auth/me", "/api/auth/change-password", "/api/auth/logout"}:
|
||||
return JSONResponse(status_code=428, content={"detail": "首次登录必须先修改密码", "code": "password_change_required"})
|
||||
if request.method not in {"GET", "HEAD", "OPTIONS"}:
|
||||
self_service_paths = {"/api/auth/change-password", "/api/auth/logout"}
|
||||
if user.get("role") == "viewer" and path not in self_service_paths:
|
||||
return JSONResponse(status_code=403, content={"detail": "只读用户不能执行修改操作"})
|
||||
if path.startswith("/api/users") and user.get("role") != "admin":
|
||||
return JSONResponse(status_code=403, content={"detail": "仅管理员可以管理用户"})
|
||||
origin = request.headers.get("origin")
|
||||
origin_host = urlparse(origin).hostname if origin else None
|
||||
if origin_host and origin_host != request.url.hostname:
|
||||
return JSONResponse(status_code=403, content={"detail": "请求来源校验失败"})
|
||||
if path.startswith("/api/users") and user.get("role") != "admin":
|
||||
return JSONResponse(status_code=403, content={"detail": "仅管理员可以管理用户"})
|
||||
return await call_next(request)
|
||||
|
||||
@app.get("/")
|
||||
def index():
|
||||
@ -88,7 +168,110 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
|
||||
@app.get("/api/health")
|
||||
def health() -> dict[str, Any]:
|
||||
return {"ok": True, "workspace": str(cfg.workspace_dir)}
|
||||
return {"ok": True, "worker_mode": cfg.worker_mode, "production_mode": cfg.production_mode}
|
||||
|
||||
@app.post("/api/auth/login")
|
||||
def login(payload: LoginRequest, request: Request, response: Response) -> dict[str, Any]:
|
||||
if not cfg.auth_enabled:
|
||||
return {"user": auth.authenticate(None), "auth_enabled": False}
|
||||
login_key = f"{request.client.host if request.client else 'unknown'}:{payload.username.strip().lower()}"
|
||||
now = time.time()
|
||||
with login_failure_lock:
|
||||
attempts = login_failures[login_key]
|
||||
while attempts and now - attempts[0] > 900:
|
||||
attempts.popleft()
|
||||
if len(attempts) >= 8:
|
||||
raise HTTPException(status_code=429, detail="登录失败次数过多,请 15 分钟后再试")
|
||||
try:
|
||||
user, token = auth.login(payload.username, payload.password)
|
||||
except AuthenticationError as exc:
|
||||
with login_failure_lock:
|
||||
login_failures[login_key].append(time.time())
|
||||
time.sleep(0.2)
|
||||
raise HTTPException(status_code=401, detail=str(exc)) from exc
|
||||
with login_failure_lock:
|
||||
login_failures.pop(login_key, None)
|
||||
response.set_cookie(
|
||||
SESSION_COOKIE,
|
||||
token,
|
||||
max_age=cfg.session_ttl_seconds,
|
||||
httponly=True,
|
||||
secure=cfg.secure_cookies,
|
||||
samesite="lax",
|
||||
path="/",
|
||||
)
|
||||
return {"user": user, "auth_enabled": True}
|
||||
|
||||
@app.post("/api/auth/logout")
|
||||
def logout(request: Request, response: Response) -> dict[str, bool]:
|
||||
auth.logout(request.cookies.get(SESSION_COOKIE))
|
||||
response.delete_cookie(SESSION_COOKIE, path="/")
|
||||
return {"ok": True}
|
||||
|
||||
@app.get("/api/auth/me")
|
||||
def auth_me(request: Request) -> dict[str, Any]:
|
||||
return {"user": request.state.user, "auth_enabled": cfg.auth_enabled}
|
||||
|
||||
@app.post("/api/auth/change-password")
|
||||
def change_password(payload: ChangePasswordRequest, request: Request, response: Response) -> dict[str, Any]:
|
||||
try:
|
||||
user = auth.change_password(
|
||||
request.state.user["id"],
|
||||
current_password=payload.current_password,
|
||||
new_password=payload.new_password,
|
||||
)
|
||||
except (AuthenticationError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
response.delete_cookie(SESSION_COOKIE, path="/")
|
||||
return {"ok": True, "user": user, "login_required": True}
|
||||
|
||||
@app.get("/api/users")
|
||||
def list_users() -> list[dict[str, Any]]:
|
||||
return store.list_users()
|
||||
|
||||
@app.post("/api/users")
|
||||
def create_user(payload: CreateUserRequest) -> dict[str, Any]:
|
||||
try:
|
||||
return auth.create_user(username=payload.username, password=payload.password, role=payload.role)
|
||||
except (ValueError, __import__("sqlite3").IntegrityError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.patch("/api/users/{user_id}")
|
||||
def update_user(user_id: str, payload: UpdateUserRequest, request: Request) -> dict[str, Any]:
|
||||
try:
|
||||
target = store.get_user(user_id)
|
||||
if payload.role is not None:
|
||||
validate_role(payload.role)
|
||||
if target["id"] == request.state.user["id"] and payload.disabled:
|
||||
raise ValueError("不能禁用当前登录账号")
|
||||
if target["role"] == "admin" and (payload.disabled or payload.role not in {None, "admin"}) and _active_admin_count(store) <= 1:
|
||||
raise ValueError("系统至少需要一个可用管理员")
|
||||
if payload.reset_password:
|
||||
validate_password(payload.reset_password)
|
||||
return auth.change_password(user_id, current_password=None, new_password=payload.reset_password, admin_reset=True)
|
||||
result = store.update_user(user_id, role=payload.role, disabled=payload.disabled)
|
||||
if payload.disabled:
|
||||
store.delete_user_sessions(user_id)
|
||||
return result
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="用户不存在") from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.delete("/api/users/{user_id}")
|
||||
def delete_user(user_id: str, request: Request) -> dict[str, bool]:
|
||||
try:
|
||||
target = store.get_user(user_id)
|
||||
if target["id"] == request.state.user["id"]:
|
||||
raise ValueError("不能删除当前登录账号")
|
||||
if target["role"] == "admin" and not target["disabled"] and _active_admin_count(store) <= 1:
|
||||
raise ValueError("系统至少需要一个可用管理员")
|
||||
store.delete_user(user_id)
|
||||
return {"ok": True}
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="用户不存在") from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get("/api/version")
|
||||
def version() -> dict[str, Any]:
|
||||
@ -120,7 +303,227 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
|
||||
@app.get("/api/doctor")
|
||||
def doctor() -> dict[str, Any]:
|
||||
return run_doctor(cfg)
|
||||
result = run_doctor(cfg)
|
||||
bos_health = storage.health()
|
||||
sqlite_wal = _sqlite_journal_mode(store) == "wal"
|
||||
result["checks"]["bos_storage"] = bos_health
|
||||
result["checks"]["sqlite_wal"] = sqlite_wal
|
||||
result["checks"]["worker_mode"] = {"ready": cfg.worker_mode in {"embedded", "api", "worker"}, "value": cfg.worker_mode}
|
||||
heartbeats = store.list_worker_heartbeats()
|
||||
latest = heartbeats[0] if heartbeats else None
|
||||
heartbeat_age = time.time() - float(latest["updated_at"]) if latest else None
|
||||
worker_ready = bool(latest and heartbeat_age is not None and heartbeat_age < 30 and latest["payload"].get("ready"))
|
||||
result["worker"] = {"ready": worker_ready, "age_seconds": heartbeat_age, "latest": latest}
|
||||
if cfg.production_mode and cfg.worker_mode == "api":
|
||||
result["ok"] = bool(bos_health.get("ready") and sqlite_wal and worker_ready)
|
||||
result["core_checks"] = ["bos_storage", "sqlite_wal", "worker"]
|
||||
return result
|
||||
|
||||
@app.get("/api/capabilities")
|
||||
def capabilities() -> dict[str, Any]:
|
||||
return _capabilities(cfg, storage, store)
|
||||
|
||||
@app.get("/api/recipes")
|
||||
def recipes() -> dict[str, Any]:
|
||||
capabilities_payload = _capabilities(cfg, storage, store)
|
||||
act_ready = bool(capabilities_payload["runtimes"]["discoverse_act"]["ready"])
|
||||
rm65_ready = bool(capabilities_payload["runtimes"]["rm65_controller"]["ready"])
|
||||
return {
|
||||
"collection": [
|
||||
{
|
||||
"id": "rm65_pick_place_6d_ik",
|
||||
"name": "RM65 茶几抓取放置",
|
||||
"enabled": rm65_ready,
|
||||
"engine": "MuJoCo / MotrixSim",
|
||||
"requirements": ["rm65", "parallel_gripper", "pick_place_target", "contact_sensors"],
|
||||
"reason": "" if rm65_ready else "RM65 物理控制器或运行时尚未安装",
|
||||
}
|
||||
],
|
||||
"training": [
|
||||
{
|
||||
"id": "discoverse_act",
|
||||
"name": "DISCOVERSE ACT",
|
||||
"enabled": act_ready,
|
||||
"commit": cfg.act_commit,
|
||||
"input_format": "discoverse_act_hdf5",
|
||||
"reason": "" if act_ready else "DISCOVERSE ACT 固定版本运行时尚未就绪",
|
||||
},
|
||||
{"id": "bc_v2", "name": "历史岭回归模型", "enabled": False, "read_only": True, "reason": "仅用于查看历史模型,禁止新建"},
|
||||
],
|
||||
"inference": [
|
||||
{"id": "discoverse_act", "name": "DISCOVERSE ACT 在线推理", "enabled": act_ready, "requires_real_simulator": True}
|
||||
],
|
||||
}
|
||||
|
||||
@app.post("/api/uploads")
|
||||
def create_upload(payload: CreateUploadRequest, request: Request) -> dict[str, Any]:
|
||||
if not storage.configured:
|
||||
raise HTTPException(status_code=503, detail="BOS 存储尚未配置")
|
||||
if payload.total_size <= 0:
|
||||
raise HTTPException(status_code=400, detail="total_size must be > 0")
|
||||
if not 1024 * 1024 <= payload.chunk_size <= 64 * 1024 * 1024:
|
||||
raise HTTPException(status_code=400, detail="chunk_size must be between 1 MiB and 64 MiB")
|
||||
if payload.checksum and (len(payload.checksum) != 64 or any(c not in "0123456789abcdefABCDEF" for c in payload.checksum)):
|
||||
raise HTTPException(status_code=400, detail="checksum must be a SHA-256 hex digest")
|
||||
filename = safe_object_name(payload.filename)
|
||||
upload_id = secrets.token_hex(16)
|
||||
local_dir = cfg.upload_parts_dir / upload_id
|
||||
local_dir.mkdir(parents=True, exist_ok=False)
|
||||
return store.create_upload(
|
||||
item_id=upload_id,
|
||||
owner_user_id=request.state.user.get("id"),
|
||||
filename=filename,
|
||||
content_type=payload.content_type,
|
||||
total_size=payload.total_size,
|
||||
chunk_size=payload.chunk_size,
|
||||
total_parts=math.ceil(payload.total_size / payload.chunk_size),
|
||||
checksum_expected=payload.checksum.lower(),
|
||||
local_dir=local_dir,
|
||||
)
|
||||
|
||||
@app.get("/api/uploads/{upload_id}")
|
||||
def get_upload(upload_id: str, request: Request) -> dict[str, Any]:
|
||||
try:
|
||||
upload = store.get_upload(upload_id)
|
||||
_check_upload_owner(upload, request.state.user)
|
||||
return upload
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
|
||||
|
||||
@app.put("/api/uploads/{upload_id}/parts/{part_number}")
|
||||
async def upload_part(upload_id: str, part_number: int, request: Request) -> dict[str, Any]:
|
||||
try:
|
||||
upload = store.get_upload(upload_id)
|
||||
_check_upload_owner(upload, request.state.user)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
|
||||
if upload["status"] != "uploading":
|
||||
raise HTTPException(status_code=409, detail="上传任务已进入终态")
|
||||
if not 1 <= part_number <= upload["total_parts"]:
|
||||
raise HTTPException(status_code=400, detail="分片序号超出范围")
|
||||
expected_size = upload["chunk_size"]
|
||||
if part_number == upload["total_parts"]:
|
||||
expected_size = upload["total_size"] - upload["chunk_size"] * (upload["total_parts"] - 1)
|
||||
target = Path(upload["local_dir"]) / f"part-{part_number:08d}"
|
||||
temporary = target.with_suffix(".tmp")
|
||||
digest = hashlib.sha256()
|
||||
written = 0
|
||||
try:
|
||||
with temporary.open("wb") as handle:
|
||||
async for chunk in request.stream():
|
||||
written += len(chunk)
|
||||
if written > expected_size:
|
||||
raise HTTPException(status_code=413, detail="分片大小超过声明值")
|
||||
handle.write(chunk)
|
||||
digest.update(chunk)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
if written != expected_size:
|
||||
raise HTTPException(status_code=400, detail=f"分片大小不匹配,期望 {expected_size} 字节,实际 {written} 字节")
|
||||
temporary.replace(target)
|
||||
finally:
|
||||
if temporary.exists():
|
||||
temporary.unlink()
|
||||
result = store.record_upload_part(upload_id, part_number)
|
||||
return {"upload": result, "part_number": part_number, "size_bytes": written, "checksum": digest.hexdigest()}
|
||||
|
||||
@app.post("/api/uploads/{upload_id}/complete")
|
||||
def complete_upload(upload_id: str, request: Request) -> dict[str, Any]:
|
||||
try:
|
||||
upload = store.get_upload(upload_id)
|
||||
_check_upload_owner(upload, request.state.user)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
|
||||
if upload["status"] == "completed":
|
||||
return {"upload": upload, "dataset": _dataset_for_storage_uri(store, upload["storage_uri"])}
|
||||
missing = sorted(set(range(1, upload["total_parts"] + 1)) - set(upload["received_parts"]))
|
||||
if missing:
|
||||
raise HTTPException(status_code=409, detail={"message": "仍有分片未上传", "missing_parts": missing[:100]})
|
||||
local_dir = Path(upload["local_dir"])
|
||||
assembled = local_dir / upload["filename"]
|
||||
digest = hashlib.sha256()
|
||||
with assembled.open("wb") as output:
|
||||
for part_number in range(1, upload["total_parts"] + 1):
|
||||
part = local_dir / f"part-{part_number:08d}"
|
||||
with part.open("rb") as handle:
|
||||
while chunk := handle.read(8 * 1024 * 1024):
|
||||
output.write(chunk)
|
||||
digest.update(chunk)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
checksum = digest.hexdigest()
|
||||
if assembled.stat().st_size != upload["total_size"] or (upload["checksum_expected"] and checksum != upload["checksum_expected"]):
|
||||
store.fail_upload(upload_id, "assembled file checksum or size mismatch")
|
||||
raise HTTPException(status_code=422, detail="完整文件校验失败,请续传错误分片后重试")
|
||||
try:
|
||||
uploaded = storage.upload_file(
|
||||
assembled,
|
||||
area="uploads",
|
||||
relative_path=f"{upload_id}/{upload['filename']}",
|
||||
metadata={"sha256": checksum, "upload-id": upload_id},
|
||||
)
|
||||
dataset_id = secrets.token_hex(16)
|
||||
dataset_dir = cfg.datasets_dir / dataset_id
|
||||
dataset_dir.mkdir(parents=True, exist_ok=False)
|
||||
if assembled.suffix.lower() == ".zip":
|
||||
with zipfile.ZipFile(assembled) as archive:
|
||||
_safe_extract(archive, dataset_dir)
|
||||
else:
|
||||
shutil.copy2(assembled, dataset_dir / upload["filename"])
|
||||
scan = scan_dataset(dataset_dir)
|
||||
manifest_path = dataset_dir / "dataset_manifest.json"
|
||||
manifest = {
|
||||
"schema": "real2sim.dataset_manifest.v2",
|
||||
"id": dataset_id,
|
||||
"name": Path(upload["filename"]).stem,
|
||||
"source_type": "browser_upload",
|
||||
"storage_uri": uploaded["storage_uri"],
|
||||
"checksum": checksum,
|
||||
"version": checksum[:12],
|
||||
"scan": scan,
|
||||
}
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
storage.upload_file(manifest_path, area="datasets", relative_path=f"{dataset_id}/dataset_manifest.json", metadata={"sha256": sha256_file(manifest_path)})
|
||||
dataset = store.create_dataset(
|
||||
item_id=dataset_id,
|
||||
name=manifest["name"],
|
||||
path=dataset_dir,
|
||||
scan=scan,
|
||||
source_type="browser_upload",
|
||||
source_uri=uploaded["storage_uri"],
|
||||
version=manifest["version"],
|
||||
manifest_path=manifest_path,
|
||||
storage_uri=uploaded["storage_uri"],
|
||||
checksum=checksum,
|
||||
cache_state="local",
|
||||
)
|
||||
completed = store.complete_upload(upload_id, checksum_actual=checksum, storage_uri=uploaded["storage_uri"])
|
||||
return {"upload": completed, "dataset": _with_view_model(dataset, "dataset")}
|
||||
except (StorageUnavailable, StorageBoundaryError) as exc:
|
||||
store.fail_upload(upload_id, str(exc))
|
||||
raise HTTPException(status_code=503, detail=str(exc)) from exc
|
||||
|
||||
@app.delete("/api/uploads/{upload_id}")
|
||||
def delete_upload(upload_id: str, request: Request) -> dict[str, bool]:
|
||||
try:
|
||||
upload = store.get_upload(upload_id)
|
||||
_check_upload_owner(upload, request.state.user)
|
||||
shutil.rmtree(Path(upload["local_dir"]), ignore_errors=True)
|
||||
store.delete_upload(upload_id)
|
||||
return {"ok": True}
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="上传任务不存在") from exc
|
||||
|
||||
@app.get("/api/storage/objects")
|
||||
def list_storage_objects(area: str = "datasets", prefix: str = "", max_keys: int = 200) -> dict[str, Any]:
|
||||
try:
|
||||
return {"bucket": storage.bucket, "prefix": storage.prefix, "objects": storage.list_objects(area=area, relative_prefix=prefix, max_keys=max_keys)}
|
||||
except (StorageUnavailable, StorageBoundaryError) as exc:
|
||||
raise HTTPException(status_code=503 if isinstance(exc, StorageUnavailable) else 400, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/datasets/import-bos")
|
||||
def import_bos_dataset(payload: ImportBosRequest) -> dict[str, Any]:
|
||||
return _queue_task("bos_import", {"prefix": payload.prefix, "name": payload.name or Path(payload.prefix.rstrip("/")).name, "max_objects": payload.max_objects})
|
||||
|
||||
@app.get("/api/datasets")
|
||||
def list_datasets() -> list[dict[str, Any]]:
|
||||
@ -138,6 +541,8 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
|
||||
@app.post("/api/datasets/upload")
|
||||
def upload_dataset(file: UploadFile = File(...)) -> dict[str, Any]:
|
||||
if cfg.production_mode:
|
||||
raise HTTPException(status_code=410, detail="生产模式请使用 /api/uploads 分片上传,旧上传接口已禁用")
|
||||
target = cfg.uploads_dir / Path(file.filename or "upload.bin").name
|
||||
with target.open("wb") as handle:
|
||||
shutil.copyfileobj(file.file, handle)
|
||||
@ -212,8 +617,15 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
raise HTTPException(status_code=404, detail="asset file not found")
|
||||
if not is_path_allowed(asset_path, cfg):
|
||||
raise HTTPException(status_code=403, detail="asset path is outside configured import whitelist")
|
||||
manifest_path = _scene_manifest_path_for_write(scene)
|
||||
manifest = _read_manifest_or_default(scene, manifest_path)
|
||||
revision_id = secrets.token_hex(16)
|
||||
revision_dir = cfg.scenes_dir / "revisions" / revision_id
|
||||
source_scene_path = Path(scene["path"])
|
||||
source_package = source_scene_path if source_scene_path.is_dir() else source_scene_path.parent
|
||||
_copy_scene_revision(source_package, revision_dir)
|
||||
revision_scene_path = revision_dir / source_scene_path.name if source_scene_path.is_file() else revision_dir
|
||||
source_manifest_path = _scene_manifest_path_for_write(scene)
|
||||
manifest_path = revision_dir / source_manifest_path.name
|
||||
manifest = _read_manifest_or_default(scene, source_manifest_path)
|
||||
artifact = store.add_artifact(task_id=None, name=payload.name or asset_path.name, path=asset_path, kind=(payload.kind or _scene_asset_kind(asset_path)))
|
||||
asset = _scene_asset_entry(
|
||||
asset_path,
|
||||
@ -226,12 +638,28 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
assets = [item for item in manifest.get("assets", []) if not (isinstance(item, dict) and str(item.get("path") or "") == str(asset_path))]
|
||||
manifest["assets"] = [asset] + assets
|
||||
manifest["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
manifest["revision"] = {"id": revision_id, "parent_scene_id": scene_id, "reason": "asset_added"}
|
||||
manifest["asset_editing"] = {"supports_added_assets": True, "last_added_asset": asset["name"]}
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
tags = list(dict.fromkeys([*(scene.get("tags") or []), "added_asset"]))
|
||||
store.update_scene(scene_id, tags=tags)
|
||||
return {"ok": True, "scene": store.get_scene(scene_id), "asset": asset, "manifest_path": str(manifest_path)}
|
||||
tags = list(dict.fromkeys([*(scene.get("tags") or []), "added_asset", "scene_revision"]))
|
||||
metadata = dict(scene.get("metadata") or {})
|
||||
metadata.update({"parent_scene_id": scene_id, "revision_reason": "asset_added", "package": str(revision_dir)})
|
||||
revision = store.create_scene(
|
||||
item_id=revision_id,
|
||||
name=f"{scene['name']} · 资产修订",
|
||||
path=revision_scene_path,
|
||||
metadata=metadata,
|
||||
source_type="scene_revision",
|
||||
source_uri=scene.get("storage_uri") or scene.get("source_uri") or str(source_scene_path),
|
||||
version=_next_revision_label(str(scene.get("version") or "")),
|
||||
parent_task_id=scene.get("parent_task_id"),
|
||||
manifest_path=manifest_path,
|
||||
metrics=scene.get("metrics") or {},
|
||||
tags=tags,
|
||||
cache_state="local",
|
||||
)
|
||||
return {"ok": True, "scene": revision, "parent_scene_id": scene_id, "asset": asset, "manifest_path": str(manifest_path)}
|
||||
|
||||
@app.patch("/api/scenes/{scene_id}")
|
||||
def update_scene(scene_id: str, payload: UpdateEntityRequest) -> dict[str, Any]:
|
||||
@ -277,9 +705,9 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
|
||||
def _queue_task(task_type: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = CreateTaskRequest(type=task_type, params=params)
|
||||
_validate_task_payload(payload)
|
||||
_validate_task_payload(payload, cfg)
|
||||
_validate_task_resources(store, payload)
|
||||
if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "simulator_validation"}:
|
||||
if payload.type not in {"scene_reconstruction", "reconstruction_preflight", "media_prepare", "expert_collection", "training", "inference", "renderer_benchmark", "scene_cover_render", "scene_splat_optimize", "hf_import", "bos_import", "simulator_validation"}:
|
||||
raise ValueError(f"unknown task type: {payload.type}")
|
||||
run_dir = cfg.runs_dir / payload.type / _next_run_name(cfg.runs_dir / payload.type, payload.type)
|
||||
return store.create_task(task_type=payload.type, params=payload.params, run_dir=run_dir)
|
||||
@ -376,6 +804,22 @@ def create_app(config: PlatformConfig | None = None) -> FastAPI:
|
||||
def list_tasks() -> list[dict[str, Any]]:
|
||||
return [_with_task_view_model(item) for item in store.list_tasks()]
|
||||
|
||||
@app.get("/api/events/tasks")
|
||||
async def task_events(request: Request):
|
||||
async def stream():
|
||||
previous = ""
|
||||
while not await request.is_disconnected():
|
||||
tasks = [_with_task_view_model(item) for item in store.list_tasks()]
|
||||
fingerprint = hashlib.sha256(json.dumps([(item["id"], item["status"], item["progress"], item["stage"], item["message"], item["updated_at"]) for item in tasks], ensure_ascii=False).encode("utf-8")).hexdigest()
|
||||
if fingerprint != previous:
|
||||
yield f"event: tasks\ndata: {json.dumps(tasks, ensure_ascii=False)}\n\n"
|
||||
previous = fingerprint
|
||||
else:
|
||||
yield ": keepalive\n\n"
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
return StreamingResponse(stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
||||
|
||||
@app.post("/api/tasks")
|
||||
def create_task(payload: CreateTaskRequest) -> dict[str, Any]:
|
||||
try:
|
||||
@ -493,6 +937,29 @@ def _scene_manifest_path_for_write(scene: dict[str, Any]) -> Path:
|
||||
return package_dir / "scene_manifest.json"
|
||||
|
||||
|
||||
def _copy_scene_revision(source: Path, target: Path) -> None:
|
||||
if not source.is_dir():
|
||||
raise FileNotFoundError(source)
|
||||
|
||||
def link_or_copy(src: str, dst: str) -> str:
|
||||
try:
|
||||
os.link(src, dst)
|
||||
return dst
|
||||
except OSError:
|
||||
return shutil.copy2(src, dst)
|
||||
|
||||
shutil.copytree(source, target, copy_function=link_or_copy)
|
||||
|
||||
|
||||
def _next_revision_label(current: str) -> str:
|
||||
import re
|
||||
|
||||
match = re.search(r"(?:^|[-_.])r(\d+)$", current, re.IGNORECASE)
|
||||
number = int(match.group(1)) + 1 if match else 2
|
||||
base = re.sub(r"(?:[-_.])r\d+$", "", current, flags=re.IGNORECASE).strip("-_.") or "v1"
|
||||
return f"{base}-r{number}"
|
||||
|
||||
|
||||
def _read_manifest_or_default(scene: dict[str, Any], manifest_path: Path) -> dict[str, Any]:
|
||||
if manifest_path.is_file():
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
@ -735,8 +1202,15 @@ def _task_type_label(task_type: str) -> str:
|
||||
}.get(task_type, task_type)
|
||||
|
||||
|
||||
def _validate_task_payload(payload: CreateTaskRequest) -> None:
|
||||
def _validate_task_payload(payload: CreateTaskRequest, cfg: PlatformConfig) -> None:
|
||||
params = payload.params
|
||||
if cfg.production_mode and bool(params.get("dry_run")):
|
||||
raise HTTPException(status_code=400, detail="生产模式禁止 dry-run 任务")
|
||||
if payload.type == "bos_import":
|
||||
if not str(params.get("prefix") or "").strip():
|
||||
raise HTTPException(status_code=400, detail="bos_import requires prefix")
|
||||
if not 1 <= int(params.get("max_objects", 1000)) <= 1000:
|
||||
raise HTTPException(status_code=400, detail="max_objects must be between 1 and 1000")
|
||||
if payload.type == "hf_import":
|
||||
repo_type = params.get("repo_type", "dataset")
|
||||
if repo_type != "dataset":
|
||||
@ -778,8 +1252,9 @@ def _validate_task_payload(payload: CreateTaskRequest) -> None:
|
||||
if payload.type == "expert_collection":
|
||||
if not (params.get("scene_id") or params.get("scene_path")):
|
||||
raise HTTPException(status_code=400, detail="expert_collection requires scene_id or scene_path")
|
||||
if params.get("policy_preset", "pick_and_place_v1") != "pick_and_place_v1":
|
||||
raise HTTPException(status_code=400, detail="当前仅支持抓取放置策略;导航和精细操作策略尚未接入执行器")
|
||||
supported_policy = "rm65_pick_place_6d_ik" if cfg.production_mode else "pick_and_place_v1"
|
||||
if params.get("policy_preset", supported_policy) != supported_policy:
|
||||
raise HTTPException(status_code=400, detail=f"当前运行模式仅允许真实控制器 {supported_policy}")
|
||||
if int(params.get("workers", 1)) != 1:
|
||||
raise HTTPException(status_code=400, detail="当前专家采集执行器仅支持 workers=1")
|
||||
if int(params.get("episodes", 4)) <= 0 or int(params.get("steps", 24)) <= 0:
|
||||
@ -787,8 +1262,9 @@ def _validate_task_payload(payload: CreateTaskRequest) -> None:
|
||||
if payload.type == "training":
|
||||
if not (params.get("dataset_id") or params.get("dataset_path")):
|
||||
raise HTTPException(status_code=400, detail="training requires dataset_id or dataset_path")
|
||||
if params.get("recipe", "bc_v2") not in {"bc_v2", "bc_fast"}:
|
||||
raise HTTPException(status_code=400, detail="当前仅支持行为克隆 v2 和快速验证;扩散策略尚未接入训练器")
|
||||
allowed_recipes = {"discoverse_act"} if cfg.production_mode else {"bc_v2", "bc_fast", "discoverse_act"}
|
||||
if params.get("recipe", "discoverse_act" if cfg.production_mode else "bc_v2") not in allowed_recipes:
|
||||
raise HTTPException(status_code=400, detail="生产模式只允许 DISCOVERSE ACT;历史岭回归配方仅可查看")
|
||||
if int(params.get("epochs", 100)) <= 0:
|
||||
raise HTTPException(status_code=400, detail="epochs must be > 0")
|
||||
if payload.type == "inference":
|
||||
@ -815,13 +1291,133 @@ def _validate_task_resources(store: PlatformStore, payload: CreateTaskRequest) -
|
||||
raise HTTPException(status_code=400, detail="训练只能使用专家采集产生的 rollout 数据集")
|
||||
if payload.type == "inference":
|
||||
if params.get("model_id"):
|
||||
store.get_model(str(params["model_id"]))
|
||||
model = store.get_model(str(params["model_id"]))
|
||||
recipe = str((model.get("metadata") or {}).get("training", {}).get("recipe") or (model.get("metadata") or {}).get("recipe") or "")
|
||||
if params.get("recipe") == "discoverse_act" and recipe and recipe != "discoverse_act":
|
||||
raise HTTPException(status_code=400, detail="所选模型不是 DISCOVERSE ACT 模型")
|
||||
if params.get("scene_id"):
|
||||
store.get_scene(str(params["scene_id"]))
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"任务引用的资源不存在: {exc.args[0]}") from exc
|
||||
|
||||
|
||||
def _active_admin_count(store: PlatformStore) -> int:
|
||||
return sum(1 for user in store.list_users() if user["role"] == "admin" and not user["disabled"])
|
||||
|
||||
|
||||
def _check_upload_owner(upload: dict[str, Any], user: dict[str, Any]) -> None:
|
||||
if user.get("role") != "admin" and upload.get("owner_user_id") != user.get("id"):
|
||||
raise HTTPException(status_code=403, detail="无权访问其他用户的上传任务")
|
||||
|
||||
|
||||
def _dataset_for_storage_uri(store: PlatformStore, storage_uri: str) -> dict[str, Any] | None:
|
||||
return next((item for item in store.list_datasets() if item.get("storage_uri") == storage_uri), None)
|
||||
|
||||
|
||||
def _sqlite_journal_mode(store: PlatformStore) -> str:
|
||||
with store._connect() as db:
|
||||
return str(db.execute("pragma journal_mode").fetchone()[0]).lower()
|
||||
|
||||
|
||||
def _capabilities(cfg: PlatformConfig, storage: BosStorage, store: PlatformStore | None = None) -> dict[str, Any]:
|
||||
anysplat_commit = _git_commit_full(cfg.anysplat_root)
|
||||
discoverse_commit = _git_commit_full(cfg.discoverse_root)
|
||||
act_root = cfg.discoverse_root / "policies" / "act"
|
||||
act_commit = _git_commit_full(act_root)
|
||||
ffmpeg_ready = shutil.which("ffmpeg") is not None
|
||||
colmap_ready = cfg.colmap_binary.is_file() or shutil.which("colmap") is not None
|
||||
mujoco_ready = _module_available("mujoco")
|
||||
motrixsim_ready = _module_available("motrixsim") or bool(shutil.which("motrixsim"))
|
||||
torch_ready = _module_available("torch")
|
||||
rm65_controller = Path(__file__).with_name("rm65_controller.py")
|
||||
local_runtime = {
|
||||
"ffmpeg": ffmpeg_ready,
|
||||
"colmap": colmap_ready,
|
||||
"anysplat": cfg.anysplat_root.is_dir() and cfg.anysplat_python.is_file() and anysplat_commit == cfg.anysplat_commit,
|
||||
"mujoco": mujoco_ready,
|
||||
"motrixsim": motrixsim_ready,
|
||||
"discoverse_act": cfg.discoverse_root.is_dir() and act_root.is_dir() and discoverse_commit == cfg.discoverse_commit and act_commit == cfg.act_commit and torch_ready,
|
||||
"rm65_controller": rm65_controller.is_file() and mujoco_ready,
|
||||
}
|
||||
worker_status: dict[str, Any] = {"connected": cfg.worker_mode != "api", "age_seconds": 0 if cfg.worker_mode != "api" else None}
|
||||
if store is not None and cfg.worker_mode == "api":
|
||||
heartbeats = store.list_worker_heartbeats()
|
||||
latest = heartbeats[0] if heartbeats else None
|
||||
age = time.time() - float(latest["updated_at"]) if latest else None
|
||||
connected = bool(latest and age is not None and age < 30)
|
||||
worker_status = {"connected": connected, "age_seconds": age, "worker_id": latest.get("worker_id") if latest else None}
|
||||
if connected:
|
||||
checks = latest["payload"].get("checks", {})
|
||||
local_runtime.update(
|
||||
ffmpeg=bool(checks.get("ffmpeg")),
|
||||
colmap=bool(checks.get("colmap")),
|
||||
anysplat=all(bool(checks.get(name)) for name in ("anysplat_root", "anysplat_python", "anysplat_pinned")),
|
||||
mujoco=bool(checks.get("mujoco_python")),
|
||||
motrixsim=bool(checks.get("motrixsim_python")),
|
||||
discoverse_act=all(bool(checks.get(name)) for name in ("discoverse_act", "torch_cuda", "act_import")),
|
||||
rm65_controller=bool(checks.get("rm65_controller")) and bool(checks.get("mujoco_python")),
|
||||
)
|
||||
else:
|
||||
for name in local_runtime:
|
||||
local_runtime[name] = False
|
||||
return {
|
||||
"schema": "real2sim.capabilities.v1",
|
||||
"production_mode": cfg.production_mode,
|
||||
"auth_enabled": cfg.auth_enabled,
|
||||
"worker_mode": cfg.worker_mode,
|
||||
"worker": worker_status,
|
||||
"storage": {
|
||||
"bos": {
|
||||
"configured": storage.configured,
|
||||
"bucket": storage.bucket,
|
||||
"prefix": storage.prefix,
|
||||
"resumable_upload": True,
|
||||
"sha256": True,
|
||||
}
|
||||
},
|
||||
"runtimes": {
|
||||
"ffmpeg": {"ready": local_runtime["ffmpeg"]},
|
||||
"colmap": {"ready": local_runtime["colmap"]},
|
||||
"anysplat": {
|
||||
"ready": local_runtime["anysplat"],
|
||||
"commit": anysplat_commit,
|
||||
"required_commit": cfg.anysplat_commit,
|
||||
"model_revision": cfg.anysplat_model_revision,
|
||||
"vggt_model_revision": cfg.vggt_model_revision,
|
||||
},
|
||||
"mujoco": {"ready": local_runtime["mujoco"]},
|
||||
"motrixsim": {"ready": local_runtime["motrixsim"]},
|
||||
"discoverse_act": {
|
||||
"ready": local_runtime["discoverse_act"],
|
||||
"discoverse_commit": discoverse_commit,
|
||||
"required_discoverse_commit": cfg.discoverse_commit,
|
||||
"act_commit": act_commit,
|
||||
"required_act_commit": cfg.act_commit,
|
||||
},
|
||||
"rm65_controller": {"ready": local_runtime["rm65_controller"]},
|
||||
"spark": {"ready": SPARK_VENDOR_PATH.is_file()},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _git_commit_full(root: Path) -> str:
|
||||
if not root.is_dir():
|
||||
return ""
|
||||
try:
|
||||
return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=3).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _module_available(name: str) -> bool:
|
||||
try:
|
||||
import importlib.util
|
||||
|
||||
return importlib.util.find_spec(name) is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]:
|
||||
return {
|
||||
"gs_playground_root": str(cfg.gs_playground_root),
|
||||
@ -831,6 +1427,11 @@ def _runtime_paths(cfg: PlatformConfig) -> dict[str, str]:
|
||||
"gsplat_python": str(cfg.gsplat_python),
|
||||
"cuda_home": str(cfg.cuda_home),
|
||||
"runtime_env_script": str(cfg.runtime_env_script),
|
||||
"discoverse_commit": cfg.discoverse_commit,
|
||||
"act_commit": cfg.act_commit,
|
||||
"anysplat_commit": cfg.anysplat_commit,
|
||||
"anysplat_model_revision": cfg.anysplat_model_revision,
|
||||
"vggt_model_revision": cfg.vggt_model_revision,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
const STATIC_VERSION = "20260710-motrix-runtime-v7";
|
||||
const STATIC_VERSION = "20260710-poc-product-v1";
|
||||
|
||||
const state = {
|
||||
datasets: [],
|
||||
@ -11,7 +11,13 @@ const state = {
|
||||
editing: null,
|
||||
assetLibrary: [],
|
||||
refreshTimer: null,
|
||||
taskEvents: null,
|
||||
selections: {},
|
||||
user: null,
|
||||
capabilities: null,
|
||||
recipes: null,
|
||||
initialized: false,
|
||||
pendingPassword: "",
|
||||
};
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
@ -25,13 +31,97 @@ const taskScopes = {
|
||||
};
|
||||
|
||||
async function api(path, options = {}) {
|
||||
const response = await fetch(path, options);
|
||||
const response = await fetch(path, { credentials: "same-origin", ...options });
|
||||
const type = response.headers.get("content-type") || "";
|
||||
const payload = type.includes("application/json") ? await response.json() : await response.text();
|
||||
if (!response.ok) throw new Error(payload?.detail || payload?.error || response.statusText);
|
||||
if (!response.ok) {
|
||||
const detail = typeof payload?.detail === "object" ? payload.detail.message || JSON.stringify(payload.detail) : payload?.detail || payload?.error || response.statusText;
|
||||
const error = new Error(detail);
|
||||
error.status = response.status;
|
||||
error.payload = payload;
|
||||
throw error;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
function showAuth(mode = "login") {
|
||||
$("appShell").hidden = true;
|
||||
$("authShell").hidden = false;
|
||||
$("loginForm").hidden = mode !== "login";
|
||||
$("passwordForm").hidden = mode !== "password";
|
||||
if (mode === "password" && state.pendingPassword) $("currentPassword").value = state.pendingPassword;
|
||||
}
|
||||
|
||||
function showApp() {
|
||||
$("authShell").hidden = true;
|
||||
$("appShell").hidden = false;
|
||||
setText("currentUserLabel", `${state.user?.username || "本地用户"} · ${roleLabel(state.user?.role)}`);
|
||||
const usersButton = $("openUsers");
|
||||
if (usersButton) usersButton.hidden = state.user?.role !== "admin";
|
||||
}
|
||||
|
||||
async function bootstrap() {
|
||||
state.capabilities = await api("/api/capabilities");
|
||||
let authState;
|
||||
try {
|
||||
authState = await api("/api/auth/me");
|
||||
} catch (error) {
|
||||
if (error.status === 401) return showAuth("login");
|
||||
throw error;
|
||||
}
|
||||
state.user = authState.user;
|
||||
if (state.user?.must_change_password) return showAuth("password");
|
||||
state.recipes = await api("/api/recipes");
|
||||
applyRuntimeOptions();
|
||||
showApp();
|
||||
await refresh();
|
||||
startTaskEvents();
|
||||
clearInterval(state.refreshTimer);
|
||||
state.refreshTimer = setInterval(() => refresh({ quiet: true }).catch((error) => console.warn(error)), 15000);
|
||||
state.initialized = true;
|
||||
}
|
||||
|
||||
function startTaskEvents() {
|
||||
state.taskEvents?.close();
|
||||
state.taskEvents = new EventSource("/api/events/tasks");
|
||||
state.taskEvents.addEventListener("tasks", (event) => {
|
||||
const previousTerminal = new Set(state.tasks.filter((task) => terminalStatus(task.status)).map((task) => task.id));
|
||||
state.tasks = JSON.parse(event.data);
|
||||
renderScopedTaskLists();
|
||||
renderOverview();
|
||||
if (state.selectedTaskId && $("taskDrawer")?.classList.contains("open")) showTask(state.selectedTaskId, { preservePreview: true }).catch(console.warn);
|
||||
if (state.tasks.some((task) => terminalStatus(task.status) && !previousTerminal.has(task.id))) refresh({ quiet: true }).catch(console.warn);
|
||||
});
|
||||
state.taskEvents.onerror = () => console.warn("任务事件连接暂时中断,浏览器将自动重连");
|
||||
}
|
||||
|
||||
function terminalStatus(status) {
|
||||
return ["succeeded", "failed", "failed_recovered", "canceled"].includes(status);
|
||||
}
|
||||
|
||||
function roleLabel(role) {
|
||||
return { admin: "管理员", engineer: "工程师", viewer: "只读" }[role] || "本地模式";
|
||||
}
|
||||
|
||||
function applyRuntimeOptions() {
|
||||
const collection = state.recipes?.collection || [];
|
||||
const training = state.recipes?.training || [];
|
||||
fillRecipeSelect("policyPreset", collection, "暂无可用真实控制器");
|
||||
fillRecipeSelect("trainRecipe", training.filter((item) => !item.read_only), "暂无可用训练配方");
|
||||
}
|
||||
|
||||
function fillRecipeSelect(id, items, emptyLabel) {
|
||||
const select = $(id);
|
||||
if (!select) return;
|
||||
const enabled = items.find((item) => item.enabled);
|
||||
const unavailableReason = items.find((item) => !item.enabled)?.reason;
|
||||
select.innerHTML = enabled
|
||||
? items.map((item) => `<option value="${escapeAttr(item.id)}" ${item.enabled ? "" : "disabled"}>${escapeHtml(item.name)}${item.enabled ? "" : ` · ${item.reason || "不可用"}`}</option>`).join("")
|
||||
: `<option value="">${escapeHtml(unavailableReason || emptyLabel)}</option>${items.map((item) => `<option value="${escapeAttr(item.id)}" disabled>${escapeHtml(item.name)} · ${escapeHtml(item.reason || "不可用")}</option>`).join("")}`;
|
||||
select.value = enabled?.id || "";
|
||||
select.disabled = !enabled;
|
||||
}
|
||||
|
||||
async function refresh({ quiet = false } = {}) {
|
||||
try {
|
||||
const [datasets, scenes, models, tasks] = await Promise.all([
|
||||
@ -77,13 +167,15 @@ function renderAll() {
|
||||
}
|
||||
|
||||
function updateActionAvailability() {
|
||||
setActionAvailability("createMediaPrepare", state.datasets.length > 0, "请先新增数据集");
|
||||
setActionAvailability("createPreflightTask", preparedDatasets().length > 0, "请先准备帧数据");
|
||||
setActionAvailability("createSceneTask", preparedDatasets().length > 0, "请先准备帧数据");
|
||||
setActionAvailability("createCollectTask", state.scenes.length > 0, "请先完成场景重建");
|
||||
setActionAvailability("createTrainTask", collectionDatasets().length > 0, "请先完成专家采集");
|
||||
setActionAvailability("createInferTask", state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型");
|
||||
setActionAvailability("addSceneAsset", state.scenes.length > 0, "请先完成场景重建");
|
||||
const writable = state.user?.role !== "viewer";
|
||||
document.body.classList.toggle("viewer-mode", !writable);
|
||||
setActionAvailability("createMediaPrepare", writable && state.datasets.length > 0, writable ? "请先新增数据集" : "只读用户不能创建任务");
|
||||
setActionAvailability("createPreflightTask", writable && preparedDatasets().length > 0, "请先准备帧数据");
|
||||
setActionAvailability("createSceneTask", writable && preparedDatasets().length > 0, "请先准备帧数据");
|
||||
setActionAvailability("createCollectTask", writable && state.scenes.length > 0 && Boolean($("policyPreset")?.value), "请先选择兼容真实控制器的场景");
|
||||
setActionAvailability("createTrainTask", writable && collectionDatasets().length > 0 && Boolean($("trainRecipe")?.value), "请先完成真实专家采集并确认 ACT 运行时");
|
||||
setActionAvailability("createInferTask", writable && state.models.length > 0 && state.scenes.length > 0, state.models.length ? "请先完成场景重建" : "请先训练模型");
|
||||
setActionAvailability("addSceneAsset", writable && state.scenes.length > 0, "请先完成场景重建");
|
||||
}
|
||||
|
||||
function setActionAvailability(id, enabled, reason) {
|
||||
@ -293,14 +385,24 @@ function renderTrainingChart() {
|
||||
const best = state.models[0];
|
||||
const score = best ? modelScore(best) : 0;
|
||||
const loss = best?.metrics?.loss ?? best?.metrics?.val_loss ?? null;
|
||||
if (!best) {
|
||||
setHtml("trainingChart", emptyState("完成一次真实训练后,这里会显示损失曲线和验证指标。"));
|
||||
return;
|
||||
}
|
||||
const history = [best?.metrics?.loss_history, best?.metrics?.train_loss_history, best?.metrics?.loss_curve]
|
||||
.find((value) => Array.isArray(value) && value.length > 1) || [];
|
||||
const finiteHistory = history.map(Number).filter(Number.isFinite).slice(-48);
|
||||
const maxLoss = finiteHistory.length ? Math.max(...finiteHistory) : 0;
|
||||
const minLoss = finiteHistory.length ? Math.min(...finiteHistory) : 0;
|
||||
const range = Math.max(maxLoss - minLoss, Number.EPSILON);
|
||||
setHtml("trainingChart", `
|
||||
<div class="chart-kpis">
|
||||
<div><span>成功率</span><strong>${best ? `${Math.round(score)}%` : "--"}</strong></div>
|
||||
<div><span>成功率</span><strong>${Number.isFinite(score) ? `${Math.round(score)}%` : "--"}</strong></div>
|
||||
<div><span>验证损失</span><strong>${loss !== null ? Number(loss).toFixed(3) : "--"}</strong></div>
|
||||
</div>
|
||||
<div class="line-chart" aria-label="训练曲线">
|
||||
${[35, 48, 54, 63, Math.max(38, score || 72), Math.max(44, Math.min(96, (score || 76) + 8))].map((height) => `<span style="height:${height}%"></span>`).join("")}
|
||||
</div>
|
||||
${finiteHistory.length > 1
|
||||
? `<div class="line-chart" aria-label="真实训练损失曲线">${finiteHistory.map((value) => `<span style="height:${Math.max(8, 100 - ((value - minLoss) / range) * 88)}%" title="${escapeAttr(value.toFixed(6))}"></span>`).join("")}</div>`
|
||||
: `<div class="inline-empty">该模型没有可绘制的损失历史;可在模型卡中查看最终指标。</div>`}
|
||||
`);
|
||||
}
|
||||
|
||||
@ -653,7 +755,7 @@ function showBulkResult(title, result) {
|
||||
|
||||
function openImportDialog(type) {
|
||||
state.currentImport = type;
|
||||
const title = { hf: "导入 HuggingFace 数据", upload: "上传数据文件", local: "导入本地目录" }[type] || "新增数据";
|
||||
const title = { hf: "导入 HuggingFace 数据", upload: "上传数据文件", bos: "从 BOS 导入", local: "导入本地目录" }[type] || "新增数据";
|
||||
setText("importTitle", title);
|
||||
setText("importSubtitle", "主流程只展示资源选择;技术参数集中在这里配置。");
|
||||
if (type === "hf") {
|
||||
@ -672,7 +774,18 @@ function openImportDialog(type) {
|
||||
</details>
|
||||
`);
|
||||
} else if (type === "upload") {
|
||||
setHtml("importFields", `<label class="span-2">选择文件<input id="uploadFile" type="file"></label>`);
|
||||
setHtml("importFields", `
|
||||
<label class="span-2">选择文件<input id="uploadFile" type="file"></label>
|
||||
<div id="uploadProgress" class="upload-progress span-2"><span></span><strong>等待选择文件</strong></div>
|
||||
<p class="field-note span-2">上传会按 8MB 分片写入 BOS,刷新或网络中断后重新选择同一文件即可续传。</p>
|
||||
`);
|
||||
} else if (type === "bos") {
|
||||
setHtml("importFields", `
|
||||
<label>BOS 区域<select id="bosArea"><option value="datasets">数据集</option><option value="uploads">上传文件</option></select></label>
|
||||
<label>显示名称<input id="bosDatasetName" placeholder="例如 Playroom 原始数据"></label>
|
||||
<label class="span-2">授权前缀<input id="bosPrefix" placeholder="team/playroom/2026-07"></label>
|
||||
<p class="field-note span-2">只能访问 real2sim-poc/ 下的授权区域,无法列举或删除其他项目数据。</p>
|
||||
`);
|
||||
} else {
|
||||
setHtml("importFields", `
|
||||
<label class="span-2">白名单目录<input id="datasetPath" placeholder="选择或粘贴配置白名单内目录"></label>
|
||||
@ -695,10 +808,24 @@ async function submitImport(event) {
|
||||
} else if (state.currentImport === "upload") {
|
||||
const file = $("uploadFile")?.files?.[0];
|
||||
if (!file) return toast("请先选择文件。");
|
||||
const body = new FormData();
|
||||
body.append("file", file);
|
||||
await api("/api/datasets/upload", { method: "POST", body });
|
||||
toast("文件已上传并登记为数据集");
|
||||
if (state.capabilities?.storage?.bos?.configured) {
|
||||
await uploadFileResumable(file);
|
||||
} else {
|
||||
const body = new FormData();
|
||||
body.append("file", file);
|
||||
await api("/api/datasets/upload", { method: "POST", body });
|
||||
}
|
||||
toast("文件已校验并登记为数据集");
|
||||
await refresh({ quiet: true });
|
||||
} else if (state.currentImport === "bos") {
|
||||
const prefix = $("bosPrefix")?.value.trim();
|
||||
if (!prefix) return toast("请填写授权前缀。");
|
||||
await api("/api/datasets/import-bos", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ prefix: `${$("bosArea")?.value || "datasets"}/${prefix}`, name: $("bosDatasetName")?.value.trim() || undefined }),
|
||||
});
|
||||
toast("BOS 导入任务已创建");
|
||||
await refresh({ quiet: true });
|
||||
} else {
|
||||
const path = $("datasetPath")?.value.trim();
|
||||
@ -714,6 +841,50 @@ async function submitImport(event) {
|
||||
$("importDialog")?.close();
|
||||
}
|
||||
|
||||
async function uploadFileResumable(file) {
|
||||
const chunkSize = 8 * 1024 * 1024;
|
||||
const resumeKey = `real2sim-upload:${file.name}:${file.size}:${file.lastModified}`;
|
||||
let upload = null;
|
||||
const savedId = localStorage.getItem(resumeKey);
|
||||
if (savedId) {
|
||||
try {
|
||||
upload = await api(`/api/uploads/${savedId}`);
|
||||
if (upload.filename !== file.name || upload.total_size !== file.size || upload.status !== "uploading") upload = null;
|
||||
} catch (_error) {
|
||||
upload = null;
|
||||
}
|
||||
}
|
||||
if (!upload) {
|
||||
upload = await api("/api/uploads", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ filename: file.name, content_type: file.type || "application/octet-stream", total_size: file.size, chunk_size: chunkSize }),
|
||||
});
|
||||
localStorage.setItem(resumeKey, upload.id);
|
||||
}
|
||||
const received = new Set(upload.received_parts || []);
|
||||
for (let part = 1; part <= upload.total_parts; part += 1) {
|
||||
if (received.has(part)) continue;
|
||||
const start = (part - 1) * upload.chunk_size;
|
||||
const end = Math.min(file.size, start + upload.chunk_size);
|
||||
await api(`/api/uploads/${upload.id}/parts/${part}`, { method: "PUT", headers: { "Content-Type": "application/octet-stream" }, body: file.slice(start, end) });
|
||||
received.add(part);
|
||||
renderUploadProgress(received.size / upload.total_parts, `已上传 ${received.size}/${upload.total_parts} 个分片`);
|
||||
}
|
||||
renderUploadProgress(0.98, "正在校验 SHA-256 并写入 BOS");
|
||||
const completed = await api(`/api/uploads/${upload.id}/complete`, { method: "POST" });
|
||||
localStorage.removeItem(resumeKey);
|
||||
renderUploadProgress(1, `上传完成 · ${completed.upload.checksum_actual.slice(0, 12)}`);
|
||||
return completed;
|
||||
}
|
||||
|
||||
function renderUploadProgress(value, label) {
|
||||
const node = $("uploadProgress");
|
||||
if (!node) return;
|
||||
node.querySelector("span").style.width = `${Math.max(0, Math.min(100, value * 100))}%`;
|
||||
node.querySelector("strong").textContent = label;
|
||||
}
|
||||
|
||||
function openViewer(sceneId) {
|
||||
state.activeSceneId = sceneId;
|
||||
const assetScene = $("assetScene");
|
||||
@ -726,9 +897,19 @@ function openViewer(sceneId) {
|
||||
}
|
||||
|
||||
function openViewerFullscreen(url) {
|
||||
const viewerUrl = url || $("openSceneViewerFull")?.dataset.viewerUrl || $("scenePreview")?.getAttribute("src");
|
||||
if (!viewerUrl) return toast("请先选择一个场景。");
|
||||
window.open(viewerUrl, "_blank", "noopener,noreferrer");
|
||||
const panel = document.querySelector(".viewer-panel");
|
||||
const button = $("openSceneViewerFull");
|
||||
const iframe = $("scenePreview");
|
||||
const viewerUrl = url || button?.dataset.viewerUrl || iframe?.getAttribute("src");
|
||||
if (!panel || !viewerUrl) return toast("请先选择一个场景。");
|
||||
const maximizing = !panel.classList.contains("is-maximized");
|
||||
panel.classList.toggle("is-maximized", maximizing);
|
||||
document.body.classList.toggle("viewer-maximized", maximizing);
|
||||
if (button) {
|
||||
button.textContent = maximizing ? "退出最大化" : "最大化预览";
|
||||
button.setAttribute("aria-pressed", String(maximizing));
|
||||
}
|
||||
if (iframe && maximizing && !iframe.getAttribute("src")) iframe.src = viewerUrl;
|
||||
}
|
||||
|
||||
function openAssetEditor(sceneId) {
|
||||
@ -822,7 +1003,7 @@ function reconstructionOptions() {
|
||||
scene_name: $("sceneName")?.value.trim() || undefined,
|
||||
renderer: $("rendererSelect")?.value || "auto",
|
||||
simulators: simulatorValues(),
|
||||
dry_run: $("dryRunScene")?.checked || false,
|
||||
dry_run: false,
|
||||
auto_prepare: $("autoPrepareScene")?.checked || false,
|
||||
allow_low_quality: $("allowLowQualityScene")?.checked || false,
|
||||
min_frames: Number($("sceneMinFrames")?.value || 30),
|
||||
@ -1267,7 +1448,100 @@ function toast(message) {
|
||||
node._timer = setTimeout(() => node.classList.remove("show"), 2600);
|
||||
}
|
||||
|
||||
async function submitLogin(event) {
|
||||
event.preventDefault();
|
||||
setText("loginError", "");
|
||||
const username = $("loginUsername").value.trim();
|
||||
const password = $("loginPassword").value;
|
||||
try {
|
||||
const result = await api("/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ username, password }),
|
||||
});
|
||||
state.user = result.user;
|
||||
state.pendingPassword = password;
|
||||
if (state.user?.must_change_password) {
|
||||
showAuth("password");
|
||||
return;
|
||||
}
|
||||
state.pendingPassword = "";
|
||||
await bootstrap();
|
||||
} catch (error) {
|
||||
setText("loginError", error.message || "登录失败");
|
||||
}
|
||||
}
|
||||
|
||||
async function submitPasswordChange(event) {
|
||||
event.preventDefault();
|
||||
setText("passwordError", "");
|
||||
const currentPassword = $("currentPassword").value;
|
||||
const newPassword = $("newPassword").value;
|
||||
if (newPassword !== $("confirmPassword").value) return setText("passwordError", "两次输入的新密码不一致");
|
||||
try {
|
||||
await api("/api/auth/change-password", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ current_password: currentPassword, new_password: newPassword }),
|
||||
});
|
||||
state.pendingPassword = "";
|
||||
$("loginPassword").value = "";
|
||||
showAuth("login");
|
||||
setText("loginError", "密码已更新,请使用新密码登录");
|
||||
} catch (error) {
|
||||
setText("passwordError", error.message || "密码更新失败");
|
||||
}
|
||||
}
|
||||
|
||||
async function logout() {
|
||||
try {
|
||||
await api("/api/auth/logout", { method: "POST" });
|
||||
} finally {
|
||||
state.taskEvents?.close();
|
||||
clearInterval(state.refreshTimer);
|
||||
state.user = null;
|
||||
state.initialized = false;
|
||||
showAuth("login");
|
||||
}
|
||||
}
|
||||
|
||||
async function openUserManager() {
|
||||
await renderUsers();
|
||||
$("usersDialog")?.showModal();
|
||||
}
|
||||
|
||||
async function renderUsers() {
|
||||
const users = await api("/api/users");
|
||||
setHtml("userList", users.map((user) => `
|
||||
<div class="user-row">
|
||||
<div><strong>${escapeHtml(user.username)}</strong><span>${escapeHtml(roleLabel(user.role))} · ${user.disabled ? "已禁用" : user.must_change_password ? "待首次改密" : "可用"}</span></div>
|
||||
<div class="row-actions">
|
||||
<button data-user-toggle="${escapeAttr(user.id)}" data-user-disabled="${user.disabled}">${user.disabled ? "启用" : "禁用"}</button>
|
||||
<button data-user-reset="${escapeAttr(user.id)}">重置密码</button>
|
||||
<button class="danger-button" data-user-delete="${escapeAttr(user.id)}">删除</button>
|
||||
</div>
|
||||
</div>
|
||||
`).join("") || emptyState("暂无团队用户。"));
|
||||
}
|
||||
|
||||
async function createUser(event) {
|
||||
event.preventDefault();
|
||||
await api("/api/users", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ username: $("newUsername").value.trim(), password: $("newUserPassword").value, role: $("newUserRole").value }),
|
||||
});
|
||||
event.target.reset();
|
||||
toast("用户已创建,首次登录必须修改密码");
|
||||
await renderUsers();
|
||||
}
|
||||
|
||||
function bindEvents() {
|
||||
$("loginForm")?.addEventListener("submit", submitLogin);
|
||||
$("passwordForm")?.addEventListener("submit", submitPasswordChange);
|
||||
$("logoutButton")?.addEventListener("click", logout);
|
||||
$("openUsers")?.addEventListener("click", openUserManager);
|
||||
$("createUserForm")?.addEventListener("submit", createUser);
|
||||
document.querySelectorAll(".tab").forEach((button) => button.addEventListener("click", () => switchTab(button.dataset.tab)));
|
||||
$("closeDrawer")?.addEventListener("click", closeDrawer);
|
||||
$("importForm")?.addEventListener("submit", submitImport);
|
||||
@ -1323,13 +1597,13 @@ function bindEvents() {
|
||||
if (!sceneId) return toast("请先选择一个场景。");
|
||||
const payload = sceneAssetPayload();
|
||||
if (!payload.path) return toast("请选择资产来源,或在高级入口填写白名单文件。");
|
||||
await api(`/api/scenes/${sceneId}/assets`, {
|
||||
const result = await api(`/api/scenes/${sceneId}/assets`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
state.activeSceneId = sceneId;
|
||||
toast("资产已添加到场景清单");
|
||||
state.activeSceneId = result.scene?.id || sceneId;
|
||||
toast("资产已添加,并创建了新的场景修订版本");
|
||||
await refresh({ quiet: true });
|
||||
renderSceneViewerShell();
|
||||
scrollIntoViewIfPossible("#assetRail");
|
||||
@ -1358,7 +1632,15 @@ function bindEvents() {
|
||||
const sceneId = $("inferScene")?.value || state.scenes[0]?.id;
|
||||
if (!modelId) return toast("请先训练模型。");
|
||||
if (!sceneId) return toast("请先完成场景重建。");
|
||||
await createTask("inference", { model_id: modelId, scene_id: sceneId, steps: Number($("inferSteps")?.value || 80), workers: Number($("inferWorkers")?.value || 1) });
|
||||
await createTask("inference", {
|
||||
model_id: modelId,
|
||||
scene_id: sceneId,
|
||||
recipe: "discoverse_act",
|
||||
episodes: Number($("inferEpisodes")?.value || 10),
|
||||
steps: Number($("inferSteps")?.value || 240),
|
||||
workers: Number($("inferWorkers")?.value || 1),
|
||||
minimum_success_rate: 0.5,
|
||||
});
|
||||
});
|
||||
|
||||
document.body.addEventListener("change", (event) => {
|
||||
@ -1378,6 +1660,37 @@ async function handleBodyClick(event) {
|
||||
const closeDialog = event.target.closest("[data-close-dialog]");
|
||||
if (closeDialog) $(closeDialog.dataset.closeDialog)?.close();
|
||||
|
||||
const userToggle = event.target.closest("[data-user-toggle]");
|
||||
if (userToggle) {
|
||||
await api(`/api/users/${userToggle.dataset.userToggle}`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ disabled: userToggle.dataset.userDisabled !== "true" }),
|
||||
});
|
||||
await renderUsers();
|
||||
}
|
||||
|
||||
const userReset = event.target.closest("[data-user-reset]");
|
||||
if (userReset) {
|
||||
const password = prompt("输入至少 12 位的新初始密码;用户下次登录仍需修改密码。") || "";
|
||||
if (password) {
|
||||
await api(`/api/users/${userReset.dataset.userReset}`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ reset_password: password }),
|
||||
});
|
||||
toast("密码已重置");
|
||||
await renderUsers();
|
||||
}
|
||||
}
|
||||
|
||||
const userDelete = event.target.closest("[data-user-delete]");
|
||||
if (userDelete && confirm("删除用户会终止其会话,但不会删除业务资源。确认继续?")) {
|
||||
await api(`/api/users/${userDelete.dataset.userDelete}`, { method: "DELETE" });
|
||||
toast("用户已删除");
|
||||
await renderUsers();
|
||||
}
|
||||
|
||||
const openImport = event.target.closest("[data-open-import]");
|
||||
if (openImport) openImportDialog(openImport.dataset.openImport);
|
||||
|
||||
@ -1482,9 +1795,20 @@ async function handleBodyClick(event) {
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener("unhandledrejection", (event) => toast(event.reason?.message || String(event.reason || "操作失败")));
|
||||
window.addEventListener("unhandledrejection", (event) => {
|
||||
if (event.reason?.status === 401) showAuth("login");
|
||||
if (event.reason?.status === 428) showAuth("password");
|
||||
toast(event.reason?.message || String(event.reason || "操作失败"));
|
||||
});
|
||||
window.addEventListener("error", (event) => event.message && toast(event.message));
|
||||
window.addEventListener("keydown", (event) => {
|
||||
if (event.key === "Escape" && document.querySelector(".viewer-panel.is-maximized")) {
|
||||
openViewerFullscreen();
|
||||
}
|
||||
});
|
||||
|
||||
bindEvents();
|
||||
refresh().catch((error) => toast(error.message));
|
||||
state.refreshTimer = setInterval(() => refresh({ quiet: true }).catch((error) => console.warn(error)), 4000);
|
||||
bootstrap().catch((error) => {
|
||||
toast(error.message || "平台初始化失败");
|
||||
if (error.status === 401) showAuth("login");
|
||||
});
|
||||
|
||||
@ -5,10 +5,43 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Real2Sim</title>
|
||||
<link rel="icon" href="data:,">
|
||||
<link rel="stylesheet" href="/static/styles.css?v=20260710-motrix-runtime-v7">
|
||||
<link rel="stylesheet" href="/static/styles.css?v=20260710-poc-product-v1">
|
||||
</head>
|
||||
<body>
|
||||
<div class="app-shell">
|
||||
<section id="authShell" class="auth-shell" aria-label="登录 Real2Sim">
|
||||
<div class="auth-brand">
|
||||
<strong>Real2Sim</strong>
|
||||
<span>机器人从真实场景走向可训练智能体</span>
|
||||
<div class="auth-flow" aria-hidden="true">
|
||||
<i class="done"></i><b></b><i></i><b></b><i></i>
|
||||
</div>
|
||||
<p>数据、场景、采集、训练与推理使用同一条可追踪链路。</p>
|
||||
</div>
|
||||
<form id="loginForm" class="auth-card">
|
||||
<div>
|
||||
<h1>登录平台</h1>
|
||||
<p>使用管理员分配的账号进入工作台</p>
|
||||
</div>
|
||||
<label>用户名<input id="loginUsername" autocomplete="username" required></label>
|
||||
<label>密码<input id="loginPassword" type="password" autocomplete="current-password" required></label>
|
||||
<p id="loginError" class="form-error" role="alert"></p>
|
||||
<button class="primary-button" type="submit">登录</button>
|
||||
</form>
|
||||
<form id="passwordForm" class="auth-card" hidden>
|
||||
<div>
|
||||
<h1>设置新密码</h1>
|
||||
<p>首次登录需要完成密码更新</p>
|
||||
</div>
|
||||
<label>当前密码<input id="currentPassword" type="password" autocomplete="current-password" required></label>
|
||||
<label>新密码<input id="newPassword" type="password" autocomplete="new-password" minlength="12" required></label>
|
||||
<label>确认新密码<input id="confirmPassword" type="password" autocomplete="new-password" minlength="12" required></label>
|
||||
<p class="password-hint">至少 12 位,并包含大小写字母、数字或符号中的三类。</p>
|
||||
<p id="passwordError" class="form-error" role="alert"></p>
|
||||
<button class="primary-button" type="submit">保存并重新登录</button>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<div id="appShell" class="app-shell" hidden>
|
||||
<aside class="sidebar">
|
||||
<div class="brand">
|
||||
<strong>Real2Sim</strong>
|
||||
@ -23,6 +56,11 @@
|
||||
<button class="tab" data-tab="train">模型训练</button>
|
||||
<button class="tab" data-tab="infer">在线推理</button>
|
||||
</nav>
|
||||
<div class="sidebar-actions">
|
||||
<button id="openUsers" hidden>用户管理</button>
|
||||
<button id="logoutButton">退出登录</button>
|
||||
<span id="currentUserLabel"></span>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<main class="main-surface">
|
||||
@ -106,9 +144,9 @@
|
||||
<strong>上传文件</strong>
|
||||
<span>zip、视频或图片包</span>
|
||||
</button>
|
||||
<button class="source-card" data-open-import="local">
|
||||
<strong>本地目录</strong>
|
||||
<span>从配置白名单导入已有数据</span>
|
||||
<button class="source-card" data-open-import="bos">
|
||||
<strong>BOS 存储</strong>
|
||||
<span>从授权的 Real2Sim 前缀导入</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@ -228,7 +266,6 @@
|
||||
<label class="check"><input id="replaceSceneWithPoseTrain" type="checkbox" checked> 用训练结果替换主场景</label>
|
||||
<label class="check"><input id="exportTextures" type="checkbox" checked> 生成纹理资产</label>
|
||||
<label class="check"><input id="keepIntermediate" type="checkbox" checked> 保留中间产物</label>
|
||||
<label class="check muted-check"><input id="dryRunScene" type="checkbox"> 仅生成样例</label>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
@ -362,10 +399,10 @@
|
||||
</div>
|
||||
<div class="form-grid">
|
||||
<label class="span-2">选择场景<select id="collectScene"></select></label>
|
||||
<label>专家策略<select id="policyPreset"><option value="pick_and_place_v1">抓取放置策略</option><option value="navigation_v1" disabled>导航采集策略(待接入)</option><option value="manipulation_v2" disabled>精细操作策略(待接入)</option></select></label>
|
||||
<label>专家控制器<select id="policyPreset"><option value="">正在读取可用控制器</option></select></label>
|
||||
<label>执行并发<input id="collectWorkers" type="number" min="1" max="1" value="1" disabled title="当前专家采集执行器仅支持单 worker"></label>
|
||||
<label>Episode 数<input id="episodes" type="number" value="4"></label>
|
||||
<label>每个 Episode 步数<input id="steps" type="number" value="24"></label>
|
||||
<label>成功 Episode 数<input id="episodes" type="number" min="1" value="50"></label>
|
||||
<label>每个 Episode 步数<input id="steps" type="number" min="80" value="240"></label>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@ -417,7 +454,7 @@
|
||||
</div>
|
||||
<div class="form-grid">
|
||||
<label class="span-2">训练数据集<select id="trainDataset"></select></label>
|
||||
<label>训练配方<select id="trainRecipe"><option value="bc_v2">行为克隆 v2</option><option value="bc_fast">快速验证</option><option value="diffusion_policy" disabled>扩散策略(待接入)</option></select></label>
|
||||
<label>训练配方<select id="trainRecipe"><option value="">正在读取可用配方</option></select></label>
|
||||
<label>Epoch 数<input id="epochs" type="number" value="100"></label>
|
||||
</div>
|
||||
</section>
|
||||
@ -488,7 +525,8 @@
|
||||
<div class="form-grid">
|
||||
<label>选择模型<select id="inferModel"></select></label>
|
||||
<label>选择场景<select id="inferScene"></select></label>
|
||||
<label>步数上限<input id="inferSteps" type="number" min="1" value="80"></label>
|
||||
<label>推理 Episode 数<input id="inferEpisodes" type="number" min="1" value="10"></label>
|
||||
<label>步数上限<input id="inferSteps" type="number" min="80" value="240"></label>
|
||||
<label>执行并发<input id="inferWorkers" type="number" min="1" max="1" value="1" disabled title="当前推理执行器仅支持单 worker"></label>
|
||||
</div>
|
||||
</section>
|
||||
@ -580,7 +618,23 @@
|
||||
</form>
|
||||
</dialog>
|
||||
|
||||
<dialog id="usersDialog">
|
||||
<div class="dialog-card">
|
||||
<div class="panel-head">
|
||||
<div><h2>用户管理</h2><p>创建、禁用和重置团队账号</p></div>
|
||||
<button type="button" data-close-dialog="usersDialog" class="ghost-button">关闭</button>
|
||||
</div>
|
||||
<form id="createUserForm" class="form-grid user-form">
|
||||
<label>用户名<input id="newUsername" required></label>
|
||||
<label>角色<select id="newUserRole"><option value="engineer">工程师</option><option value="viewer">只读用户</option><option value="admin">管理员</option></select></label>
|
||||
<label class="span-2">初始密码<input id="newUserPassword" type="password" minlength="12" required></label>
|
||||
<button class="primary-button span-2" type="submit">创建用户</button>
|
||||
</form>
|
||||
<div id="userList" class="user-list"></div>
|
||||
</div>
|
||||
</dialog>
|
||||
|
||||
<div id="toast" class="toast" role="status"></div>
|
||||
<script src="/static/app.js?v=20260710-motrix-runtime-v7"></script>
|
||||
<script src="/static/app.js?v=20260710-poc-product-v1"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
182
src/real2sim_platform/storage.py
Normal file
182
src/real2sim_platform/storage.py
Normal file
@ -0,0 +1,182 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Iterable
|
||||
|
||||
from .config import PlatformConfig
|
||||
|
||||
|
||||
ALLOWED_AREAS = {"uploads", "datasets", "scenes", "models", "runs", "backups"}
|
||||
_SAFE_COMPONENT = re.compile(r"[^a-zA-Z0-9._-]+")
|
||||
|
||||
|
||||
class StorageUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class StorageBoundaryError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class BosStorage:
|
||||
def __init__(self, config: PlatformConfig) -> None:
|
||||
self.config = config
|
||||
self.bucket = config.bos_bucket.strip()
|
||||
self.prefix = normalize_prefix(config.bos_prefix)
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.bucket and self.config.bos_access_key_id and self.config.bos_secret_access_key)
|
||||
|
||||
def health(self) -> dict[str, Any]:
|
||||
result = {"configured": self.configured, "bucket": self.bucket, "prefix": self.prefix, "endpoint": self.config.bos_endpoint}
|
||||
if not self.configured:
|
||||
result["ready"] = False
|
||||
result["reason"] = "BOS credentials or bucket are not configured"
|
||||
return result
|
||||
try:
|
||||
self.list_objects(area="uploads", relative_prefix="", max_keys=1)
|
||||
result["ready"] = True
|
||||
except Exception as exc:
|
||||
result["ready"] = False
|
||||
result["reason"] = str(exc)
|
||||
return result
|
||||
|
||||
def object_key(self, area: str, relative_path: str) -> str:
|
||||
if area not in ALLOWED_AREAS:
|
||||
raise StorageBoundaryError(f"unsupported BOS area: {area}")
|
||||
relative = normalize_relative_path(relative_path)
|
||||
return f"{self.prefix}/{area}/{relative}" if relative else f"{self.prefix}/{area}/"
|
||||
|
||||
def uri_for_key(self, key: str) -> str:
|
||||
checked = self.assert_owned_key(key)
|
||||
return f"bos://{self.bucket}/{checked}"
|
||||
|
||||
def assert_owned_key(self, key: str) -> str:
|
||||
normalized = str(PurePosixPath(str(key).strip().lstrip("/")))
|
||||
owned_root = f"{self.prefix}/"
|
||||
if not normalized.startswith(owned_root) or normalized == self.prefix:
|
||||
raise StorageBoundaryError("object key is outside the Real2Sim BOS prefix")
|
||||
area = normalized[len(owned_root):].split("/", 1)[0]
|
||||
if area not in ALLOWED_AREAS:
|
||||
raise StorageBoundaryError("object key is outside an approved Real2Sim area")
|
||||
if ".." in PurePosixPath(normalized).parts:
|
||||
raise StorageBoundaryError("parent traversal is not allowed")
|
||||
return normalized
|
||||
|
||||
def upload_file(self, local_path: Path, *, area: str, relative_path: str, metadata: dict[str, str] | None = None) -> dict[str, Any]:
|
||||
if not local_path.is_file():
|
||||
raise FileNotFoundError(local_path)
|
||||
key = self.object_key(area, relative_path)
|
||||
client = self._get_client()
|
||||
kwargs: dict[str, Any] = {}
|
||||
if metadata:
|
||||
kwargs["user_metadata"] = {str(k): str(v) for k, v in metadata.items()}
|
||||
client.put_object_from_file(self.bucket, key, str(local_path), **kwargs)
|
||||
return {
|
||||
"key": key,
|
||||
"storage_uri": self.uri_for_key(key),
|
||||
"size_bytes": local_path.stat().st_size,
|
||||
"checksum": sha256_file(local_path),
|
||||
}
|
||||
|
||||
def download_file(self, storage_uri: str, target_path: Path) -> Path:
|
||||
key = self.key_from_uri(storage_uri)
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._get_client().get_object_to_file(self.bucket, key, str(target_path))
|
||||
return target_path
|
||||
|
||||
def list_objects(self, *, area: str, relative_prefix: str = "", max_keys: int = 1000) -> list[dict[str, Any]]:
|
||||
prefix = self.object_key(area, relative_prefix)
|
||||
response = self._get_client().list_objects(self.bucket, prefix=prefix, max_keys=max(1, min(1000, int(max_keys))))
|
||||
contents: Iterable[Any] = getattr(response, "contents", None) or []
|
||||
return [
|
||||
{
|
||||
"key": self.assert_owned_key(str(getattr(item, "key", ""))),
|
||||
"storage_uri": self.uri_for_key(str(getattr(item, "key", ""))),
|
||||
"size_bytes": int(getattr(item, "size", 0) or 0),
|
||||
"etag": str(getattr(item, "e_tag", getattr(item, "etag", "")) or "").strip('"'),
|
||||
"last_modified": str(getattr(item, "last_modified", "") or ""),
|
||||
}
|
||||
for item in contents
|
||||
if getattr(item, "key", None)
|
||||
]
|
||||
|
||||
def delete_object(self, storage_uri: str) -> None:
|
||||
key = self.key_from_uri(storage_uri)
|
||||
self._get_client().delete_object(self.bucket, key)
|
||||
|
||||
def key_from_uri(self, storage_uri: str) -> str:
|
||||
prefix = f"bos://{self.bucket}/"
|
||||
if not storage_uri.startswith(prefix):
|
||||
raise StorageBoundaryError("BOS URI bucket does not match configured bucket")
|
||||
return self.assert_owned_key(storage_uri[len(prefix):])
|
||||
|
||||
def _get_client(self):
|
||||
if not self.configured:
|
||||
raise StorageUnavailable("BOS storage is not configured")
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
try:
|
||||
from baidubce.auth.bce_credentials import BceCredentials
|
||||
from baidubce.bce_client_configuration import BceClientConfiguration
|
||||
from baidubce.services.bos.bos_client import BosClient
|
||||
except ImportError as exc:
|
||||
raise StorageUnavailable("bce-python-sdk is not installed") from exc
|
||||
credentials = BceCredentials(self.config.bos_access_key_id, self.config.bos_secret_access_key)
|
||||
configuration = BceClientConfiguration(credentials=credentials, endpoint=self.config.bos_endpoint)
|
||||
self._client = BosClient(configuration)
|
||||
return self._client
|
||||
|
||||
|
||||
def normalize_prefix(value: str) -> str:
|
||||
parts = PurePosixPath(str(value).strip().strip("/"))
|
||||
if not str(parts) or str(parts) == "." or ".." in parts.parts:
|
||||
raise StorageBoundaryError("BOS prefix must be a non-empty relative path")
|
||||
return str(parts)
|
||||
|
||||
|
||||
def normalize_relative_path(value: str) -> str:
|
||||
raw = str(value or "").replace("\\", "/").strip().lstrip("/")
|
||||
if not raw:
|
||||
return ""
|
||||
path = PurePosixPath(raw)
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise StorageBoundaryError("BOS path traversal is not allowed")
|
||||
return str(path)
|
||||
|
||||
|
||||
def safe_object_name(filename: str) -> str:
|
||||
name = Path(filename).name.strip()
|
||||
cleaned = _SAFE_COMPONENT.sub("-", name).strip(".-")
|
||||
if not cleaned:
|
||||
raise ValueError("filename is empty after sanitization")
|
||||
return cleaned[:180]
|
||||
|
||||
|
||||
def sha256_file(path: Path, chunk_size: int = 8 * 1024 * 1024) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(chunk_size):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def atomic_write_part(path: Path, chunks: Iterable[bytes]) -> tuple[int, str]:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with temporary.open("wb") as handle:
|
||||
for chunk in chunks:
|
||||
handle.write(chunk)
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
temporary.replace(path)
|
||||
return size, digest.hexdigest()
|
||||
@ -28,13 +28,16 @@ class PlatformStore:
|
||||
manifest_path: Path | None = None,
|
||||
metrics: dict[str, Any] | None = None,
|
||||
tags: list[str] | None = None,
|
||||
storage_uri: str = "",
|
||||
checksum: str = "",
|
||||
cache_state: str = "local",
|
||||
) -> dict[str, Any]:
|
||||
item_id = item_id or uuid.uuid4().hex
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"insert into datasets(id,name,path,scan_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(item_id, name, str(path), _json(scan), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now),
|
||||
"insert into datasets(id,name,path,scan_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(item_id, name, str(path), _json(scan), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now),
|
||||
)
|
||||
return self.get_dataset(item_id)
|
||||
|
||||
@ -65,13 +68,16 @@ class PlatformStore:
|
||||
manifest_path: Path | None = None,
|
||||
metrics: dict[str, Any] | None = None,
|
||||
tags: list[str] | None = None,
|
||||
storage_uri: str = "",
|
||||
checksum: str = "",
|
||||
cache_state: str = "local",
|
||||
) -> dict[str, Any]:
|
||||
item_id = item_id or uuid.uuid4().hex
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"insert into scenes(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now),
|
||||
"insert into scenes(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now),
|
||||
)
|
||||
return self.get_scene(item_id)
|
||||
|
||||
@ -125,13 +131,16 @@ class PlatformStore:
|
||||
manifest_path: Path | None = None,
|
||||
metrics: dict[str, Any] | None = None,
|
||||
tags: list[str] | None = None,
|
||||
storage_uri: str = "",
|
||||
checksum: str = "",
|
||||
cache_state: str = "local",
|
||||
) -> dict[str, Any]:
|
||||
item_id = item_id or uuid.uuid4().hex
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"insert into models(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), now, now),
|
||||
"insert into models(id,name,path,metadata_json,source_type,source_uri,version,parent_task_id,manifest_path,metrics_json,tags_json,storage_uri,checksum,cache_state,created_at,updated_at) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(item_id, name, str(path), _json(metadata), source_type, source_uri, version, parent_task_id, str(manifest_path or ""), _json(metrics or {}), _json(tags or []), storage_uri, checksum, cache_state, now, now),
|
||||
)
|
||||
return self.get_model(item_id)
|
||||
|
||||
@ -148,6 +157,18 @@ class PlatformStore:
|
||||
def delete_model(self, item_id: str) -> None:
|
||||
self._delete("models", item_id)
|
||||
|
||||
def update_entity_storage(self, table: str, item_id: str, *, storage_uri: str, checksum: str = "", cache_state: str = "local") -> dict[str, Any]:
|
||||
if table not in {"datasets", "scenes", "models"}:
|
||||
raise ValueError("unsupported entity table")
|
||||
mapper = {"datasets": self._dataset_row, "scenes": self._scene_row, "models": self._model_row}[table]
|
||||
self._get(table, item_id, mapper)
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
f"update {table} set storage_uri=?,checksum=?,cache_state=?,updated_at=? where id=?",
|
||||
(storage_uri, checksum, cache_state, _now(), item_id),
|
||||
)
|
||||
return self._get(table, item_id, mapper)
|
||||
|
||||
def create_task(self, *, task_type: str, params: dict[str, Any], run_dir: Path) -> dict[str, Any]:
|
||||
item_id = uuid.uuid4().hex
|
||||
now = _now()
|
||||
@ -164,7 +185,33 @@ class PlatformStore:
|
||||
def get_task(self, item_id: str) -> dict[str, Any]:
|
||||
return self._get("tasks", item_id, self._task_row)
|
||||
|
||||
def claim_next_queued_task(self, worker_id: str) -> dict[str, Any] | None:
|
||||
db = self._connect()
|
||||
try:
|
||||
db.execute("begin immediate")
|
||||
row = db.execute("select * from tasks where status='queued' order by created_at limit 1").fetchone()
|
||||
if row is None:
|
||||
db.commit()
|
||||
return None
|
||||
now = _now()
|
||||
cursor = db.execute(
|
||||
"update tasks set status='running',worker_id=?,claimed_at=?,attempt=attempt+1,progress=max(progress,1),stage='启动中',message='GPU worker 已领取任务',updated_at=? where id=? and status='queued'",
|
||||
(worker_id, now, now, row["id"]),
|
||||
)
|
||||
if cursor.rowcount != 1:
|
||||
db.rollback()
|
||||
return None
|
||||
claimed = db.execute("select * from tasks where id=?", (row["id"],)).fetchone()
|
||||
db.commit()
|
||||
return self._task_row(claimed)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def next_queued_task(self) -> dict[str, Any] | None:
|
||||
"""Compatibility helper for older callers; new workers must claim atomically."""
|
||||
with self._connect() as db:
|
||||
row = db.execute("select * from tasks where status='queued' order by created_at limit 1").fetchone()
|
||||
return self._task_row(row) if row else None
|
||||
@ -258,6 +305,153 @@ class PlatformStore:
|
||||
)
|
||||
return self.get_artifact(item_id)
|
||||
|
||||
def create_user(self, *, username: str, password_hash: str, role: str, must_change_password: bool = True) -> dict[str, Any]:
|
||||
item_id = uuid.uuid4().hex
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"insert into users(id,username,password_hash,role,disabled,must_change_password,created_at,updated_at) values(?,?,?,?,0,?,?,?)",
|
||||
(item_id, username, password_hash, role, int(must_change_password), now, now),
|
||||
)
|
||||
return self.get_user(item_id)
|
||||
|
||||
def count_users(self) -> int:
|
||||
with self._connect() as db:
|
||||
return int(db.execute("select count(*) from users").fetchone()[0])
|
||||
|
||||
def list_users(self) -> list[dict[str, Any]]:
|
||||
with self._connect() as db:
|
||||
rows = db.execute("select * from users order by created_at").fetchall()
|
||||
return [self._user_row(row) for row in rows]
|
||||
|
||||
def get_user(self, user_id: str) -> dict[str, Any]:
|
||||
return self._get("users", user_id, self._user_row)
|
||||
|
||||
def get_user_by_username(self, username: str) -> dict[str, Any] | None:
|
||||
with self._connect() as db:
|
||||
row = db.execute("select * from users where lower(username)=lower(?)", (username,)).fetchone()
|
||||
return self._user_row(row, include_password=True) if row else None
|
||||
|
||||
def update_user(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
role: str | None = None,
|
||||
disabled: bool | None = None,
|
||||
password_hash: str | None = None,
|
||||
must_change_password: bool | None = None,
|
||||
last_login: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.get_user(user_id)
|
||||
fields: list[str] = []
|
||||
values: list[Any] = []
|
||||
for name, value in (("role", role), ("password_hash", password_hash), ("last_login", last_login)):
|
||||
if value is not None:
|
||||
fields.append(f"{name}=?")
|
||||
values.append(value)
|
||||
for name, value in (("disabled", disabled), ("must_change_password", must_change_password)):
|
||||
if value is not None:
|
||||
fields.append(f"{name}=?")
|
||||
values.append(int(value))
|
||||
if fields:
|
||||
fields.append("updated_at=?")
|
||||
values.extend([_now(), user_id])
|
||||
with self._connect() as db:
|
||||
db.execute(f"update users set {','.join(fields)} where id=?", values)
|
||||
return self.get_user(user_id)
|
||||
|
||||
def delete_user(self, user_id: str) -> None:
|
||||
self._delete("users", user_id)
|
||||
|
||||
def create_session(self, *, token_hash: str, user_id: str, expires_at: float) -> None:
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"insert into sessions(token_hash,user_id,expires_at,created_at,last_seen) values(?,?,?,?,?)",
|
||||
(token_hash, user_id, expires_at, now, now),
|
||||
)
|
||||
|
||||
def get_session_user(self, token_hash: str) -> dict[str, Any] | None:
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
row = db.execute(
|
||||
"select users.* from sessions join users on users.id=sessions.user_id where sessions.token_hash=? and sessions.expires_at>? and users.disabled=0",
|
||||
(token_hash, now),
|
||||
).fetchone()
|
||||
if row:
|
||||
db.execute("update sessions set last_seen=? where token_hash=?", (now, token_hash))
|
||||
db.execute("delete from sessions where expires_at<=?", (now,))
|
||||
return self._user_row(row) if row else None
|
||||
|
||||
def delete_session(self, token_hash: str) -> None:
|
||||
with self._connect() as db:
|
||||
db.execute("delete from sessions where token_hash=?", (token_hash,))
|
||||
|
||||
def delete_user_sessions(self, user_id: str) -> None:
|
||||
with self._connect() as db:
|
||||
db.execute("delete from sessions where user_id=?", (user_id,))
|
||||
|
||||
def create_upload(
|
||||
self,
|
||||
*,
|
||||
owner_user_id: str | None,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
total_size: int,
|
||||
chunk_size: int,
|
||||
total_parts: int,
|
||||
checksum_expected: str,
|
||||
local_dir: Path,
|
||||
item_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
item_id = item_id or uuid.uuid4().hex
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"insert into uploads(id,owner_user_id,filename,content_type,total_size,chunk_size,total_parts,received_parts_json,checksum_expected,checksum_actual,status,local_dir,storage_uri,error,created_at,updated_at) values(?,?,?,?,?,?,?,'[]',?,?,'uploading',?,'','',?,?)",
|
||||
(item_id, owner_user_id, filename, content_type, total_size, chunk_size, total_parts, checksum_expected, "", str(local_dir), now, now),
|
||||
)
|
||||
return self.get_upload(item_id)
|
||||
|
||||
def get_upload(self, upload_id: str) -> dict[str, Any]:
|
||||
return self._get("uploads", upload_id, self._upload_row)
|
||||
|
||||
def record_upload_part(self, upload_id: str, part_number: int) -> dict[str, Any]:
|
||||
upload = self.get_upload(upload_id)
|
||||
parts = sorted(set([*upload["received_parts"], int(part_number)]))
|
||||
with self._connect() as db:
|
||||
db.execute("update uploads set received_parts_json=?,updated_at=? where id=?", (_json(parts), _now(), upload_id))
|
||||
return self.get_upload(upload_id)
|
||||
|
||||
def complete_upload(self, upload_id: str, *, checksum_actual: str, storage_uri: str) -> dict[str, Any]:
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"update uploads set status='completed',checksum_actual=?,storage_uri=?,error='',updated_at=? where id=?",
|
||||
(checksum_actual, storage_uri, _now(), upload_id),
|
||||
)
|
||||
return self.get_upload(upload_id)
|
||||
|
||||
def fail_upload(self, upload_id: str, error: str) -> dict[str, Any]:
|
||||
with self._connect() as db:
|
||||
db.execute("update uploads set status='failed',error=?,updated_at=? where id=?", (error, _now(), upload_id))
|
||||
return self.get_upload(upload_id)
|
||||
|
||||
def delete_upload(self, upload_id: str) -> None:
|
||||
self._delete("uploads", upload_id)
|
||||
|
||||
def update_worker_heartbeat(self, worker_id: str, payload: dict[str, Any]) -> None:
|
||||
now = _now()
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"insert into worker_heartbeats(worker_id,payload_json,updated_at) values(?,?,?) on conflict(worker_id) do update set payload_json=excluded.payload_json,updated_at=excluded.updated_at",
|
||||
(worker_id, _json(payload), now),
|
||||
)
|
||||
|
||||
def list_worker_heartbeats(self) -> list[dict[str, Any]]:
|
||||
with self._connect() as db:
|
||||
rows = db.execute("select * from worker_heartbeats order by updated_at desc").fetchall()
|
||||
return [{"worker_id": row["worker_id"], "payload": json.loads(row["payload_json"] or "{}"), "updated_at": row["updated_at"]} for row in rows]
|
||||
|
||||
def list_task_artifacts(self, task_id: str) -> list[dict[str, Any]]:
|
||||
with self._connect() as db:
|
||||
rows = db.execute("select * from artifacts where task_id=? order by created_at", (task_id,)).fetchall()
|
||||
@ -268,6 +462,8 @@ class PlatformStore:
|
||||
|
||||
def _init(self) -> None:
|
||||
with self._connect() as db:
|
||||
db.execute("pragma journal_mode=WAL")
|
||||
db.execute("pragma synchronous=NORMAL")
|
||||
db.execute("create table if not exists datasets(id text primary key,name text,path text,scan_json text,created_at real,updated_at real)")
|
||||
db.execute("create table if not exists scenes(id text primary key,name text,path text,metadata_json text,created_at real,updated_at real)")
|
||||
db.execute("create table if not exists models(id text primary key,name text,path text,metadata_json text,created_at real,updated_at real)")
|
||||
@ -277,6 +473,16 @@ class PlatformStore:
|
||||
"run_dir text,error text,cancel_requested integer not null default 0,created_at real,updated_at real)"
|
||||
)
|
||||
db.execute("create table if not exists artifacts(id text primary key,task_id text,name text,path text,kind text,created_at real)")
|
||||
db.execute(
|
||||
"create table if not exists users(id text primary key,username text not null unique,password_hash text not null,role text not null,disabled integer not null default 0,must_change_password integer not null default 1,last_login real,created_at real,updated_at real)"
|
||||
)
|
||||
db.execute(
|
||||
"create table if not exists sessions(token_hash text primary key,user_id text not null,expires_at real not null,created_at real,last_seen real,foreign key(user_id) references users(id) on delete cascade)"
|
||||
)
|
||||
db.execute(
|
||||
"create table if not exists uploads(id text primary key,owner_user_id text,filename text not null,content_type text,total_size integer not null,chunk_size integer not null,total_parts integer not null,received_parts_json text not null default '[]',checksum_expected text,checksum_actual text,status text not null,local_dir text not null,storage_uri text,error text,created_at real,updated_at real)"
|
||||
)
|
||||
db.execute("create table if not exists worker_heartbeats(worker_id text primary key,payload_json text not null,updated_at real not null)")
|
||||
for table in ["datasets", "scenes", "models"]:
|
||||
self._ensure_column(db, table, "source_type", "text not null default 'local'")
|
||||
self._ensure_column(db, table, "source_uri", "text not null default ''")
|
||||
@ -285,14 +491,24 @@ class PlatformStore:
|
||||
self._ensure_column(db, table, "manifest_path", "text not null default ''")
|
||||
self._ensure_column(db, table, "metrics_json", "text not null default '{}'")
|
||||
self._ensure_column(db, table, "tags_json", "text not null default '[]'")
|
||||
self._ensure_column(db, table, "storage_uri", "text not null default ''")
|
||||
self._ensure_column(db, table, "checksum", "text not null default ''")
|
||||
self._ensure_column(db, table, "cache_state", "text not null default 'local'")
|
||||
self._ensure_column(db, "tasks", "pid", "integer")
|
||||
self._ensure_column(db, "tasks", "progress", "integer not null default 0")
|
||||
self._ensure_column(db, "tasks", "stage", "text not null default ''")
|
||||
self._ensure_column(db, "tasks", "message", "text not null default ''")
|
||||
self._ensure_column(db, "tasks", "worker_id", "text not null default ''")
|
||||
self._ensure_column(db, "tasks", "claimed_at", "real")
|
||||
self._ensure_column(db, "tasks", "attempt", "integer not null default 0")
|
||||
db.execute("create index if not exists idx_tasks_status_created on tasks(status,created_at)")
|
||||
db.execute("create index if not exists idx_sessions_user on sessions(user_id)")
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
db = sqlite3.connect(self.db_path)
|
||||
db = sqlite3.connect(self.db_path, timeout=30)
|
||||
db.row_factory = sqlite3.Row
|
||||
db.execute("pragma busy_timeout=5000")
|
||||
db.execute("pragma foreign_keys=on")
|
||||
return db
|
||||
|
||||
def _ensure_column(self, db: sqlite3.Connection, table: str, column: str, spec: str) -> None:
|
||||
@ -359,6 +575,9 @@ class PlatformStore:
|
||||
"manifest_path": row["manifest_path"],
|
||||
"metrics": json.loads(row["metrics_json"] or "{}"),
|
||||
"tags": json.loads(row["tags_json"] or "[]"),
|
||||
"storage_uri": row["storage_uri"],
|
||||
"checksum": row["checksum"],
|
||||
"cache_state": row["cache_state"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
@ -377,6 +596,9 @@ class PlatformStore:
|
||||
"message": row["message"] or "",
|
||||
"cancel_requested": bool(row["cancel_requested"]),
|
||||
"pid": row["pid"],
|
||||
"worker_id": row["worker_id"],
|
||||
"claimed_at": row["claimed_at"],
|
||||
"attempt": int(row["attempt"] or 0),
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
@ -384,6 +606,41 @@ class PlatformStore:
|
||||
def _artifact_row(self, row: sqlite3.Row) -> dict[str, Any]:
|
||||
return {"id": row["id"], "task_id": row["task_id"], "name": row["name"], "path": row["path"], "kind": row["kind"], "created_at": row["created_at"]}
|
||||
|
||||
def _user_row(self, row: sqlite3.Row, *, include_password: bool = False) -> dict[str, Any]:
|
||||
item = {
|
||||
"id": row["id"],
|
||||
"username": row["username"],
|
||||
"role": row["role"],
|
||||
"disabled": bool(row["disabled"]),
|
||||
"must_change_password": bool(row["must_change_password"]),
|
||||
"last_login": row["last_login"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
if include_password:
|
||||
item["password_hash"] = row["password_hash"]
|
||||
return item
|
||||
|
||||
def _upload_row(self, row: sqlite3.Row) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"owner_user_id": row["owner_user_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"] or "application/octet-stream",
|
||||
"total_size": int(row["total_size"]),
|
||||
"chunk_size": int(row["chunk_size"]),
|
||||
"total_parts": int(row["total_parts"]),
|
||||
"received_parts": json.loads(row["received_parts_json"] or "[]"),
|
||||
"checksum_expected": row["checksum_expected"] or "",
|
||||
"checksum_actual": row["checksum_actual"] or "",
|
||||
"status": row["status"],
|
||||
"local_dir": row["local_dir"],
|
||||
"storage_uri": row["storage_uri"] or "",
|
||||
"error": row["error"] or "",
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
def _json(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
@ -7,7 +7,7 @@ from typing import Any
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .adapters import TaskCanceled, execute_task, index_artifacts_for_task
|
||||
from .adapters import TaskCanceled, execute_task, index_artifacts_for_task, run_doctor, sync_task_outputs
|
||||
from .config import PlatformConfig
|
||||
from .store import PlatformStore
|
||||
|
||||
@ -18,40 +18,65 @@ class TaskWorker:
|
||||
self.config = config
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._last_heartbeat = 0.0
|
||||
|
||||
def start(self) -> None:
|
||||
if self._thread and self._thread.is_alive():
|
||||
return
|
||||
self._stop.clear()
|
||||
self.store.recover_interrupted_tasks()
|
||||
self._heartbeat(force=True)
|
||||
self._thread = threading.Thread(target=self._loop, name="real2sim-platform-worker", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=5)
|
||||
|
||||
def _loop(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
task = self.store.next_queued_task()
|
||||
self._heartbeat()
|
||||
task = self.store.claim_next_queued_task(self.config.worker_id)
|
||||
if task is None:
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
self._run(task)
|
||||
|
||||
def _heartbeat(self, *, force: bool = False) -> None:
|
||||
now = time.time()
|
||||
if not force and now - self._last_heartbeat < 10:
|
||||
return
|
||||
report = run_doctor(self.config)
|
||||
self.store.update_worker_heartbeat(
|
||||
self.config.worker_id,
|
||||
{
|
||||
"ready": bool(report.get("ok")),
|
||||
"checks": report.get("checks", {}),
|
||||
"warnings": report.get("warnings", []),
|
||||
"pid": __import__("os").getpid(),
|
||||
},
|
||||
)
|
||||
self._last_heartbeat = now
|
||||
|
||||
def _run(self, task: dict[str, Any]) -> None:
|
||||
if self.store.get_task(task["id"])["cancel_requested"]:
|
||||
self.store.update_task(task["id"], status="canceled", error="task canceled before start", progress=0, stage="已取消", message="任务启动前已取消")
|
||||
return
|
||||
self.store.update_task(task["id"], status="running", progress=5, stage="启动中", message="任务已进入执行队列")
|
||||
self.store.update_task(task["id"], status="running", progress=5, stage="启动中", message=f"{self.config.worker_id} 正在初始化任务")
|
||||
try:
|
||||
result = execute_task(self.config, self.store, task)
|
||||
result = sync_task_outputs(self.config, self.store, task, result)
|
||||
except TaskCanceled as exc:
|
||||
self._write_trace(task, exc)
|
||||
index_artifacts_for_task(self.store, task["id"], Path(task["run_dir"]))
|
||||
self._persist_failed_run(task)
|
||||
self.store.update_task(task["id"], status="canceled", error=str(exc), progress=0, stage="已取消", message=str(exc))
|
||||
return
|
||||
except Exception as exc:
|
||||
self._write_trace(task, exc)
|
||||
index_artifacts_for_task(self.store, task["id"], Path(task["run_dir"]))
|
||||
self._persist_failed_run(task)
|
||||
self.store.update_task(task["id"], status="failed", error=str(exc), stage="执行失败", message=str(exc))
|
||||
return
|
||||
status = "canceled" if self.store.get_task(task["id"])["cancel_requested"] else "succeeded"
|
||||
@ -65,3 +90,11 @@ class TaskWorker:
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write("\nERROR: " + str(exc) + "\n")
|
||||
handle.write(traceback.format_exc() + "\n")
|
||||
|
||||
def _persist_failed_run(self, task: dict[str, Any]) -> None:
|
||||
try:
|
||||
sync_task_outputs(self.config, self.store, task, {})
|
||||
except Exception as exc:
|
||||
log_path = Path(task["run_dir"]) / "logs" / "task.log"
|
||||
with log_path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(f"\nfailed to persist terminal task artifacts: {exc}\n")
|
||||
|
||||
37
src/real2sim_platform/worker_main.py
Normal file
37
src/real2sim_platform/worker_main.py
Normal file
@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import signal
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .config import ensure_workspace, load_config
|
||||
from .store import PlatformStore
|
||||
from .worker import TaskWorker
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Real2Sim serial GPU worker")
|
||||
parser.add_argument("--config", type=Path)
|
||||
args = parser.parse_args()
|
||||
config = load_config(args.config)
|
||||
ensure_workspace(config)
|
||||
store = PlatformStore(config.db_path)
|
||||
worker = TaskWorker(store, config)
|
||||
stopping = False
|
||||
|
||||
def stop(*_args) -> None:
|
||||
nonlocal stopping
|
||||
stopping = True
|
||||
worker.stop()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
worker.start()
|
||||
while not stopping:
|
||||
time.sleep(1)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -215,11 +215,14 @@ def test_scene_asset_append_updates_manifest_and_artifact(tmp_path: Path) -> Non
|
||||
assert response.status_code == 200, response.text
|
||||
payload = response.json()
|
||||
assert payload["asset"]["name"] == "fixture metadata"
|
||||
manifest = client.get(f"/api/scenes/{scene['id']}/manifest").json()
|
||||
assert payload["scene"]["id"] != scene["id"]
|
||||
assert payload["parent_scene_id"] == scene["id"]
|
||||
manifest = client.get(f"/api/scenes/{payload['scene']['id']}/manifest").json()
|
||||
added = [item for item in manifest["assets"] if item["name"] == "fixture metadata"]
|
||||
assert added
|
||||
assert added[0]["role"] == "simulator_import"
|
||||
assert added[0]["transform"]["translation"] == [1, 2, 3]
|
||||
assert manifest["revision"]["parent_scene_id"] == scene["id"]
|
||||
assert client.get(added[0]["url"]).status_code == 200
|
||||
|
||||
|
||||
|
||||
185
tests/test_production_platform.py
Normal file
185
tests/test_production_platform.py
Normal file
@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from real2sim_platform.config import PlatformConfig
|
||||
from real2sim_platform.rm65_controller import collect
|
||||
from real2sim_platform.server import create_app
|
||||
from real2sim_platform.storage import BosStorage, StorageBoundaryError, normalize_relative_path
|
||||
from real2sim_platform.store import PlatformStore
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_auth_first_login_rbac_and_user_crud(tmp_path: Path) -> None:
|
||||
config = PlatformConfig(
|
||||
app_root=ROOT,
|
||||
workspace_dir=tmp_path / "workspace",
|
||||
worker_mode="api",
|
||||
auth_enabled=True,
|
||||
bootstrap_admin_password="Admin!Password123",
|
||||
)
|
||||
with TestClient(create_app(config)) as client:
|
||||
assert client.get("/api/datasets").status_code == 401
|
||||
login = client.post("/api/auth/login", json={"username": "admin", "password": "Admin!Password123"})
|
||||
assert login.status_code == 200
|
||||
assert login.json()["user"]["must_change_password"] is True
|
||||
assert client.get("/api/datasets").status_code == 428
|
||||
changed = client.post(
|
||||
"/api/auth/change-password",
|
||||
json={"current_password": "Admin!Password123", "new_password": "Changed!Password456"},
|
||||
)
|
||||
assert changed.status_code == 200
|
||||
assert client.get("/api/datasets").status_code == 401
|
||||
assert client.post("/api/auth/login", json={"username": "admin", "password": "Changed!Password456"}).status_code == 200
|
||||
viewer = client.post(
|
||||
"/api/users",
|
||||
json={"username": "viewer.user", "password": "Viewer!Password123", "role": "viewer"},
|
||||
)
|
||||
assert viewer.status_code == 200
|
||||
assert viewer.json()["role"] == "viewer"
|
||||
assert client.patch(f"/api/users/{viewer.json()['id']}", json={"disabled": True}).json()["disabled"] is True
|
||||
|
||||
viewer_config = PlatformConfig(
|
||||
app_root=ROOT,
|
||||
workspace_dir=config.workspace_dir,
|
||||
worker_mode="api",
|
||||
auth_enabled=True,
|
||||
bootstrap_admin_password="unused-Password!123",
|
||||
)
|
||||
store = PlatformStore(viewer_config.db_path)
|
||||
viewer_id = next(item["id"] for item in store.list_users() if item["username"] == "viewer.user")
|
||||
store.update_user(viewer_id, disabled=False)
|
||||
with TestClient(create_app(viewer_config)) as viewer_client:
|
||||
assert viewer_client.post("/api/auth/login", json={"username": "viewer.user", "password": "Viewer!Password123"}).status_code == 200
|
||||
assert viewer_client.get("/api/auth/me").status_code == 200
|
||||
assert viewer_client.post("/api/auth/change-password", json={"current_password": "Viewer!Password123", "new_password": "Viewer!Changed456"}).status_code == 200
|
||||
assert viewer_client.post("/api/auth/login", json={"username": "viewer.user", "password": "Viewer!Changed456"}).status_code == 200
|
||||
assert viewer_client.get("/api/datasets").status_code == 200
|
||||
assert viewer_client.post("/api/tasks", json={"type": "renderer_benchmark", "params": {}}).status_code == 403
|
||||
|
||||
|
||||
def test_bos_prefix_boundary_is_enforced(tmp_path: Path) -> None:
|
||||
config = PlatformConfig(
|
||||
workspace_dir=tmp_path,
|
||||
bos_bucket="testxyz",
|
||||
bos_prefix="real2sim-poc",
|
||||
bos_access_key_id="configured-but-not-used",
|
||||
bos_secret_access_key="configured-but-not-used",
|
||||
)
|
||||
storage = BosStorage(config)
|
||||
assert storage.object_key("datasets", "team/a.json") == "real2sim-poc/datasets/team/a.json"
|
||||
assert storage.key_from_uri("bos://testxyz/real2sim-poc/models/m1/card.json") == "real2sim-poc/models/m1/card.json"
|
||||
with pytest.raises(StorageBoundaryError):
|
||||
storage.object_key("datasets", "../../droboticflow/secret")
|
||||
with pytest.raises(StorageBoundaryError):
|
||||
storage.assert_owned_key("droboticflow/workspaces/data")
|
||||
with pytest.raises(StorageBoundaryError):
|
||||
storage.key_from_uri("bos://another-bucket/real2sim-poc/datasets/a")
|
||||
with pytest.raises(StorageBoundaryError):
|
||||
normalize_relative_path("../outside")
|
||||
|
||||
|
||||
def test_atomic_task_claim_has_no_duplicates(tmp_path: Path) -> None:
|
||||
store = PlatformStore(tmp_path / "state.db")
|
||||
for index in range(24):
|
||||
store.create_task(task_type="renderer_benchmark", params={"index": index}, run_dir=tmp_path / f"run-{index}")
|
||||
|
||||
def claim(worker: int):
|
||||
task = store.claim_next_queued_task(f"worker-{worker}")
|
||||
return task["id"] if task else None
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor:
|
||||
claimed = [item for item in executor.map(claim, range(24)) if item]
|
||||
assert len(claimed) == 24
|
||||
assert len(set(claimed)) == 24
|
||||
assert all(task["status"] == "running" and task["attempt"] == 1 for task in store.list_tasks())
|
||||
|
||||
|
||||
def test_api_capabilities_follow_live_worker_heartbeat(tmp_path: Path) -> None:
|
||||
config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api")
|
||||
store = PlatformStore(config.db_path)
|
||||
store.update_worker_heartbeat(
|
||||
"worker-a800",
|
||||
{
|
||||
"ready": True,
|
||||
"checks": {
|
||||
"ffmpeg": True,
|
||||
"colmap": True,
|
||||
"anysplat_root": True,
|
||||
"anysplat_python": True,
|
||||
"anysplat_pinned": True,
|
||||
"mujoco_python": True,
|
||||
"motrixsim_python": True,
|
||||
"discoverse_act": True,
|
||||
"torch_cuda": True,
|
||||
"act_import": True,
|
||||
"rm65_controller": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
with TestClient(create_app(config)) as client:
|
||||
capabilities = client.get("/api/capabilities").json()
|
||||
assert capabilities["worker"]["connected"] is True
|
||||
assert capabilities["runtimes"]["anysplat"]["ready"] is True
|
||||
assert capabilities["runtimes"]["discoverse_act"]["ready"] is True
|
||||
assert capabilities["runtimes"]["rm65_controller"]["ready"] is True
|
||||
|
||||
|
||||
def test_login_rate_limit_blocks_repeated_failures(tmp_path: Path) -> None:
|
||||
config = PlatformConfig(
|
||||
app_root=ROOT,
|
||||
workspace_dir=tmp_path / "workspace",
|
||||
worker_mode="api",
|
||||
auth_enabled=True,
|
||||
bootstrap_admin_password="Admin!Password123",
|
||||
)
|
||||
with TestClient(create_app(config)) as client:
|
||||
for _ in range(8):
|
||||
assert client.post("/api/auth/login", json={"username": "admin", "password": "wrong-password"}).status_code == 401
|
||||
assert client.post("/api/auth/login", json={"username": "admin", "password": "wrong-password"}).status_code == 429
|
||||
|
||||
|
||||
def test_production_task_gates_reject_fake_routes(tmp_path: Path) -> None:
|
||||
config = PlatformConfig(app_root=ROOT, workspace_dir=tmp_path / "workspace", worker_mode="api", production_mode=True)
|
||||
client = TestClient(create_app(config))
|
||||
dry_run = client.post(
|
||||
"/api/tasks",
|
||||
json={"type": "scene_reconstruction", "params": {"input_path": str(tmp_path), "dry_run": True}},
|
||||
)
|
||||
assert dry_run.status_code == 400
|
||||
legacy = client.post(
|
||||
"/api/tasks",
|
||||
json={"type": "training", "params": {"dataset_path": str(tmp_path), "recipe": "bc_v2"}},
|
||||
)
|
||||
assert legacy.status_code == 400
|
||||
fake_expert = client.post(
|
||||
"/api/tasks",
|
||||
json={"type": "expert_collection", "params": {"scene_path": str(tmp_path), "policy_preset": "pick_and_place_v1"}},
|
||||
)
|
||||
assert fake_expert.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_real_rm65_physical_collection_writes_act_hdf5(tmp_path: Path) -> None:
|
||||
scene = ROOT / "workspace/motrixsim_assets/parlour_rm65_pick_place/simulator_assets/rm65_table_pick_place.xml"
|
||||
if not scene.is_file():
|
||||
pytest.skip("real imported RM65 scene asset is not present")
|
||||
manifest = collect(scene, tmp_path / "collection", required_successes=1, max_attempts=2, max_steps=240, seed=7, width=160, height=120)
|
||||
assert manifest["successful_episodes"] == 1
|
||||
assert manifest["episodes"][0]["lift_height"] >= 0.10
|
||||
assert manifest["episodes"][0]["place_error"] <= 0.05
|
||||
episode = tmp_path / "collection/episode_0.hdf5"
|
||||
with h5py.File(episode, "r") as root:
|
||||
assert root["/observations/qpos"].shape[1] == 7
|
||||
assert root["/action"].shape[1] == 7
|
||||
assert root["/observations/images/overview"].shape[0] > 0
|
||||
assert root["/observations/contact"].shape[1] == 2
|
||||
assert bool(root.attrs["success"]) is True
|
||||
assert (tmp_path / "collection/episode_0.mp4").stat().st_size > 0
|
||||
21
vendor/gs_playground/LICENSE
vendored
Normal file
21
vendor/gs_playground/LICENSE
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Yufei Jia (贾宇飞)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
1
vendor/gs_playground/UPSTREAM_COMMIT
vendored
Normal file
1
vendor/gs_playground/UPSTREAM_COMMIT
vendored
Normal file
@ -0,0 +1 @@
|
||||
e10805930b2b652f5608c41f671c3460b47c0c72
|
||||
9
vendor/gs_playground/real2sim/__init__.py
vendored
Normal file
9
vendor/gs_playground/real2sim/__init__.py
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
"""Real2Sim pipeline tools for GS-Playground."""
|
||||
|
||||
from .errors import PipelineError, MissingStageCommand, StageArtifactError
|
||||
|
||||
__all__ = [
|
||||
"PipelineError",
|
||||
"MissingStageCommand",
|
||||
"StageArtifactError",
|
||||
]
|
||||
5
vendor/gs_playground/real2sim/__main__.py
vendored
Normal file
5
vendor/gs_playground/real2sim/__main__.py
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
from .cli import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1023
vendor/gs_playground/real2sim/builtin.py
vendored
Normal file
1023
vendor/gs_playground/real2sim/builtin.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
72
vendor/gs_playground/real2sim/camera.py
vendored
Normal file
72
vendor/gs_playground/real2sim/camera.py
vendored
Normal file
@ -0,0 +1,72 @@
|
||||
"""Camera utilities shared by Real2Sim packaging and viewers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .errors import StageArtifactError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceCamera:
|
||||
position: np.ndarray
|
||||
xmat: np.ndarray
|
||||
fovy_degrees: float
|
||||
width: int
|
||||
height: int
|
||||
y_up: bool = False
|
||||
|
||||
|
||||
def load_anysplat_source_camera(path: Path) -> SourceCamera:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
if not isinstance(payload, dict):
|
||||
raise StageArtifactError(f"camera JSON must be an object: {path}")
|
||||
|
||||
intrinsics = payload.get("intrinsics")
|
||||
if not isinstance(intrinsics, dict):
|
||||
raise StageArtifactError(f"camera JSON missing intrinsics: {path}")
|
||||
for key in ("width", "height", "fy"):
|
||||
if key not in intrinsics:
|
||||
raise StageArtifactError(f"camera JSON intrinsics missing {key}: {path}")
|
||||
|
||||
extrinsic_value: Any = payload.get("pred_context_pose", {}).get("extrinsic")
|
||||
if extrinsic_value is None:
|
||||
extrinsic_value = payload.get("extrinsic")
|
||||
extrinsic = _first_matrix(extrinsic_value, rows=4, cols=4, path=path)
|
||||
c2w = np.linalg.inv(extrinsic).astype(np.float32)
|
||||
|
||||
width = int(intrinsics["width"])
|
||||
height = int(intrinsics["height"])
|
||||
fy = float(intrinsics["fy"])
|
||||
if width <= 0 or height <= 0 or fy <= 0:
|
||||
raise StageArtifactError(f"camera JSON has invalid dimensions or focal length: {path}")
|
||||
|
||||
fovy = math.degrees(2.0 * math.atan(height / (2.0 * fy)))
|
||||
return SourceCamera(
|
||||
position=c2w[:3, 3].astype(np.float32),
|
||||
xmat=c2w[:3, :3].reshape(9).astype(np.float32),
|
||||
fovy_degrees=fovy,
|
||||
width=width,
|
||||
height=height,
|
||||
y_up=False,
|
||||
)
|
||||
|
||||
|
||||
def _first_matrix(value: Any, *, rows: int, cols: int, path: Path) -> np.ndarray:
|
||||
if value is None:
|
||||
raise StageArtifactError(f"camera JSON missing extrinsic matrix: {path}")
|
||||
arr = np.asarray(value, dtype=np.float32)
|
||||
while arr.ndim > 2 and arr.shape[0] == 1:
|
||||
arr = arr[0]
|
||||
if arr.ndim == 3 and arr.shape[1:] == (rows, cols):
|
||||
arr = arr[0]
|
||||
if arr.shape != (rows, cols):
|
||||
raise StageArtifactError(f"camera JSON extrinsic must have shape {(rows, cols)}, got {arr.shape}: {path}")
|
||||
return arr
|
||||
489
vendor/gs_playground/real2sim/cli.py
vendored
Normal file
489
vendor/gs_playground/real2sim/cli.py
vendored
Normal file
@ -0,0 +1,489 @@
|
||||
"""Command line interface for GS-Playground Real2Sim."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from .contracts import write_json
|
||||
from .paper_strict import check_paper_strict, format_report, validate_paper_strict
|
||||
from .pipeline import Real2SimPipeline
|
||||
from .pruning import PruneOptions
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="python -m real2sim", description="GS-Playground Real2Sim pipeline")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
run_parser = subparsers.add_parser("run", help="Run the full image/video-to-sim-ready-scene pipeline")
|
||||
run_parser.add_argument("--input", type=Path, required=True, help="Input RGB image or video")
|
||||
run_parser.add_argument("--output", type=Path, required=True, help="Output package directory")
|
||||
run_parser.add_argument("--work-dir", type=Path, default=None, help="Intermediate artifact directory")
|
||||
run_parser.add_argument("--prompt", default="all task-relevant objects", help="Open-vocabulary detection prompt")
|
||||
run_parser.add_argument("--config", type=Path, default=None, help="Real2Sim stage command config JSON")
|
||||
run_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"], help="Demo robot config to emit")
|
||||
run_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"], help="Real2Sim backend")
|
||||
run_parser.add_argument("--dynamic-objects", action="store_true", help="Emit free joints for reconstructed objects")
|
||||
run_parser.add_argument("--resume", action="store_true", help="Reuse existing work-dir artifacts")
|
||||
run_parser.add_argument("--dry-run", action="store_true", help="Print/validate configured commands without requiring outputs")
|
||||
run_parser.add_argument("--paper-strict", action="store_true", help="Require exact paper-equivalent external stages and runtime prerequisites")
|
||||
run_parser.add_argument("--synthetic-multiview", action="store_true", help="Expand a single input image into synthetic multiview frames before scene reconstruction")
|
||||
run_parser.add_argument(
|
||||
"--synthetic-multiview-mode",
|
||||
default="object",
|
||||
choices=["object", "scene"],
|
||||
help="Prompt style for synthetic multiview expansion: object turntable or scene completion",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--pipeline-mode",
|
||||
default="full",
|
||||
choices=["full", "scene-only", "object-only"],
|
||||
help="Pipeline scope: full (scene+objects), scene-only, or object-only",
|
||||
)
|
||||
run_parser.add_argument("--skip-scene-reconstruction", action="store_true", help="Skip scene reconstruction in an existing work-dir run")
|
||||
run_parser.add_argument(
|
||||
"--skip-object-reconstruction",
|
||||
action="store_true",
|
||||
help="Skip object reconstruction and keep only the reconstructed scene",
|
||||
)
|
||||
run_parser.add_argument("--no-prune", action="store_true", help="Skip 3DGS pruning")
|
||||
run_parser.add_argument("--keep-ratio", type=float, default=None, help="Pruning keep ratio")
|
||||
run_parser.set_defaults(func=_run)
|
||||
|
||||
scene_parser = subparsers.add_parser("generate-scene", help="Generate a scene-only Real2Sim package")
|
||||
scene_parser.add_argument("--input", type=Path, required=True, help="Input RGB image, image directory, or video")
|
||||
scene_parser.add_argument("--output", type=Path, required=True, help="Output scene package directory")
|
||||
scene_parser.add_argument("--work-dir", type=Path, default=None, help="Intermediate artifact directory")
|
||||
scene_parser.add_argument("--prompt", default="static navigable scene", help="Scene reconstruction prompt/context")
|
||||
scene_parser.add_argument("--config", type=Path, default=None, help="Real2Sim stage command config JSON")
|
||||
scene_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"], help="Demo robot config to emit")
|
||||
scene_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"], help="Scene generation backend")
|
||||
scene_parser.add_argument("--dynamic-objects", action="store_true", help=argparse.SUPPRESS)
|
||||
scene_parser.add_argument("--resume", action="store_true", help="Reuse existing work-dir artifacts")
|
||||
scene_parser.add_argument("--dry-run", action="store_true", help="Print/validate configured commands without requiring outputs")
|
||||
scene_parser.add_argument("--paper-strict", action="store_true", help="Require exact paper-equivalent external scene stages")
|
||||
scene_parser.add_argument("--synthetic-multiview", action="store_true", help="Expand a single input image into synthetic multiview frames before scene reconstruction")
|
||||
scene_parser.add_argument(
|
||||
"--scene-completion",
|
||||
action="store_true",
|
||||
help="Alias for scene-style synthetic multiview completion when a single image has insufficient scene evidence",
|
||||
)
|
||||
scene_parser.add_argument("--no-prune", action="store_true", help="Skip 3DGS pruning")
|
||||
scene_parser.add_argument("--keep-ratio", type=float, default=None, help="Pruning keep ratio")
|
||||
scene_parser.set_defaults(func=_generate_scene)
|
||||
|
||||
package_parser = subparsers.add_parser("package-existing", help="Package existing 3DGS/mesh assets as a GS-Playground scene")
|
||||
package_parser.add_argument("--source", type=Path, required=True, help="Original capture or source path")
|
||||
package_parser.add_argument("--scene-ply", type=Path, required=True, help="Scene/background 3DGS PLY")
|
||||
package_parser.add_argument("--scene-mesh", type=Path, default=None, help="Optional scene mesh OBJ")
|
||||
package_parser.add_argument("--objects-json", type=Path, default=None, help="Optional object asset list JSON")
|
||||
package_parser.add_argument("--cameras-json", type=Path, default=None, help="Optional AnySplat camera intrinsics/extrinsics JSON")
|
||||
package_parser.add_argument("--depth-map", type=Path, default=None, help="Optional monocular depth artifact used for alignment")
|
||||
package_parser.add_argument("--background-image", type=Path, default=None, help="Optional inpainted background image artifact")
|
||||
package_parser.add_argument("--preview-image", type=Path, default=None, help="Optional source-view 3DGS preview image artifact")
|
||||
package_parser.add_argument("--mesh-cleanup-report", type=Path, default=None, help="Optional scene mesh cleanup report JSON")
|
||||
package_parser.add_argument("--texture-bake-report", type=Path, default=None, help="Optional COLMAP texture bake report JSON")
|
||||
package_parser.add_argument("--scene-structure-report", type=Path, default=None, help="Optional scene structure report JSON")
|
||||
package_parser.add_argument("--output", type=Path, required=True, help="Output package directory")
|
||||
package_parser.add_argument("--prompt", default="Navigate in the reconstructed Real2Sim scene")
|
||||
package_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"])
|
||||
package_parser.add_argument("--dynamic-objects", action="store_true")
|
||||
package_parser.set_defaults(func=_package_existing)
|
||||
|
||||
check_parser = subparsers.add_parser("check", help="Check configured external model commands and local packages")
|
||||
check_parser.add_argument("--config", type=Path, default=None)
|
||||
check_parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"])
|
||||
check_parser.add_argument("--backend", default="auto", choices=["auto", "external", "builtin", "hybrid"])
|
||||
check_parser.add_argument("--dynamic-objects", action="store_true")
|
||||
check_parser.add_argument("--paper-strict", action="store_true", help="Validate exact paper-equivalent runtime prerequisites")
|
||||
check_parser.set_defaults(func=_check)
|
||||
|
||||
example_parser = subparsers.add_parser("write-example-config", help="Write a config template for external model commands")
|
||||
example_parser.add_argument("--output", type=Path, required=True)
|
||||
example_parser.set_defaults(func=_write_example_config)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
return int(args.func(args))
|
||||
|
||||
|
||||
def _pipeline(args) -> Real2SimPipeline:
|
||||
return Real2SimPipeline.from_config_file(
|
||||
repo_root=REPO_ROOT,
|
||||
config_file=getattr(args, "config", None),
|
||||
robot=args.robot,
|
||||
dynamic_objects=args.dynamic_objects,
|
||||
dry_run=getattr(args, "dry_run", False),
|
||||
backend="external" if getattr(args, "paper_strict", False) else getattr(args, "backend", "auto"),
|
||||
)
|
||||
|
||||
|
||||
def _run(args) -> int:
|
||||
mode = str(getattr(args, "pipeline_mode", "full"))
|
||||
skip_scene = bool(getattr(args, "skip_scene_reconstruction", False))
|
||||
skip_objects = bool(getattr(args, "skip_object_reconstruction", False))
|
||||
|
||||
if mode != "full" and (skip_scene or skip_objects):
|
||||
raise SystemExit("cannot combine --pipeline-mode with --skip-scene-reconstruction/--skip-object-reconstruction")
|
||||
|
||||
enable_scene_reconstruction = True
|
||||
enable_object_reconstruction = True
|
||||
if mode == "scene-only":
|
||||
enable_scene_reconstruction = True
|
||||
enable_object_reconstruction = False
|
||||
elif mode == "object-only":
|
||||
enable_scene_reconstruction = False
|
||||
enable_object_reconstruction = True
|
||||
else:
|
||||
enable_scene_reconstruction = not skip_scene
|
||||
enable_object_reconstruction = not skip_objects
|
||||
|
||||
if not enable_scene_reconstruction and not enable_object_reconstruction:
|
||||
raise SystemExit("both scene and object reconstruction are disabled")
|
||||
|
||||
resume = args.resume
|
||||
if mode == "object-only":
|
||||
resume = True
|
||||
|
||||
pipeline = _pipeline(args)
|
||||
paper_strict = bool(getattr(args, "paper_strict", False))
|
||||
if paper_strict and (args.no_prune or args.keep_ratio is not None):
|
||||
raise SystemExit("--paper-strict cannot be combined with --no-prune or --keep-ratio; use pruning.mode=speedy_splat")
|
||||
if paper_strict:
|
||||
report = check_paper_strict(pipeline.config, REPO_ROOT)
|
||||
for line in format_report(report):
|
||||
print(line)
|
||||
if args.dry_run:
|
||||
return 0 if report["ok"] else 1
|
||||
validate_paper_strict(pipeline.config, REPO_ROOT)
|
||||
if args.dry_run:
|
||||
status = pipeline.check()
|
||||
required = {"grounding_dino", "sam2", "instance_refinement", "lama", "anysplat", "sam3d", "pose_alignment"}
|
||||
if args.backend in {"auto", "builtin", "hybrid"} and status["builtin_ready"]:
|
||||
print("builtin: ready (numpy)")
|
||||
if args.backend == "builtin":
|
||||
return 0
|
||||
for stage, info in status["stages"].items():
|
||||
state = "configured" if info["configured"] else "missing"
|
||||
print(f"{stage}: {state} ({info['env']})")
|
||||
external_ready = all(status["stages"][stage]["configured"] for stage in required)
|
||||
return 0 if external_ready or (args.backend in {"auto", "hybrid"} and status["builtin_ready"]) else 1
|
||||
|
||||
prune = None
|
||||
if args.no_prune or args.keep_ratio is not None:
|
||||
keep_ratio = args.keep_ratio if args.keep_ratio is not None else 0.3
|
||||
prune = PruneOptions(enabled=not args.no_prune, keep_ratio=keep_ratio)
|
||||
result = pipeline.run(
|
||||
input_path=args.input,
|
||||
output_dir=args.output,
|
||||
work_dir=args.work_dir,
|
||||
prompt=args.prompt,
|
||||
resume=resume,
|
||||
prune_options=prune,
|
||||
enable_scene_reconstruction=enable_scene_reconstruction,
|
||||
enable_object_reconstruction=enable_object_reconstruction,
|
||||
synthetic_multiview=args.synthetic_multiview,
|
||||
synthetic_multiview_mode=getattr(args, "synthetic_multiview_mode", "object"),
|
||||
)
|
||||
print(f"Real2Sim package: {result.package.root}")
|
||||
print(f"Scene XML: {result.package.scene_xml}")
|
||||
print(f"Demo config: {result.package.config_json}")
|
||||
print(f"Manifest: {result.package.manifest_json}")
|
||||
return 0
|
||||
|
||||
|
||||
def _generate_scene(args) -> int:
|
||||
pipeline = _pipeline(args)
|
||||
paper_strict = bool(getattr(args, "paper_strict", False))
|
||||
if paper_strict and (args.no_prune or args.keep_ratio is not None):
|
||||
raise SystemExit("--paper-strict cannot be combined with --no-prune or --keep-ratio; use pruning.mode=speedy_splat")
|
||||
if paper_strict:
|
||||
report = check_paper_strict(pipeline.config, REPO_ROOT)
|
||||
for line in format_report(report):
|
||||
print(line)
|
||||
if args.dry_run:
|
||||
return 0 if report["ok"] else 1
|
||||
validate_paper_strict(pipeline.config, REPO_ROOT)
|
||||
if args.dry_run:
|
||||
status = pipeline.check()
|
||||
if args.backend in {"auto", "builtin", "hybrid"} and status["builtin_ready"]:
|
||||
print("builtin: ready (numpy)")
|
||||
return 0 if args.backend == "builtin" else 0
|
||||
required = {"grounding_dino", "sam2", "instance_refinement", "lama", "anysplat"}
|
||||
for stage, info in status["stages"].items():
|
||||
state = "configured" if info["configured"] else "missing"
|
||||
print(f"{stage}: {state} ({info['env']})")
|
||||
return 0 if all(status["stages"][stage]["configured"] for stage in required) else 1
|
||||
|
||||
prune = None
|
||||
if args.no_prune or args.keep_ratio is not None:
|
||||
keep_ratio = args.keep_ratio if args.keep_ratio is not None else 0.3
|
||||
prune = PruneOptions(enabled=not args.no_prune, keep_ratio=keep_ratio)
|
||||
result = pipeline.run(
|
||||
input_path=args.input,
|
||||
output_dir=args.output,
|
||||
work_dir=args.work_dir,
|
||||
prompt=args.prompt,
|
||||
resume=args.resume,
|
||||
prune_options=prune,
|
||||
enable_scene_reconstruction=True,
|
||||
enable_object_reconstruction=False,
|
||||
synthetic_multiview=args.synthetic_multiview or getattr(args, "scene_completion", False),
|
||||
synthetic_multiview_mode="scene",
|
||||
)
|
||||
print(f"Real2Sim scene package: {result.package.root}")
|
||||
print(f"Scene PLY: {result.manifest.scene.gaussian_ply}")
|
||||
print(f"Scene XML: {result.package.scene_xml}")
|
||||
print(f"Demo config: {result.package.config_json}")
|
||||
print(f"Manifest: {result.package.manifest_json}")
|
||||
return 0
|
||||
|
||||
|
||||
def _package_existing(args) -> int:
|
||||
result = _pipeline(args).package_existing(
|
||||
source=args.source,
|
||||
scene_ply=args.scene_ply,
|
||||
scene_mesh=args.scene_mesh,
|
||||
objects_json=args.objects_json,
|
||||
output_dir=args.output,
|
||||
prompt=args.prompt,
|
||||
cameras_json=args.cameras_json,
|
||||
depth_map=args.depth_map,
|
||||
background_image=args.background_image,
|
||||
preview_image=args.preview_image,
|
||||
mesh_cleanup_report=args.mesh_cleanup_report,
|
||||
texture_bake_report=args.texture_bake_report,
|
||||
scene_structure_report=args.scene_structure_report,
|
||||
)
|
||||
print(f"Real2Sim package: {result.root}")
|
||||
print(f"Scene XML: {result.scene_xml}")
|
||||
print(f"Demo config: {result.config_json}")
|
||||
print(f"Manifest: {result.manifest_json}")
|
||||
return 0
|
||||
|
||||
|
||||
def _check(args) -> int:
|
||||
status = _pipeline(args).check()
|
||||
paper_strict = bool(getattr(args, "paper_strict", False))
|
||||
if paper_strict:
|
||||
status["paper_strict"] = check_paper_strict(_pipeline(args).config, REPO_ROOT)
|
||||
write_json(Path("/tmp/gsplayground_real2sim_check.json"), status)
|
||||
for stage, info in status["stages"].items():
|
||||
state = "configured" if info["configured"] else "missing"
|
||||
print(f"{stage}: {state} ({info['env']})")
|
||||
for module, ok in status["python_packages"].items():
|
||||
prefix = "controller_python." if paper_strict else ""
|
||||
print(f"{prefix}{module}: {'ok' if ok else 'missing'}")
|
||||
if paper_strict:
|
||||
for module in status["paper_strict"].get("python_modules", []):
|
||||
stage = module.get("stage", "unknown")
|
||||
name = module.get("name", "unknown")
|
||||
state = "ok" if module.get("ok") else "missing"
|
||||
print(f"paper_strict.{stage}.{name}: {state}")
|
||||
for line in format_report(status["paper_strict"]):
|
||||
print(line)
|
||||
print("Wrote /tmp/gsplayground_real2sim_check.json")
|
||||
return 0 if not paper_strict or status["paper_strict"]["ok"] else 1
|
||||
|
||||
|
||||
def _write_example_config(args) -> int:
|
||||
write_json(args.output, example_config())
|
||||
print(f"Wrote {args.output}")
|
||||
return 0
|
||||
|
||||
|
||||
def example_config() -> dict:
|
||||
return {
|
||||
"stages": {
|
||||
"synthetic_multiview": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/synthetic_multiview_expand.py",
|
||||
"--image",
|
||||
"{image}",
|
||||
"--output-dir",
|
||||
"{synthetic_multiview_dir}",
|
||||
"--frames-dir",
|
||||
"{synthetic_frames_dir}",
|
||||
"--prompt",
|
||||
"{synthetic_multiview_prompt}",
|
||||
"--size",
|
||||
"1536x1024",
|
||||
]
|
||||
},
|
||||
"grounding_dino": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/grounding_dino_detect.py",
|
||||
"--image",
|
||||
"{image}",
|
||||
"--prompt",
|
||||
"{prompt}",
|
||||
"--output",
|
||||
"{detections_json}",
|
||||
]
|
||||
},
|
||||
"sam2": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/sam2_segment.py",
|
||||
"--image",
|
||||
"{image}",
|
||||
"--detections",
|
||||
"{detections_json}",
|
||||
"--masks-dir",
|
||||
"{masks_dir}",
|
||||
"--output",
|
||||
"{instances_json}",
|
||||
]
|
||||
},
|
||||
"instance_refinement": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/instance_refine.py",
|
||||
"--instances",
|
||||
"{instances_json}",
|
||||
"--output",
|
||||
"{instances_json}",
|
||||
]
|
||||
},
|
||||
"lama": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/lama_inpaint.py",
|
||||
"--image",
|
||||
"{image}",
|
||||
"--instances",
|
||||
"{instances_json}",
|
||||
"--output",
|
||||
"{background_image}",
|
||||
"--prompt",
|
||||
"{prompt}",
|
||||
"--work-dir",
|
||||
"{work_dir}",
|
||||
"--redetect-command-env",
|
||||
"REAL2SIM_GROUNDING_DINO_COMMAND",
|
||||
"--segment-command-env",
|
||||
"REAL2SIM_SAM2_COMMAND",
|
||||
"--require-redetect-loop",
|
||||
]
|
||||
},
|
||||
"depth_estimation": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/depth_anything_depth.py",
|
||||
"--image",
|
||||
"{background_image}",
|
||||
"--output",
|
||||
"{depth_map}",
|
||||
"--model",
|
||||
"depth-anything/Depth-Anything-V2-Small-hf",
|
||||
]
|
||||
},
|
||||
"anysplat": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/anysplat_scene.py",
|
||||
"--image",
|
||||
"{background_image}",
|
||||
"--frames-dir",
|
||||
"{frames_dir}",
|
||||
"--output-dir",
|
||||
"{scene_reconstruction_dir}",
|
||||
"--ply",
|
||||
"{scene_ply}",
|
||||
"--mesh",
|
||||
"{scene_mesh}",
|
||||
"--cameras",
|
||||
"{cameras_json}",
|
||||
"--depth",
|
||||
"{depth_map}",
|
||||
"--max-views",
|
||||
"8",
|
||||
]
|
||||
},
|
||||
"sam3d": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/sam3d_object.py",
|
||||
"--image",
|
||||
"{image}",
|
||||
"--mask",
|
||||
"{mask}",
|
||||
"--instance",
|
||||
"{instance_json}",
|
||||
"--output-dir",
|
||||
"{object_dir}",
|
||||
"--ply",
|
||||
"{object_ply}",
|
||||
"--mesh",
|
||||
"{object_mesh}",
|
||||
"--pose",
|
||||
"{object_pose}",
|
||||
"--cameras",
|
||||
"{cameras_json}",
|
||||
"--match-camera-preprocess",
|
||||
"--object-centric-preprocess",
|
||||
"--object-preprocess-size",
|
||||
"896",
|
||||
"--object-crop-margin",
|
||||
"0.12",
|
||||
]
|
||||
},
|
||||
"pose_alignment": {
|
||||
"command": [
|
||||
"{python}",
|
||||
"/path/to/refine_pose_alignment.py",
|
||||
"--assets",
|
||||
"{aligned_assets_json}",
|
||||
"--instances",
|
||||
"{instances_json}",
|
||||
"--depth",
|
||||
"{depth_map}",
|
||||
"--cameras",
|
||||
"{cameras_json}",
|
||||
"--work-dir",
|
||||
"{work_dir}",
|
||||
"--output",
|
||||
"{aligned_assets_json}",
|
||||
]
|
||||
},
|
||||
},
|
||||
"builtin": {
|
||||
"max_scene_points": 180000,
|
||||
"max_object_points": 50000,
|
||||
"max_objects": 10,
|
||||
"min_component_area_ratio": 0.003,
|
||||
"stage_min_component_area_ratio": 0.0005,
|
||||
"focal_scale": 0.9,
|
||||
"external_depth_weight": 0.65,
|
||||
"color_enhance_strength": 0.35,
|
||||
"scene_point_scale": 0.018,
|
||||
"object_point_scale": 0.014,
|
||||
"opacity_logit": 4.5,
|
||||
"mask_close_passes": 2,
|
||||
"mask_dilate_passes": 1,
|
||||
"max_mask_overlap": 0.6,
|
||||
},
|
||||
"pruning": {
|
||||
"enabled": True,
|
||||
"keep_ratio": 0.85,
|
||||
"mode": "speedy_splat",
|
||||
"external_command": [
|
||||
"{python}",
|
||||
"scripts/real2sim_external/speedy_splat_prune.py",
|
||||
"--input",
|
||||
"{input}",
|
||||
"--output",
|
||||
"{output}",
|
||||
"--image",
|
||||
"{source_image}",
|
||||
"--cameras",
|
||||
"{cameras_json}",
|
||||
"--keep-ratio",
|
||||
"{keep_ratio}",
|
||||
],
|
||||
},
|
||||
}
|
||||
97
vendor/gs_playground/real2sim/commands.py
vendored
Normal file
97
vendor/gs_playground/real2sim/commands.py
vendored
Normal file
@ -0,0 +1,97 @@
|
||||
"""External command execution for Real2Sim stages.
|
||||
|
||||
The paper pipeline relies on large model systems that are not vendored in this
|
||||
preview repository. This module keeps those integrations explicit: each stage
|
||||
declares the command it needs and the artifacts it must produce.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Union
|
||||
|
||||
from .errors import MissingStageCommand, PipelineError
|
||||
|
||||
|
||||
CommandValue = Union[str, Sequence[str]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageCommand:
|
||||
name: str
|
||||
env_var: str
|
||||
description: str
|
||||
required: bool = True
|
||||
|
||||
|
||||
class _SafeFormatDict(dict[str, str]):
|
||||
def __missing__(self, key: str) -> str:
|
||||
raise KeyError(f"unknown command placeholder: {key}")
|
||||
|
||||
|
||||
def _format_arg(value: str, context: Mapping[str, Any]) -> str:
|
||||
flattened = {key: str(val) for key, val in context.items()}
|
||||
return value.format_map(_SafeFormatDict(flattened))
|
||||
|
||||
|
||||
def resolve_command(stage: StageCommand, config: Mapping[str, Any]) -> CommandValue | None:
|
||||
stages = config.get("stages", {})
|
||||
stage_config = stages.get(stage.name, {}) if isinstance(stages, Mapping) else {}
|
||||
configured = stage_config.get("command") if isinstance(stage_config, Mapping) else None
|
||||
env_command = os.environ.get(stage.env_var)
|
||||
command = command_preferring_env(configured, env_command)
|
||||
if command:
|
||||
return command
|
||||
|
||||
if stage.required:
|
||||
raise MissingStageCommand(
|
||||
f"{stage.name} is required but no command was configured. "
|
||||
f"Set stages.{stage.name}.command in the Real2Sim config or export {stage.env_var}. "
|
||||
f"Expected stage: {stage.description}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def command_preferring_env(configured: CommandValue | None, env_command: str | None) -> CommandValue | None:
|
||||
if env_command:
|
||||
return env_command
|
||||
return configured
|
||||
|
||||
|
||||
def command_contains_placeholder(command: Any) -> bool:
|
||||
if command is None:
|
||||
return False
|
||||
parts = command if isinstance(command, (list, tuple)) else [str(command)]
|
||||
return any("/path/to/" in str(part) for part in parts)
|
||||
|
||||
|
||||
def format_command(command: CommandValue, context: Mapping[str, Any]) -> list[str]:
|
||||
if isinstance(command, str):
|
||||
return [_format_arg(arg, context) for arg in shlex.split(command)]
|
||||
return [_format_arg(str(arg), context) for arg in command]
|
||||
|
||||
|
||||
def run_command(
|
||||
stage: StageCommand,
|
||||
command: CommandValue,
|
||||
context: Mapping[str, Any],
|
||||
*,
|
||||
cwd: Path,
|
||||
dry_run: bool = False,
|
||||
) -> list[str]:
|
||||
argv = format_command(command, context)
|
||||
if not argv:
|
||||
raise PipelineError(f"{stage.name} command is empty")
|
||||
|
||||
if dry_run:
|
||||
return argv
|
||||
|
||||
completed = subprocess.run(argv, cwd=cwd, check=False)
|
||||
if completed.returncode != 0:
|
||||
raise PipelineError(f"{stage.name} command failed with exit code {completed.returncode}: {argv}")
|
||||
return argv
|
||||
185
vendor/gs_playground/real2sim/contracts.py
vendored
Normal file
185
vendor/gs_playground/real2sim/contracts.py
vendored
Normal file
@ -0,0 +1,185 @@
|
||||
"""Artifact contracts for Real2Sim stage boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .errors import StageArtifactError
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
if not path.exists():
|
||||
raise StageArtifactError(f"missing JSON artifact: {path}")
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def require_file(path: Path, description: str) -> Path:
|
||||
if not path.is_file():
|
||||
raise StageArtifactError(f"missing {description}: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def require_dir(path: Path, description: str) -> Path:
|
||||
if not path.is_dir():
|
||||
raise StageArtifactError(f"missing {description}: {path}")
|
||||
return path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pose:
|
||||
position: tuple[float, float, float] = (0.0, 0.0, 0.0)
|
||||
quat_wxyz: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0)
|
||||
scale: tuple[float, float, float] = (1.0, 1.0, 1.0)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, path: Path) -> "Pose":
|
||||
data = read_json(path)
|
||||
if not isinstance(data, Mapping):
|
||||
raise StageArtifactError(f"pose artifact must be a JSON object: {path}")
|
||||
|
||||
pos = data.get("position", data.get("translation", [0.0, 0.0, 0.0]))
|
||||
quat = data.get("quat_wxyz", data.get("quaternion_wxyz"))
|
||||
if quat is None and "quaternion_xyzw" in data:
|
||||
qx, qy, qz, qw = _float_list(data["quaternion_xyzw"], 4, "quaternion_xyzw", path)
|
||||
quat = [qw, qx, qy, qz]
|
||||
if quat is None:
|
||||
quat = [1.0, 0.0, 0.0, 0.0]
|
||||
|
||||
scale_raw = data.get("scale", [1.0, 1.0, 1.0])
|
||||
if isinstance(scale_raw, (int, float)):
|
||||
scale = (float(scale_raw), float(scale_raw), float(scale_raw))
|
||||
else:
|
||||
scale = tuple(_float_list(scale_raw, 3, "scale", path))
|
||||
|
||||
return cls(
|
||||
position=tuple(_float_list(pos, 3, "position", path)),
|
||||
quat_wxyz=tuple(_float_list(quat, 4, "quat_wxyz", path)),
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
def to_json(self) -> dict[str, list[float]]:
|
||||
return {
|
||||
"position": list(self.position),
|
||||
"quat_wxyz": list(self.quat_wxyz),
|
||||
"scale": list(self.scale),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Instance:
|
||||
instance_id: str
|
||||
label: str
|
||||
score: float | None = None
|
||||
bbox_xyxy: tuple[float, float, float, float] | None = None
|
||||
mask_path: Path | None = None
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, raw: Mapping[str, Any], base_dir: Path) -> "Instance":
|
||||
raw_id = raw.get("id", raw.get("instance_id", raw.get("name")))
|
||||
label = raw.get("label", raw.get("class", raw_id))
|
||||
if raw_id is None or label is None:
|
||||
raise StageArtifactError(f"instance entry requires id/name and label: {raw}")
|
||||
bbox = raw.get("bbox_xyxy", raw.get("bbox"))
|
||||
mask = raw.get("mask_path", raw.get("mask"))
|
||||
return cls(
|
||||
instance_id=str(raw_id),
|
||||
label=str(label),
|
||||
score=float(raw["score"]) if raw.get("score") is not None else None,
|
||||
bbox_xyxy=tuple(_float_list(bbox, 4, "bbox", base_dir)) if bbox is not None else None,
|
||||
mask_path=(base_dir / str(mask)).resolve() if mask else None,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Real2SimAsset:
|
||||
name: str
|
||||
label: str
|
||||
gaussian_ply: Path
|
||||
mesh_obj: Path | None = None
|
||||
pose: Pose = field(default_factory=Pose)
|
||||
dynamic: bool = False
|
||||
source_instance_id: str | None = None
|
||||
|
||||
def to_json(self, root: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"label": self.label,
|
||||
"gaussian_ply": _relative_or_absolute(self.gaussian_ply, root),
|
||||
"mesh_obj": _relative_or_absolute(self.mesh_obj, root) if self.mesh_obj else None,
|
||||
"pose": self.pose.to_json(),
|
||||
"dynamic": self.dynamic,
|
||||
"source_instance_id": self.source_instance_id,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Real2SimManifest:
|
||||
source: Path
|
||||
prompt: str
|
||||
scene: Real2SimAsset
|
||||
objects: tuple[Real2SimAsset, ...] = ()
|
||||
camera_artifacts: dict[str, Path] = field(default_factory=dict)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_json(self, root: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"source": _relative_or_absolute(self.source, root),
|
||||
"prompt": self.prompt,
|
||||
"scene": self.scene.to_json(root),
|
||||
"objects": [obj.to_json(root) for obj in self.objects],
|
||||
"camera_artifacts": {
|
||||
key: _relative_or_absolute(value, root)
|
||||
for key, value in sorted(self.camera_artifacts.items())
|
||||
},
|
||||
"metadata": self.metadata,
|
||||
}
|
||||
|
||||
|
||||
def load_instances(path: Path) -> list[Instance]:
|
||||
data = read_json(path)
|
||||
entries = data.get("instances", data) if isinstance(data, Mapping) else data
|
||||
if not isinstance(entries, list):
|
||||
raise StageArtifactError(f"instances artifact must be a list or object with instances list: {path}")
|
||||
return [Instance.from_mapping(entry, path.parent) for entry in entries]
|
||||
|
||||
|
||||
def validate_detections(path: Path) -> None:
|
||||
data = read_json(path)
|
||||
entries = data.get("detections", data) if isinstance(data, Mapping) else data
|
||||
if not isinstance(entries, list):
|
||||
raise StageArtifactError(f"detections artifact must be a list or object with detections list: {path}")
|
||||
for entry in entries:
|
||||
if not isinstance(entry, Mapping):
|
||||
raise StageArtifactError(f"detection entries must be JSON objects: {path}")
|
||||
if "bbox" not in entry and "bbox_xyxy" not in entry:
|
||||
raise StageArtifactError(f"detection entry missing bbox/bbox_xyxy: {entry}")
|
||||
|
||||
|
||||
def _float_list(value: Any, size: int, field_name: str, path: Path) -> list[float]:
|
||||
if not isinstance(value, Iterable) or isinstance(value, (str, bytes)):
|
||||
raise StageArtifactError(f"{field_name} must be a list with {size} values in {path}")
|
||||
vals = [float(item) for item in value]
|
||||
if len(vals) != size:
|
||||
raise StageArtifactError(f"{field_name} must have {size} values in {path}, got {len(vals)}")
|
||||
return vals
|
||||
|
||||
|
||||
def _relative_or_absolute(path: Path | None, root: Path) -> str | None:
|
||||
if path is None:
|
||||
return None
|
||||
try:
|
||||
return path.resolve().relative_to(root.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.resolve().as_posix()
|
||||
15
vendor/gs_playground/real2sim/errors.py
vendored
Normal file
15
vendor/gs_playground/real2sim/errors.py
vendored
Normal file
@ -0,0 +1,15 @@
|
||||
"""Pipeline-specific exceptions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class PipelineError(RuntimeError):
|
||||
"""Base class for Real2Sim pipeline failures."""
|
||||
|
||||
|
||||
class MissingStageCommand(PipelineError):
|
||||
"""Raised when a required external model stage is not configured."""
|
||||
|
||||
|
||||
class StageArtifactError(PipelineError):
|
||||
"""Raised when a stage does not produce the expected artifacts."""
|
||||
140
vendor/gs_playground/real2sim/image_io.py
vendored
Normal file
140
vendor/gs_playground/real2sim/image_io.py
vendored
Normal file
@ -0,0 +1,140 @@
|
||||
"""Small image I/O helpers used by the built-in Real2Sim baseline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .errors import PipelineError
|
||||
|
||||
|
||||
def read_rgb(path: Path) -> np.ndarray:
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".ppm":
|
||||
return _read_ppm(path)
|
||||
|
||||
try:
|
||||
import matplotlib.image as mpimg
|
||||
except Exception as exc: # pragma: no cover - depends on runtime env
|
||||
raise PipelineError(
|
||||
f"reading {suffix or 'this image'} requires matplotlib in this environment. "
|
||||
"Use a .ppm input or run through the project uv environment."
|
||||
) from exc
|
||||
|
||||
arr = np.asarray(mpimg.imread(path), dtype=np.float32)
|
||||
if arr.ndim == 2:
|
||||
arr = np.repeat(arr[:, :, None], 3, axis=2)
|
||||
if arr.shape[-1] == 4:
|
||||
alpha = arr[:, :, 3:4]
|
||||
arr = arr[:, :, :3] * alpha + (1.0 - alpha)
|
||||
else:
|
||||
arr = arr[:, :, :3]
|
||||
if arr.dtype.kind in {"u", "i"} or arr.max() > 1.5:
|
||||
arr = arr / 255.0
|
||||
return np.clip(arr, 0.0, 1.0).astype(np.float32)
|
||||
|
||||
|
||||
def read_gray(path: Path) -> np.ndarray:
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".pgm":
|
||||
return _read_pgm(path)
|
||||
|
||||
try:
|
||||
import matplotlib.image as mpimg
|
||||
except Exception as exc: # pragma: no cover - depends on runtime env
|
||||
raise PipelineError(
|
||||
f"reading {suffix or 'this mask'} requires matplotlib in this environment. "
|
||||
"Use a .pgm mask or run through the project uv environment."
|
||||
) from exc
|
||||
|
||||
arr = np.asarray(mpimg.imread(path), dtype=np.float32)
|
||||
if arr.ndim == 3:
|
||||
arr = arr[:, :, :3].mean(axis=2)
|
||||
if arr.dtype.kind in {"u", "i"} or arr.max() > 1.5:
|
||||
arr = arr / 255.0
|
||||
return np.clip(arr, 0.0, 1.0).astype(np.float32)
|
||||
|
||||
|
||||
def write_rgb(path: Path, image: np.ndarray) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
arr = np.clip(image, 0.0, 1.0)
|
||||
if path.suffix.lower() == ".ppm":
|
||||
_write_ppm(path, arr)
|
||||
return
|
||||
try:
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
mpimg.imsave(path, arr)
|
||||
except Exception:
|
||||
_write_ppm(path, arr)
|
||||
|
||||
|
||||
def write_gray(path: Path, image: np.ndarray) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
arr = np.clip(image, 0.0, 1.0)
|
||||
if path.suffix.lower() == ".pgm":
|
||||
_write_pgm(path, arr)
|
||||
return
|
||||
try:
|
||||
import matplotlib.image as mpimg
|
||||
|
||||
mpimg.imsave(path, arr, cmap="gray")
|
||||
except Exception:
|
||||
_write_pgm(path, arr)
|
||||
|
||||
|
||||
def _read_ppm(path: Path) -> np.ndarray:
|
||||
with path.open("rb") as f:
|
||||
magic = f.readline().strip()
|
||||
if magic != b"P6":
|
||||
raise PipelineError(f"only binary P6 PPM input is supported without matplotlib: {path}")
|
||||
dims = _next_ppm_token_line(f)
|
||||
width, height = [int(x) for x in dims.split()]
|
||||
maxval = int(_next_ppm_token_line(f))
|
||||
if maxval <= 0 or maxval > 255:
|
||||
raise PipelineError(f"unsupported PPM max value {maxval}: {path}")
|
||||
data = np.frombuffer(f.read(width * height * 3), dtype=np.uint8)
|
||||
if data.size != width * height * 3:
|
||||
raise PipelineError(f"truncated PPM input: {path}")
|
||||
return data.reshape(height, width, 3).astype(np.float32) / float(maxval)
|
||||
|
||||
|
||||
def _read_pgm(path: Path) -> np.ndarray:
|
||||
with path.open("rb") as f:
|
||||
magic = f.readline().strip()
|
||||
if magic != b"P5":
|
||||
raise PipelineError(f"only binary P5 PGM input is supported without matplotlib: {path}")
|
||||
dims = _next_ppm_token_line(f)
|
||||
width, height = [int(x) for x in dims.split()]
|
||||
maxval = int(_next_ppm_token_line(f))
|
||||
if maxval <= 0 or maxval > 255:
|
||||
raise PipelineError(f"unsupported PGM max value {maxval}: {path}")
|
||||
data = np.frombuffer(f.read(width * height), dtype=np.uint8)
|
||||
if data.size != width * height:
|
||||
raise PipelineError(f"truncated PGM input: {path}")
|
||||
return data.reshape(height, width).astype(np.float32) / float(maxval)
|
||||
|
||||
|
||||
def _write_ppm(path: Path, image: np.ndarray) -> None:
|
||||
arr = (np.clip(image, 0.0, 1.0) * 255.0 + 0.5).astype(np.uint8)
|
||||
with path.open("wb") as f:
|
||||
f.write(f"P6\n{arr.shape[1]} {arr.shape[0]}\n255\n".encode("ascii"))
|
||||
f.write(arr[:, :, :3].tobytes())
|
||||
|
||||
|
||||
def _write_pgm(path: Path, image: np.ndarray) -> None:
|
||||
arr = (np.clip(image, 0.0, 1.0) * 255.0 + 0.5).astype(np.uint8)
|
||||
with path.open("wb") as f:
|
||||
f.write(f"P5\n{arr.shape[1]} {arr.shape[0]}\n255\n".encode("ascii"))
|
||||
f.write(arr.tobytes())
|
||||
|
||||
|
||||
def _next_ppm_token_line(f) -> str:
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
raise PipelineError("unexpected EOF while reading PPM header")
|
||||
line = line.strip()
|
||||
if line and not line.startswith(b"#"):
|
||||
return line.decode("ascii")
|
||||
3145
vendor/gs_playground/real2sim/packaging.py
vendored
Normal file
3145
vendor/gs_playground/real2sim/packaging.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
320
vendor/gs_playground/real2sim/paper_strict.py
vendored
Normal file
320
vendor/gs_playground/real2sim/paper_strict.py
vendored
Normal file
@ -0,0 +1,320 @@
|
||||
"""Paper-strict Real2Sim runtime validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util as importlib_util
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .commands import command_contains_placeholder, command_preferring_env
|
||||
from .errors import PipelineError
|
||||
|
||||
|
||||
REQUIRED_STAGES = (
|
||||
"grounding_dino",
|
||||
"sam2",
|
||||
"instance_refinement",
|
||||
"lama",
|
||||
"anysplat",
|
||||
"sam3d",
|
||||
"pose_alignment",
|
||||
)
|
||||
MIN_SAM3D_VRAM_MIB = 32768
|
||||
REQUIRED_STAGE_PYTHON_MODULES = {
|
||||
"grounding_dino": ("transformers",),
|
||||
"sam2": ("sam2",),
|
||||
"instance_refinement": ("numpy", "PIL"),
|
||||
"lama": ("simple_lama_inpainting",),
|
||||
"anysplat": ("torch", "pytorch3d", "gsplat", "plyfile", "src.model.model.anysplat", "src.model.ply_export"),
|
||||
"sam3d": ("torch", "pytorch3d", "kaolin", "gsplat", "sam3d_objects", "inference"),
|
||||
"pose_alignment": ("numpy", "PIL"),
|
||||
"speedy_splat": ("gaussian_renderer",),
|
||||
}
|
||||
|
||||
|
||||
def validate_paper_strict(
|
||||
config: Mapping[str, Any],
|
||||
repo_root: Path,
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
gpu_memory_mib: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
report = check_paper_strict(config, repo_root, env=env, gpu_memory_mib=gpu_memory_mib)
|
||||
if report["ok"]:
|
||||
return report
|
||||
summary = "; ".join(f"{failure['name']}: {failure['message']}" for failure in report["failures"])
|
||||
raise PipelineError(f"paper-strict Real2Sim prerequisites are not satisfied: {summary}")
|
||||
|
||||
|
||||
def check_paper_strict(
|
||||
config: Mapping[str, Any],
|
||||
repo_root: Path,
|
||||
*,
|
||||
env: Mapping[str, str] | None = None,
|
||||
gpu_memory_mib: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
env_map = dict(os.environ if env is None else env)
|
||||
failures: list[dict[str, str]] = []
|
||||
checks: list[dict[str, Any]] = []
|
||||
|
||||
stages = config.get("stages", {}) if isinstance(config.get("stages", {}), Mapping) else {}
|
||||
for stage in REQUIRED_STAGES:
|
||||
command = configured_stage_command(stages, stage, env_map)
|
||||
ok = bool(command)
|
||||
checks.append({"name": f"stage_{stage}", "ok": ok})
|
||||
if not ok:
|
||||
failures.append({"name": f"stage_{stage}", "message": f"missing command for {stage}"})
|
||||
elif command_contains_placeholder(command):
|
||||
failures.append({"name": f"stage_{stage}", "message": f"command for {stage} still contains a placeholder path"})
|
||||
else:
|
||||
executable_error = command_executable_error(command)
|
||||
if executable_error:
|
||||
failures.append({"name": f"stage_{stage}", "message": executable_error})
|
||||
|
||||
pruning = config.get("pruning", {}) if isinstance(config.get("pruning", {}), Mapping) else {}
|
||||
pruning_mode = str(pruning.get("mode", "")).replace("-", "_").lower()
|
||||
speedy_command = command_preferring_env(pruning.get("external_command"), env_map.get("REAL2SIM_SPEEDY_SPLAT_COMMAND", ""))
|
||||
if pruning_mode != "speedy_splat":
|
||||
failures.append({"name": "speedy_splat_mode", "message": "pruning.mode must be speedy_splat"})
|
||||
if not speedy_command:
|
||||
failures.append({"name": "speedy_splat_command", "message": "missing Speedy-Splat external command"})
|
||||
elif command_contains_placeholder(speedy_command):
|
||||
failures.append({"name": "speedy_splat_command", "message": "Speedy-Splat command still contains a placeholder path"})
|
||||
else:
|
||||
executable_error = command_executable_error(speedy_command)
|
||||
if executable_error:
|
||||
failures.append({"name": "speedy_splat_command", "message": executable_error})
|
||||
|
||||
nested_grounding = configured_stage_command(stages, "grounding_dino", env_map)
|
||||
nested_sam2 = configured_stage_command(stages, "sam2", env_map)
|
||||
if not nested_grounding or not nested_sam2:
|
||||
failures.append(
|
||||
{
|
||||
"name": "inpainting_redetect_loop",
|
||||
"message": "strict LaMa recovery requires configured GroundingDINO and SAM2 commands",
|
||||
}
|
||||
)
|
||||
|
||||
anysplat_root = Path(env_map.get("REAL2SIM_ANYSPLAT_ROOT", ""))
|
||||
require_path(failures, "anysplat_root", anysplat_root, "REAL2SIM_ANYSPLAT_ROOT")
|
||||
if anysplat_root:
|
||||
require_path(failures, "anysplat_api", anysplat_root / "src" / "model" / "model" / "anysplat.py", "AnySplat API")
|
||||
require_path(failures, "anysplat_export", anysplat_root / "src" / "model" / "ply_export.py", "AnySplat export_ply")
|
||||
|
||||
sam3d_root = Path(env_map.get("REAL2SIM_SAM3D_ROOT", ""))
|
||||
require_path(failures, "sam3d_root", sam3d_root, "REAL2SIM_SAM3D_ROOT")
|
||||
if sam3d_root:
|
||||
require_path(failures, "sam3d_inference", sam3d_root / "notebook" / "inference.py", "SAM-3D notebook inference API")
|
||||
sam3d_config = Path(env_map.get("REAL2SIM_SAM3D_CONFIG", "")) if env_map.get("REAL2SIM_SAM3D_CONFIG") else sam3d_root / "checkpoints" / "hf" / "pipeline.yaml"
|
||||
require_path(failures, "sam3d_config", sam3d_config, "SAM-3D checkpoint pipeline.yaml")
|
||||
|
||||
sam2_checkpoint = Path(env_map.get("REAL2SIM_SAM2_CHECKPOINT", ""))
|
||||
require_path(failures, "sam2_checkpoint", sam2_checkpoint, "REAL2SIM_SAM2_CHECKPOINT")
|
||||
speedy_root = Path(env_map.get("REAL2SIM_SPEEDY_SPLAT_ROOT", ""))
|
||||
require_path(failures, "speedy_splat_root", speedy_root, "REAL2SIM_SPEEDY_SPLAT_ROOT")
|
||||
|
||||
module_checks = []
|
||||
for stage, module_names in REQUIRED_STAGE_PYTHON_MODULES.items():
|
||||
command = speedy_command if stage == "speedy_splat" else configured_stage_command(stages, stage, env_map)
|
||||
if not command:
|
||||
continue
|
||||
if command_contains_placeholder(command) or command_executable_error(command):
|
||||
continue
|
||||
python_executable = python_executable_for_command(command)
|
||||
extra_paths = stage_module_extra_paths(stage, env_map)
|
||||
for module_name in module_names:
|
||||
ok = module_available(module_name, python_executable, repo_root, env_map, extra_paths=extra_paths)
|
||||
module_checks.append(
|
||||
{
|
||||
"stage": stage,
|
||||
"name": module_name,
|
||||
"ok": ok,
|
||||
"python": python_executable,
|
||||
"extra_paths": list(extra_paths),
|
||||
}
|
||||
)
|
||||
if not ok:
|
||||
failures.append(
|
||||
{
|
||||
"name": f"python_module_{module_name}",
|
||||
"message": (
|
||||
f"missing required Python module for paper-strict Real2Sim stage "
|
||||
f"{stage}: {module_name} in {python_executable}"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
detected_vram = detect_gpu_memory_mib() if gpu_memory_mib is None else gpu_memory_mib
|
||||
if detected_vram is None:
|
||||
failures.append({"name": "sam3d_vram", "message": "could not determine NVIDIA GPU memory"})
|
||||
elif detected_vram < MIN_SAM3D_VRAM_MIB:
|
||||
failures.append(
|
||||
{
|
||||
"name": "sam3d_vram",
|
||||
"message": f"SAM-3D official setup requires at least {MIN_SAM3D_VRAM_MIB} MiB VRAM, found {detected_vram} MiB",
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": not failures,
|
||||
"checks": checks,
|
||||
"python_modules": module_checks,
|
||||
"failures": failures,
|
||||
"gpu_memory_mib": detected_vram,
|
||||
"required_stages": list(REQUIRED_STAGES),
|
||||
}
|
||||
|
||||
|
||||
def stage_env(stage: str) -> str:
|
||||
return {
|
||||
"grounding_dino": "REAL2SIM_GROUNDING_DINO_COMMAND",
|
||||
"sam2": "REAL2SIM_SAM2_COMMAND",
|
||||
"instance_refinement": "REAL2SIM_INSTANCE_REFINEMENT_COMMAND",
|
||||
"lama": "REAL2SIM_LAMA_COMMAND",
|
||||
"anysplat": "REAL2SIM_ANYSPLAT_COMMAND",
|
||||
"sam3d": "REAL2SIM_SAM3D_COMMAND",
|
||||
"pose_alignment": "REAL2SIM_POSE_ALIGNMENT_COMMAND",
|
||||
}[stage]
|
||||
|
||||
|
||||
def configured_stage_command(stages: Mapping[str, Any], stage: str, env_map: Mapping[str, str]) -> Any:
|
||||
stage_config = stages.get(stage, {}) if isinstance(stages, Mapping) else {}
|
||||
command = stage_config.get("command") if isinstance(stage_config, Mapping) else None
|
||||
return command_preferring_env(command, env_map.get(stage_env(stage), ""))
|
||||
|
||||
|
||||
def require_path(failures: list[dict[str, str]], name: str, path: Path, description: str) -> None:
|
||||
if not str(path) or str(path) == "." or not path.exists():
|
||||
failures.append({"name": name, "message": f"missing {description}: {path}"})
|
||||
|
||||
|
||||
def command_executable_error(command: Any) -> str | None:
|
||||
executable = command_executable(command)
|
||||
if not executable or ("{" in executable and "}" in executable):
|
||||
return None
|
||||
if os.sep in executable or (os.altsep and os.altsep in executable):
|
||||
path = Path(executable)
|
||||
if not path.exists():
|
||||
return f"command executable does not exist: {executable}"
|
||||
if path.is_dir():
|
||||
return f"command executable is a directory: {executable}"
|
||||
if not os.access(path, os.X_OK):
|
||||
return f"command executable is not executable: {executable}"
|
||||
return None
|
||||
if shutil.which(executable) is None:
|
||||
return f"command executable not found on PATH: {executable}"
|
||||
return None
|
||||
|
||||
|
||||
def command_executable(command: Any) -> str | None:
|
||||
if command is None:
|
||||
return None
|
||||
if isinstance(command, str):
|
||||
try:
|
||||
parts = shlex.split(command)
|
||||
except ValueError:
|
||||
return ""
|
||||
elif isinstance(command, (list, tuple)):
|
||||
parts = [str(part) for part in command]
|
||||
else:
|
||||
parts = [str(command)]
|
||||
return parts[0] if parts else None
|
||||
|
||||
|
||||
def python_executable_for_command(command: Any) -> str:
|
||||
executable = command_executable(command)
|
||||
if not executable or ("{" in executable and "}" in executable):
|
||||
return sys.executable
|
||||
return executable
|
||||
|
||||
|
||||
def stage_module_extra_paths(stage: str, env_map: Mapping[str, str]) -> tuple[str, ...]:
|
||||
paths: list[str] = []
|
||||
if stage == "anysplat":
|
||||
root = env_map.get("REAL2SIM_ANYSPLAT_ROOT", "")
|
||||
if root:
|
||||
paths.append(root)
|
||||
elif stage == "sam3d":
|
||||
root = env_map.get("REAL2SIM_SAM3D_ROOT", "")
|
||||
if root:
|
||||
paths.extend([(Path(root) / "notebook").as_posix(), root])
|
||||
elif stage == "speedy_splat":
|
||||
root = env_map.get("REAL2SIM_SPEEDY_SPLAT_ROOT", "")
|
||||
if root:
|
||||
paths.append(root)
|
||||
return tuple(paths)
|
||||
|
||||
|
||||
def module_available(
|
||||
module_name: str,
|
||||
python_executable: str,
|
||||
repo_root: Path,
|
||||
env_map: Mapping[str, str],
|
||||
*,
|
||||
extra_paths: tuple[str, ...] = (),
|
||||
) -> bool:
|
||||
if python_executable == sys.executable and not extra_paths and not env_map.get("PYTHONPATH"):
|
||||
try:
|
||||
return importlib_util.find_spec(module_name) is not None
|
||||
except (ImportError, AttributeError, ValueError):
|
||||
return False
|
||||
|
||||
module_env = dict(os.environ)
|
||||
module_env.update(env_map)
|
||||
module_env.setdefault("LIDRA_SKIP_INIT", "1")
|
||||
pythonpath_parts = [path for path in extra_paths if path]
|
||||
if module_env.get("PYTHONPATH"):
|
||||
pythonpath_parts.append(module_env["PYTHONPATH"])
|
||||
if pythonpath_parts:
|
||||
module_env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts)
|
||||
code = "import importlib, sys; importlib.import_module(sys.argv[1])"
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[python_executable, "-c", code, module_name],
|
||||
cwd=repo_root,
|
||||
env=module_env,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except (FileNotFoundError, NotADirectoryError, PermissionError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
return completed.returncode == 0
|
||||
|
||||
|
||||
def detect_gpu_memory_mib() -> int | None:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if completed.returncode != 0:
|
||||
return None
|
||||
values = []
|
||||
for line in completed.stdout.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
values.append(int(float(line.split()[0])))
|
||||
except ValueError:
|
||||
continue
|
||||
return max(values) if values else None
|
||||
|
||||
|
||||
def format_report(report: Mapping[str, Any]) -> list[str]:
|
||||
lines = [f"paper_strict: {'ok' if report.get('ok') else 'failed'}"]
|
||||
for failure in report.get("failures", []):
|
||||
lines.append(f"{failure['name']}: {failure['message']}")
|
||||
return lines
|
||||
2819
vendor/gs_playground/real2sim/pipeline.py
vendored
Normal file
2819
vendor/gs_playground/real2sim/pipeline.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
94
vendor/gs_playground/real2sim/pruning.py
vendored
Normal file
94
vendor/gs_playground/real2sim/pruning.py
vendored
Normal file
@ -0,0 +1,94 @@
|
||||
"""3DGS pruning helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Sequence, Union
|
||||
|
||||
from .commands import format_command
|
||||
from .errors import PipelineError
|
||||
|
||||
|
||||
CommandValue = Union[str, Sequence[str]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PruneOptions:
|
||||
enabled: bool = True
|
||||
keep_ratio: float = 0.3
|
||||
keep_count: int | None = None
|
||||
mode: str = "opacity_area"
|
||||
external_command: CommandValue | None = None
|
||||
|
||||
|
||||
def prune_ply(
|
||||
src: Path,
|
||||
dst: Path,
|
||||
options: PruneOptions,
|
||||
repo_root: Path,
|
||||
*,
|
||||
extra_context: Mapping[str, Any] | None = None,
|
||||
) -> Path:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not options.enabled or options.keep_ratio >= 1.0:
|
||||
shutil.copy2(src, dst)
|
||||
return dst
|
||||
|
||||
if options.mode.replace("-", "_").lower() in {"speedy_splat", "speedysplat"}:
|
||||
return prune_with_speedy_splat(src, dst, options, repo_root, extra_context=extra_context)
|
||||
|
||||
script = repo_root / "benchmark" / "scripts" / "prune_gaussians.py"
|
||||
if not script.is_file():
|
||||
raise PipelineError(f"pruning script not found: {script}")
|
||||
|
||||
cmd = [sys.executable, script.as_posix(), "--input", src.as_posix(), "--output", dst.as_posix(), "--mode", options.mode]
|
||||
if options.keep_count is not None:
|
||||
cmd.extend(["--keep-count", str(options.keep_count)])
|
||||
else:
|
||||
cmd.extend(["--keep-ratio", str(options.keep_ratio)])
|
||||
|
||||
completed = subprocess.run(cmd, cwd=repo_root, check=False)
|
||||
if completed.returncode != 0:
|
||||
raise PipelineError(f"3DGS pruning failed with exit code {completed.returncode}: {cmd}")
|
||||
return dst
|
||||
|
||||
|
||||
def prune_with_speedy_splat(
|
||||
src: Path,
|
||||
dst: Path,
|
||||
options: PruneOptions,
|
||||
repo_root: Path,
|
||||
*,
|
||||
extra_context: Mapping[str, Any] | None = None,
|
||||
) -> Path:
|
||||
if options.external_command is None:
|
||||
raise PipelineError(
|
||||
"Speedy-Splat pruning requires pruning.external_command or an equivalent "
|
||||
"configured command. The built-in opacity_area pruner is not paper-equivalent."
|
||||
)
|
||||
extras = dict(extra_context or {})
|
||||
context = {
|
||||
"input": src,
|
||||
"output": dst,
|
||||
"keep_ratio": options.keep_ratio,
|
||||
"keep_count": options.keep_count or "",
|
||||
"mode": options.mode,
|
||||
"python": sys.executable,
|
||||
"source": extras.get("source", ""),
|
||||
"source_image": extras.get("source_image", extras.get("image", "")),
|
||||
"image": extras.get("image", extras.get("source_image", "")),
|
||||
"cameras_json": extras.get("cameras_json", extras.get("cameras", "")),
|
||||
"cameras": extras.get("cameras", extras.get("cameras_json", "")),
|
||||
}
|
||||
argv = format_command(options.external_command, context)
|
||||
completed = subprocess.run(argv, cwd=repo_root, check=False)
|
||||
if completed.returncode != 0:
|
||||
raise PipelineError(f"Speedy-Splat pruning failed with exit code {completed.returncode}: {argv}")
|
||||
if not dst.exists():
|
||||
raise PipelineError(f"Speedy-Splat command did not write output PLY: {dst}")
|
||||
return dst
|
||||
281
vendor/gs_playground/scripts/real2sim_capture_acceptance.py
vendored
Normal file
281
vendor/gs_playground/scripts/real2sim_capture_acceptance.py
vendored
Normal file
@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a capture acceptance report for high-fidelity Real2Sim reconstruction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Evaluate whether a capture is ready for high-fidelity Real2Sim reconstruction")
|
||||
parser.add_argument("--sfm-report", type=Path, required=True, help="COLMAP capture report JSON")
|
||||
parser.add_argument("--output-json", type=Path, required=True)
|
||||
parser.add_argument("--output-md", type=Path, required=True)
|
||||
parser.add_argument("--min-frames", type=int, default=30)
|
||||
parser.add_argument("--min-registered-ratio", type=float, default=0.70)
|
||||
parser.add_argument("--min-registered-images", type=int, default=20)
|
||||
parser.add_argument("--min-sparse-points", type=int, default=5000)
|
||||
parser.add_argument("--min-verified-pairs", type=int, default=40)
|
||||
parser.add_argument(
|
||||
"--min-camera-path-length",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Minimum COLMAP camera path length for space-level capture acceptance",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-camera-planar-coverage-ratio",
|
||||
type=float,
|
||||
default=0.12,
|
||||
help="Minimum second-largest camera extent / largest extent ratio; rejects nearly 1D camera tracks",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-camera-step-ratio",
|
||||
type=float,
|
||||
default=0.55,
|
||||
help="Maximum largest camera step / full path length ratio; rejects discontinuous captures",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
sfm = json.loads(args.sfm_report.read_text(encoding="utf-8"))
|
||||
report = build_acceptance_report(sfm, args)
|
||||
args.output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
args.output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
return 0 if report["accepted"] else 2
|
||||
|
||||
|
||||
def build_acceptance_report(sfm: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]:
|
||||
stats = sfm.get("database_stats") if isinstance(sfm.get("database_stats"), dict) else {}
|
||||
camera_metrics = camera_space_metrics(sfm)
|
||||
metrics = {
|
||||
"frame_count": int(sfm.get("frame_count", 0) or 0),
|
||||
"registered_image_count": int(sfm.get("registered_image_count", 0) or 0),
|
||||
"registered_ratio": float(sfm.get("registered_ratio", 0.0) or 0.0),
|
||||
"sparse_point_count": int(sfm.get("sparse_point_count", 0) or 0),
|
||||
"keypoint_total": int(stats.get("keypoint_total", 0) or 0),
|
||||
"keypoint_mean": float(stats.get("keypoint_mean", 0.0) or 0.0),
|
||||
"matched_pair_count": int(stats.get("matched_pair_count", 0) or 0),
|
||||
"verified_pair_count": int(stats.get("verified_pair_count", 0) or 0),
|
||||
"verified_pair_mean": float(stats.get("verified_pair_mean", 0.0) or 0.0),
|
||||
**camera_metrics,
|
||||
}
|
||||
gates = {
|
||||
"enough_frames": gate(metrics["frame_count"] >= args.min_frames, metrics["frame_count"], args.min_frames),
|
||||
"enough_registered_images": gate(
|
||||
metrics["registered_image_count"] >= args.min_registered_images,
|
||||
metrics["registered_image_count"],
|
||||
args.min_registered_images,
|
||||
),
|
||||
"registered_ratio": gate(
|
||||
metrics["registered_ratio"] >= args.min_registered_ratio,
|
||||
round(metrics["registered_ratio"], 4),
|
||||
args.min_registered_ratio,
|
||||
),
|
||||
"sparse_points": gate(metrics["sparse_point_count"] >= args.min_sparse_points, metrics["sparse_point_count"], args.min_sparse_points),
|
||||
"verified_pairs": gate(metrics["verified_pair_count"] >= args.min_verified_pairs, metrics["verified_pair_count"], args.min_verified_pairs),
|
||||
"camera_path_length": gate(
|
||||
metrics["camera_path_length"] >= args.min_camera_path_length,
|
||||
round(metrics["camera_path_length"], 4),
|
||||
args.min_camera_path_length,
|
||||
),
|
||||
"camera_planar_coverage": gate(
|
||||
metrics["camera_planar_coverage_ratio"] >= args.min_camera_planar_coverage_ratio,
|
||||
round(metrics["camera_planar_coverage_ratio"], 4),
|
||||
args.min_camera_planar_coverage_ratio,
|
||||
),
|
||||
"camera_step_consistency": gate(
|
||||
metrics["camera_max_step_ratio"] is not None and metrics["camera_max_step_ratio"] <= args.max_camera_step_ratio,
|
||||
round(metrics["camera_max_step_ratio"], 4) if metrics["camera_max_step_ratio"] is not None else None,
|
||||
args.max_camera_step_ratio,
|
||||
),
|
||||
}
|
||||
accepted = all(item["passed"] for item in gates.values())
|
||||
blockers = [name for name, item in gates.items() if not item["passed"]]
|
||||
return {
|
||||
"format": "gs-playground-real2sim-capture-acceptance-v1",
|
||||
"accepted": accepted,
|
||||
"target_quality": "polycam_realitycapture_like_space",
|
||||
"decision": "ready_for_pose_initialized_reconstruction" if accepted else "recapture_required_before_high_fidelity_reconstruction",
|
||||
"metrics": metrics,
|
||||
"gates": gates,
|
||||
"blockers": blockers,
|
||||
"sfm_warnings": sfm.get("warnings", []),
|
||||
"sfm_recommendations": sfm.get("recommendations", []),
|
||||
"capture_requirements": capture_requirements(blockers, metrics),
|
||||
}
|
||||
|
||||
|
||||
def camera_space_metrics(sfm: dict[str, Any]) -> dict[str, Any]:
|
||||
path = sfm.get("camera_path") if isinstance(sfm.get("camera_path"), dict) else {}
|
||||
poses = camera_positions(sfm.get("camera_poses"))
|
||||
if poses:
|
||||
extents = axis_extents(poses)
|
||||
path_length = path_length_from_positions(poses)
|
||||
step_distances = [
|
||||
distance(poses[index - 1], poses[index])
|
||||
for index in range(1, len(poses))
|
||||
]
|
||||
closure_distance = distance(poses[0], poses[-1]) if len(poses) >= 2 else 0.0
|
||||
else:
|
||||
extents = numeric_list(path.get("bounds_extent"), expected=3) or [0.0, 0.0, 0.0]
|
||||
path_length = float(path.get("path_length", 0.0) or 0.0)
|
||||
step_distances = []
|
||||
closure_distance = None
|
||||
|
||||
sorted_extents = sorted((max(float(value), 0.0) for value in extents), reverse=True)
|
||||
dominant_extent = sorted_extents[0] if sorted_extents else 0.0
|
||||
secondary_extent = sorted_extents[1] if len(sorted_extents) >= 2 else 0.0
|
||||
planar_ratio = secondary_extent / dominant_extent if dominant_extent > 1e-9 else 0.0
|
||||
max_step = max(step_distances) if step_distances else None
|
||||
median_step = percentile(step_distances, 0.5) if step_distances else None
|
||||
max_step_ratio = max_step / path_length if max_step is not None and path_length > 1e-9 else None
|
||||
closure_ratio = closure_distance / path_length if closure_distance is not None and path_length > 1e-9 else None
|
||||
return {
|
||||
"camera_pose_count": int(path.get("pose_count", len(poses)) or len(poses)),
|
||||
"camera_path_length": round(float(path_length), 6),
|
||||
"camera_bounds_extent": round_list(extents),
|
||||
"camera_dominant_extent": round(float(dominant_extent), 6),
|
||||
"camera_secondary_extent": round(float(secondary_extent), 6),
|
||||
"camera_planar_coverage_ratio": round(float(planar_ratio), 6),
|
||||
"camera_max_step_distance": round(float(max_step), 6) if max_step is not None else None,
|
||||
"camera_median_step_distance": round(float(median_step), 6) if median_step is not None else None,
|
||||
"camera_max_step_ratio": round(float(max_step_ratio), 6) if max_step_ratio is not None else None,
|
||||
"camera_loop_closure_distance": round(float(closure_distance), 6) if closure_distance is not None else None,
|
||||
"camera_loop_closure_ratio": round(float(closure_ratio), 6) if closure_ratio is not None else None,
|
||||
}
|
||||
|
||||
|
||||
def camera_positions(raw_poses: Any) -> list[list[float]]:
|
||||
positions: list[list[float]] = []
|
||||
if not isinstance(raw_poses, list):
|
||||
return positions
|
||||
for pose in raw_poses:
|
||||
if not isinstance(pose, dict):
|
||||
continue
|
||||
position = numeric_list(pose.get("position"), expected=3)
|
||||
if position is not None:
|
||||
positions.append(position)
|
||||
return positions
|
||||
|
||||
|
||||
def numeric_list(value: Any, *, expected: int) -> list[float] | None:
|
||||
if not isinstance(value, list) or len(value) != expected:
|
||||
return None
|
||||
try:
|
||||
values = [float(item) for item in value]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not all(math.isfinite(item) for item in values):
|
||||
return None
|
||||
return values
|
||||
|
||||
|
||||
def axis_extents(positions: list[list[float]]) -> list[float]:
|
||||
return [
|
||||
max(position[axis] for position in positions) - min(position[axis] for position in positions)
|
||||
for axis in range(3)
|
||||
]
|
||||
|
||||
|
||||
def path_length_from_positions(positions: list[list[float]]) -> float:
|
||||
return sum(distance(positions[index - 1], positions[index]) for index in range(1, len(positions)))
|
||||
|
||||
|
||||
def distance(a: list[float], b: list[float]) -> float:
|
||||
return math.sqrt(sum((a[index] - b[index]) ** 2 for index in range(3)))
|
||||
|
||||
|
||||
def percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(float(value) for value in values)
|
||||
index = min(max(int(round((len(ordered) - 1) * fraction)), 0), len(ordered) - 1)
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def round_list(values: list[float]) -> list[float]:
|
||||
return [round(float(value), 6) for value in values]
|
||||
|
||||
|
||||
def gate(passed: bool, value: Any, threshold: int | float) -> dict[str, Any]:
|
||||
return {"passed": bool(passed), "value": value, "threshold": threshold}
|
||||
|
||||
|
||||
def capture_requirements(blockers: list[str], metrics: dict[str, Any]) -> list[str]:
|
||||
requirements = [
|
||||
"Capture one room-sized volume with continuous overlap; do not jump between unrelated viewpoints.",
|
||||
"Move slowly with strong sideways parallax around desks, cabinets, floor, and wall features.",
|
||||
"Keep 30-80 sharp frames after blur filtering; avoid fast pans, glass-only regions, and blank walls.",
|
||||
]
|
||||
if "registered_ratio" in blockers or "enough_registered_images" in blockers:
|
||||
requirements.append(
|
||||
f"Current registration is {metrics['registered_image_count']} images / {metrics['frame_count']} frames; recapture until COLMAP registers at least 70 percent."
|
||||
)
|
||||
if "sparse_points" in blockers:
|
||||
requirements.append(
|
||||
f"Current sparse points are {metrics['sparse_point_count']}; add textured overlap and lateral baseline until sparse points exceed 5000."
|
||||
)
|
||||
if "verified_pairs" in blockers:
|
||||
requirements.append(
|
||||
f"Current verified pairs are {metrics['verified_pair_count']}; increase adjacent overlap and reduce motion blur."
|
||||
)
|
||||
if "camera_path_length" in blockers:
|
||||
requirements.append(
|
||||
f"Current registered camera path length is {metrics['camera_path_length']}; walk a longer continuous path with sideways parallax around the room volume."
|
||||
)
|
||||
if "camera_planar_coverage" in blockers:
|
||||
requirements.append(
|
||||
f"Current camera planar coverage ratio is {metrics['camera_planar_coverage_ratio']}; avoid a single straight-line sweep and capture from at least two lateral directions."
|
||||
)
|
||||
if "camera_step_consistency" in blockers:
|
||||
requirements.append(
|
||||
f"Current camera max-step ratio is {metrics['camera_max_step_ratio']}; remove discontinuous jumps or recapture as one continuous overlapping trajectory."
|
||||
)
|
||||
return requirements
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
metrics = report["metrics"]
|
||||
lines = [
|
||||
"# Real2Sim Capture Acceptance Report",
|
||||
"",
|
||||
f"Decision: `{report['decision']}`",
|
||||
f"Accepted: `{str(report['accepted']).lower()}`",
|
||||
"",
|
||||
"## Metrics",
|
||||
"",
|
||||
"| Metric | Value |",
|
||||
"| --- | ---: |",
|
||||
]
|
||||
for key, value in metrics.items():
|
||||
lines.append(f"| `{key}` | {value} |")
|
||||
lines.extend(["", "## Gates", "", "| Gate | Value | Threshold | Passed |", "| --- | ---: | ---: | --- |"])
|
||||
for name, item in report["gates"].items():
|
||||
lines.append(f"| `{name}` | {item['value']} | {item['threshold']} | `{str(item['passed']).lower()}` |")
|
||||
if report["blockers"]:
|
||||
lines.extend(["", "## Blockers", ""])
|
||||
for blocker in report["blockers"]:
|
||||
lines.append(f"- `{blocker}`")
|
||||
if report["capture_requirements"]:
|
||||
lines.extend(["", "## Capture Requirements", ""])
|
||||
for requirement in report["capture_requirements"]:
|
||||
lines.append(f"- {requirement}")
|
||||
if report["sfm_warnings"]:
|
||||
lines.extend(["", "## SfM Warnings", ""])
|
||||
for warning in report["sfm_warnings"]:
|
||||
lines.append(f"- {warning}")
|
||||
if report["sfm_recommendations"]:
|
||||
lines.extend(["", "## SfM Recommendations", ""])
|
||||
for recommendation in report["sfm_recommendations"]:
|
||||
lines.append(f"- {recommendation}")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
695
vendor/gs_playground/scripts/real2sim_colmap_capture_report.py
vendored
Normal file
695
vendor/gs_playground/scripts/real2sim_colmap_capture_report.py
vendored
Normal file
@ -0,0 +1,695 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a COLMAP SfM preflight report for Real2Sim video keyframes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"}
|
||||
|
||||
|
||||
class ColmapCommandError(RuntimeError):
|
||||
def __init__(self, command: list[str], returncode: int) -> None:
|
||||
super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}")
|
||||
self.command = command
|
||||
self.returncode = returncode
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate a COLMAP capture-quality report from selected frames")
|
||||
parser.add_argument("--frames-dir", type=Path, required=True, help="Directory containing selected image frames")
|
||||
parser.add_argument("--work-dir", type=Path, required=True, help="COLMAP intermediate directory")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output JSON report")
|
||||
parser.add_argument("--colmap-bin", default="colmap")
|
||||
parser.add_argument("--matcher", choices=["exhaustive", "sequential"], default="sequential")
|
||||
parser.add_argument("--sequential-overlap", type=int, default=10)
|
||||
parser.add_argument("--camera-model", default="SIMPLE_PINHOLE")
|
||||
parser.add_argument("--max-image-size", type=int, default=0)
|
||||
parser.add_argument("--sift-max-num-features", type=int, default=8192)
|
||||
parser.add_argument("--sift-peak-threshold", type=float, default=None)
|
||||
parser.add_argument("--feature-threads", type=int, default=1)
|
||||
parser.add_argument("--matcher-threads", type=int, default=1)
|
||||
parser.add_argument("--guided-matching", action="store_true")
|
||||
parser.add_argument("--mapper-random-seed", type=int, default=0)
|
||||
parser.add_argument("--min-registered-ratio", type=float, default=0.70)
|
||||
parser.add_argument("--min-sparse-points", type=int, default=500)
|
||||
parser.add_argument("--require-pass", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_colmap_report(args)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if args.require_pass and not report["passed"]:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def run_colmap_report(args: argparse.Namespace) -> dict[str, Any]:
|
||||
frames = collect_frames(args.frames_dir)
|
||||
if not frames:
|
||||
raise SystemExit(f"frames directory contains no supported images: {args.frames_dir}")
|
||||
colmap = resolve_colmap(args.colmap_bin)
|
||||
|
||||
work = args.work_dir.resolve()
|
||||
if work.exists():
|
||||
shutil.rmtree(work)
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
database = work / "database.db"
|
||||
sparse = work / "sparse"
|
||||
sparse.mkdir(parents=True, exist_ok=True)
|
||||
text_model = work / "sparse_text"
|
||||
|
||||
commands: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
try:
|
||||
feature_command = [
|
||||
colmap,
|
||||
"feature_extractor",
|
||||
"--database_path",
|
||||
database.as_posix(),
|
||||
"--image_path",
|
||||
args.frames_dir.resolve().as_posix(),
|
||||
"--ImageReader.single_camera",
|
||||
"1",
|
||||
"--ImageReader.camera_model",
|
||||
str(args.camera_model).upper(),
|
||||
]
|
||||
feature_gpu_option = colmap_option(colmap, "feature_extractor", ["--FeatureExtraction.use_gpu", "--SiftExtraction.use_gpu"])
|
||||
if feature_gpu_option is not None:
|
||||
feature_command.extend([feature_gpu_option, "0"])
|
||||
feature_threads_option = colmap_option(
|
||||
colmap,
|
||||
"feature_extractor",
|
||||
["--FeatureExtraction.num_threads", "--SiftExtraction.num_threads"],
|
||||
)
|
||||
if feature_threads_option is not None and int(args.feature_threads) > 0:
|
||||
feature_command.extend([feature_threads_option, str(int(args.feature_threads))])
|
||||
if int(args.max_image_size) > 0:
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"feature_extractor",
|
||||
feature_command,
|
||||
["--SiftExtraction.max_image_size"],
|
||||
str(int(args.max_image_size)),
|
||||
)
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"feature_extractor",
|
||||
feature_command,
|
||||
["--SiftExtraction.max_num_features"],
|
||||
str(max(int(args.sift_max_num_features), 1)),
|
||||
)
|
||||
if args.sift_peak_threshold is not None:
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"feature_extractor",
|
||||
feature_command,
|
||||
["--SiftExtraction.peak_threshold"],
|
||||
str(float(args.sift_peak_threshold)),
|
||||
)
|
||||
run_logged(feature_command, commands)
|
||||
matcher_command = build_matcher_command(
|
||||
colmap,
|
||||
args.matcher,
|
||||
database,
|
||||
sequential_overlap=args.sequential_overlap,
|
||||
matcher_threads=int(args.matcher_threads),
|
||||
guided_matching=bool(args.guided_matching),
|
||||
)
|
||||
run_logged(matcher_command, commands)
|
||||
mapper_command = [
|
||||
colmap,
|
||||
"mapper",
|
||||
"--database_path",
|
||||
database.as_posix(),
|
||||
"--image_path",
|
||||
args.frames_dir.resolve().as_posix(),
|
||||
"--output_path",
|
||||
sparse.as_posix(),
|
||||
]
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"mapper",
|
||||
mapper_command,
|
||||
["--Mapper.random_seed"],
|
||||
str(int(args.mapper_random_seed)),
|
||||
)
|
||||
run_logged(mapper_command, commands)
|
||||
except ColmapCommandError as exc:
|
||||
warnings.append(str(exc))
|
||||
|
||||
model_dir, registered_images, sparse_points, model_candidates = select_best_model_dir(
|
||||
colmap,
|
||||
sparse,
|
||||
text_model,
|
||||
commands,
|
||||
)
|
||||
if model_dir is None:
|
||||
warnings.append("COLMAP mapper produced no sparse model")
|
||||
|
||||
registered_ratio = len(registered_images) / max(float(len(frames)), 1.0)
|
||||
passed = registered_ratio >= args.min_registered_ratio and sparse_points >= args.min_sparse_points
|
||||
if registered_ratio < args.min_registered_ratio:
|
||||
warnings.append(
|
||||
f"registered image ratio {registered_ratio:.3f} < {float(args.min_registered_ratio):.3f}"
|
||||
)
|
||||
if sparse_points < args.min_sparse_points:
|
||||
warnings.append(f"sparse point count {sparse_points} < {int(args.min_sparse_points)}")
|
||||
database_stats = read_database_stats(database)
|
||||
geometry = read_best_model_geometry(text_model, work, warnings)
|
||||
|
||||
return {
|
||||
"format": "gs-playground-real2sim-colmap-capture-report-v1",
|
||||
"frames_dir": args.frames_dir.resolve().as_posix(),
|
||||
"frame_count": len(frames),
|
||||
"frame_names": [path.name for path in frames],
|
||||
"colmap_bin": colmap,
|
||||
"matcher": args.matcher,
|
||||
"camera_model": str(args.camera_model).upper(),
|
||||
"max_image_size": int(args.max_image_size),
|
||||
"sift_max_num_features": int(args.sift_max_num_features),
|
||||
"sift_peak_threshold": float(args.sift_peak_threshold) if args.sift_peak_threshold is not None else None,
|
||||
"feature_threads": int(args.feature_threads),
|
||||
"matcher_threads": int(args.matcher_threads),
|
||||
"model_dir": model_dir.as_posix() if model_dir is not None else None,
|
||||
"model_candidates": model_candidates,
|
||||
"registered_image_count": len(registered_images),
|
||||
"registered_ratio": round(float(registered_ratio), 4),
|
||||
"registered_images": registered_images,
|
||||
"sparse_point_count": int(sparse_points),
|
||||
"best_model_text_dir": text_model.as_posix() if model_dir is not None else None,
|
||||
"camera_models": geometry["camera_models"],
|
||||
"camera_poses": geometry["camera_poses"],
|
||||
"camera_path": geometry["camera_path"],
|
||||
"sparse_bounds": geometry["sparse_bounds"],
|
||||
"coverage_preview": geometry["coverage_preview"],
|
||||
"database_stats": database_stats,
|
||||
"gates": {
|
||||
"min_registered_ratio": float(args.min_registered_ratio),
|
||||
"min_sparse_points": int(args.min_sparse_points),
|
||||
},
|
||||
"passed": bool(passed),
|
||||
"warnings": warnings,
|
||||
"recommendations": capture_recommendations(
|
||||
passed=bool(passed),
|
||||
registered_ratio=registered_ratio,
|
||||
sparse_points=sparse_points,
|
||||
database_stats=database_stats,
|
||||
warnings=warnings,
|
||||
),
|
||||
"commands": commands,
|
||||
}
|
||||
|
||||
|
||||
def collect_frames(frames_dir: Path) -> list[Path]:
|
||||
if not frames_dir.is_dir():
|
||||
raise SystemExit(f"frames directory not found: {frames_dir}")
|
||||
return sorted(path for path in frames_dir.iterdir() if is_frame_image(path))
|
||||
|
||||
|
||||
def is_frame_image(path: Path) -> bool:
|
||||
if not path.is_file() or path.suffix.lower() not in IMAGE_SUFFIXES:
|
||||
return False
|
||||
name = path.name.lower()
|
||||
return "contact_sheet" not in name and "preview" not in name
|
||||
|
||||
|
||||
def resolve_colmap(colmap_bin: str) -> str:
|
||||
path = shutil.which(colmap_bin)
|
||||
if path is not None:
|
||||
return path
|
||||
candidate = Path(colmap_bin)
|
||||
if candidate.is_file():
|
||||
return candidate.resolve().as_posix()
|
||||
raise SystemExit(f"COLMAP binary not found: {colmap_bin}")
|
||||
|
||||
|
||||
def run_logged(command: list[str], commands: list[dict[str, Any]]) -> None:
|
||||
completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
commands.append(
|
||||
{
|
||||
"argv": list(command),
|
||||
"returncode": completed.returncode,
|
||||
"output_tail": "\n".join(completed.stdout.splitlines()[-80:]),
|
||||
}
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
sys.stderr.write(completed.stdout)
|
||||
raise ColmapCommandError(command, completed.returncode)
|
||||
|
||||
|
||||
def build_matcher_command(
|
||||
colmap: str,
|
||||
matcher: str,
|
||||
database: Path,
|
||||
*,
|
||||
sequential_overlap: int,
|
||||
matcher_threads: int,
|
||||
guided_matching: bool,
|
||||
) -> list[str]:
|
||||
command_name = "sequential_matcher" if matcher == "sequential" else "exhaustive_matcher"
|
||||
command = [
|
||||
colmap,
|
||||
command_name,
|
||||
"--database_path",
|
||||
database.as_posix(),
|
||||
]
|
||||
matcher_gpu_option = colmap_option(colmap, command_name, ["--FeatureMatching.use_gpu", "--SiftMatching.use_gpu"])
|
||||
if matcher_gpu_option is not None:
|
||||
command.extend([matcher_gpu_option, "0"])
|
||||
matcher_threads_option = colmap_option(
|
||||
colmap,
|
||||
command_name,
|
||||
["--FeatureMatching.num_threads", "--SiftMatching.num_threads"],
|
||||
)
|
||||
if matcher_threads_option is not None and matcher_threads > 0:
|
||||
command.extend([matcher_threads_option, str(matcher_threads)])
|
||||
if guided_matching:
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
command_name,
|
||||
command,
|
||||
["--FeatureMatching.guided_matching", "--SiftMatching.guided_matching"],
|
||||
"1",
|
||||
)
|
||||
append_cpu_bruteforce_option(colmap, command_name, command)
|
||||
if command_name == "sequential_matcher":
|
||||
overlap_option = colmap_option(colmap, command_name, ["--SequentialMatching.overlap"])
|
||||
if overlap_option is not None:
|
||||
command.extend([overlap_option, str(max(int(sequential_overlap), 1))])
|
||||
return command
|
||||
|
||||
|
||||
def colmap_option(colmap: str, command: str, names: list[str]) -> str | None:
|
||||
completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
help_text = completed.stdout
|
||||
for name in names:
|
||||
if name in help_text:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def append_cpu_bruteforce_option(colmap: str, command: str, argv: list[str]) -> None:
|
||||
option = colmap_option(colmap, command, ["--SiftMatching.cpu_brute_force_matcher"])
|
||||
if option is not None:
|
||||
argv.extend([option, "1"])
|
||||
|
||||
|
||||
def append_option_if_supported(
|
||||
colmap: str,
|
||||
command: str,
|
||||
argv: list[str],
|
||||
option_names: list[str],
|
||||
value: str,
|
||||
) -> None:
|
||||
option = colmap_option(colmap, command, option_names)
|
||||
if option is not None:
|
||||
argv.extend([option, value])
|
||||
|
||||
|
||||
def select_best_model_dir(
|
||||
colmap: str,
|
||||
sparse_dir: Path,
|
||||
text_model: Path,
|
||||
commands: list[dict[str, Any]],
|
||||
) -> tuple[Path | None, list[str], int, list[dict[str, Any]]]:
|
||||
candidates = sorted(path for path in sparse_dir.iterdir() if path.is_dir()) if sparse_dir.is_dir() else []
|
||||
if not candidates:
|
||||
return None, [], 0, []
|
||||
|
||||
candidate_text_root = text_model.parent / f"{text_model.name}_candidates"
|
||||
if candidate_text_root.exists():
|
||||
shutil.rmtree(candidate_text_root)
|
||||
candidate_text_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
scored: list[tuple[int, int, Path, Path, list[str]]] = []
|
||||
summaries: list[dict[str, Any]] = []
|
||||
for candidate in candidates:
|
||||
candidate_text = candidate_text_root / candidate.name
|
||||
candidate_text.mkdir(parents=True, exist_ok=True)
|
||||
run_logged(
|
||||
[
|
||||
colmap,
|
||||
"model_converter",
|
||||
"--input_path",
|
||||
candidate.as_posix(),
|
||||
"--output_path",
|
||||
candidate_text.as_posix(),
|
||||
"--output_type",
|
||||
"TXT",
|
||||
],
|
||||
commands,
|
||||
)
|
||||
registered = parse_registered_images(candidate_text / "images.txt")
|
||||
points = count_sparse_points(candidate_text / "points3D.txt")
|
||||
summaries.append(
|
||||
{
|
||||
"model_dir": candidate.as_posix(),
|
||||
"registered_image_count": len(registered),
|
||||
"sparse_point_count": points,
|
||||
}
|
||||
)
|
||||
scored.append((len(registered), points, candidate, candidate_text, registered))
|
||||
|
||||
registered_count, points, model_dir, candidate_text, registered_images = max(
|
||||
scored,
|
||||
key=lambda item: (item[0], item[1]),
|
||||
)
|
||||
if text_model.exists():
|
||||
shutil.rmtree(text_model)
|
||||
shutil.copytree(candidate_text, text_model)
|
||||
return model_dir, registered_images, int(points), summaries
|
||||
|
||||
|
||||
def parse_registered_images(path: Path) -> list[str]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()]
|
||||
data_lines = [line for line in lines if line and not line.startswith("#")]
|
||||
registered: list[str] = []
|
||||
for index in range(0, len(data_lines), 2):
|
||||
parts = data_lines[index].split(maxsplit=9)
|
||||
if len(parts) >= 10:
|
||||
registered.append(parts[9])
|
||||
return registered
|
||||
|
||||
|
||||
def count_sparse_points(path: Path) -> int:
|
||||
if not path.is_file():
|
||||
return 0
|
||||
return sum(
|
||||
1
|
||||
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()
|
||||
if line.strip() and not line.startswith("#")
|
||||
)
|
||||
|
||||
|
||||
def read_best_model_geometry(text_model: Path, work_dir: Path, warnings: list[str]) -> dict[str, Any]:
|
||||
camera_models = parse_camera_models(text_model / "cameras.txt")
|
||||
camera_poses = parse_camera_poses(text_model / "images.txt")
|
||||
sparse_points, sparse_bounds = parse_sparse_points(text_model / "points3D.txt")
|
||||
camera_path = summarize_camera_path(camera_poses)
|
||||
coverage_preview = None
|
||||
if camera_poses and sparse_points:
|
||||
try:
|
||||
coverage_preview = write_coverage_preview(
|
||||
camera_poses,
|
||||
sparse_points,
|
||||
work_dir / "sfm_capture_coverage.png",
|
||||
).as_posix()
|
||||
except Exception as exc:
|
||||
warnings.append(f"failed to write COLMAP coverage preview: {exc}")
|
||||
return {
|
||||
"camera_models": camera_models,
|
||||
"camera_poses": camera_poses,
|
||||
"camera_path": camera_path,
|
||||
"sparse_bounds": sparse_bounds,
|
||||
"coverage_preview": coverage_preview,
|
||||
}
|
||||
|
||||
|
||||
def parse_camera_models(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
cameras: list[dict[str, Any]] = []
|
||||
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
cameras.append(
|
||||
{
|
||||
"camera_id": int(parts[0]),
|
||||
"model": parts[1],
|
||||
"width": int(parts[2]),
|
||||
"height": int(parts[3]),
|
||||
"params": [float(value) for value in parts[4:]],
|
||||
}
|
||||
)
|
||||
return cameras
|
||||
|
||||
|
||||
def parse_camera_poses(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()]
|
||||
data_lines = [line for line in lines if line and not line.startswith("#")]
|
||||
poses: list[dict[str, Any]] = []
|
||||
for index in range(0, len(data_lines), 2):
|
||||
parts = data_lines[index].split(maxsplit=9)
|
||||
if len(parts) < 10:
|
||||
continue
|
||||
qvec = [float(value) for value in parts[1:5]]
|
||||
tvec = [float(value) for value in parts[5:8]]
|
||||
position = camera_center_from_qt(qvec, tvec)
|
||||
poses.append(
|
||||
{
|
||||
"image_id": int(parts[0]),
|
||||
"image_name": parts[9],
|
||||
"camera_id": int(parts[8]),
|
||||
"qvec_wxyz": round_list(qvec),
|
||||
"tvec": round_list(tvec),
|
||||
"position": round_list(position),
|
||||
}
|
||||
)
|
||||
return sorted(poses, key=lambda item: item["image_name"])
|
||||
|
||||
|
||||
def camera_center_from_qt(qvec: list[float], tvec: list[float]) -> list[float]:
|
||||
rotation = qvec_to_rotmat(qvec)
|
||||
return [
|
||||
-sum(rotation[row][col] * tvec[row] for row in range(3))
|
||||
for col in range(3)
|
||||
]
|
||||
|
||||
|
||||
def qvec_to_rotmat(qvec: list[float]) -> list[list[float]]:
|
||||
w, x, y, z = qvec
|
||||
return [
|
||||
[1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w],
|
||||
[2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w],
|
||||
[2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y],
|
||||
]
|
||||
|
||||
|
||||
def parse_sparse_points(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
if not path.is_file():
|
||||
return [], {}
|
||||
points: list[dict[str, Any]] = []
|
||||
mins = [math.inf, math.inf, math.inf]
|
||||
maxs = [-math.inf, -math.inf, -math.inf]
|
||||
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
xyz = [float(parts[1]), float(parts[2]), float(parts[3])]
|
||||
color = [int(float(parts[4])), int(float(parts[5])), int(float(parts[6]))]
|
||||
for axis in range(3):
|
||||
mins[axis] = min(mins[axis], xyz[axis])
|
||||
maxs[axis] = max(maxs[axis], xyz[axis])
|
||||
if len(points) < 20000:
|
||||
points.append({"xyz": xyz, "rgb": color})
|
||||
if any(not math.isfinite(value) for value in mins + maxs):
|
||||
return points, {}
|
||||
center = [(mins[axis] + maxs[axis]) / 2.0 for axis in range(3)]
|
||||
extent = [maxs[axis] - mins[axis] for axis in range(3)]
|
||||
robust_mins = [percentile([point["xyz"][axis] for point in points], 0.02) for axis in range(3)]
|
||||
robust_maxs = [percentile([point["xyz"][axis] for point in points], 0.98) for axis in range(3)]
|
||||
return points, {
|
||||
"bounds_min": round_list(mins),
|
||||
"bounds_max": round_list(maxs),
|
||||
"bounds_center": round_list(center),
|
||||
"bounds_extent": round_list(extent),
|
||||
"robust_bounds_min": round_list(robust_mins),
|
||||
"robust_bounds_max": round_list(robust_maxs),
|
||||
"robust_bounds_extent": round_list([robust_maxs[axis] - robust_mins[axis] for axis in range(3)]),
|
||||
"preview_point_count": len(points),
|
||||
}
|
||||
|
||||
|
||||
def summarize_camera_path(camera_poses: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
if not camera_poses:
|
||||
return {}
|
||||
positions = [pose["position"] for pose in camera_poses]
|
||||
mins = [min(position[axis] for position in positions) for axis in range(3)]
|
||||
maxs = [max(position[axis] for position in positions) for axis in range(3)]
|
||||
path_length = sum(
|
||||
distance(positions[index - 1], positions[index])
|
||||
for index in range(1, len(positions))
|
||||
)
|
||||
return {
|
||||
"pose_count": len(camera_poses),
|
||||
"path_length": round(float(path_length), 6),
|
||||
"start_image": camera_poses[0]["image_name"],
|
||||
"end_image": camera_poses[-1]["image_name"],
|
||||
"bounds_min": round_list(mins),
|
||||
"bounds_max": round_list(maxs),
|
||||
"bounds_extent": round_list([maxs[axis] - mins[axis] for axis in range(3)]),
|
||||
}
|
||||
|
||||
|
||||
def write_coverage_preview(camera_poses: list[dict[str, Any]], sparse_points: list[dict[str, Any]], output: Path) -> Path:
|
||||
all_xyz = [point["xyz"] for point in sparse_points] + [pose["position"] for pose in camera_poses]
|
||||
extents = [
|
||||
max(point[axis] for point in all_xyz) - min(point[axis] for point in all_xyz)
|
||||
for axis in range(3)
|
||||
]
|
||||
axes = sorted(range(3), key=lambda axis: extents[axis], reverse=True)[:2]
|
||||
axis_x, axis_y = axes[0], axes[1]
|
||||
xs = [point[axis_x] for point in all_xyz]
|
||||
ys = [point[axis_y] for point in all_xyz]
|
||||
min_x, max_x = robust_axis_bounds(xs)
|
||||
min_y, max_y = robust_axis_bounds(ys)
|
||||
pad_x = max((max_x - min_x) * 0.08, 1e-6)
|
||||
pad_y = max((max_y - min_y) * 0.08, 1e-6)
|
||||
min_x -= pad_x
|
||||
max_x += pad_x
|
||||
min_y -= pad_y
|
||||
max_y += pad_y
|
||||
|
||||
width, height = 1200, 820
|
||||
margin = 70
|
||||
image = Image.new("RGB", (width, height), "white")
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.rectangle((margin, margin, width - margin, height - margin), outline=(210, 210, 210))
|
||||
|
||||
def project(point: list[float]) -> tuple[int, int]:
|
||||
px = margin + (point[axis_x] - min_x) / max(max_x - min_x, 1e-9) * (width - 2 * margin)
|
||||
py = height - margin - (point[axis_y] - min_y) / max(max_y - min_y, 1e-9) * (height - 2 * margin)
|
||||
return clamp_int(px, 0, width - 1), clamp_int(py, 0, height - 1)
|
||||
|
||||
for point in sparse_points:
|
||||
x, y = project(point["xyz"])
|
||||
rgb = point.get("rgb", [150, 150, 150])
|
||||
color = tuple(max(45, min(210, int(value))) for value in rgb)
|
||||
draw.point((x, y), fill=color)
|
||||
|
||||
camera_xy = [project(pose["position"]) for pose in camera_poses]
|
||||
if len(camera_xy) >= 2:
|
||||
draw.line(camera_xy, fill=(28, 98, 201), width=3)
|
||||
for idx, (x, y) in enumerate(camera_xy):
|
||||
radius = 5 if idx not in {0, len(camera_xy) - 1} else 8
|
||||
color = (220, 48, 48) if idx == 0 else (34, 148, 83) if idx == len(camera_xy) - 1 else (28, 98, 201)
|
||||
draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=color, outline=(255, 255, 255))
|
||||
|
||||
draw.text((margin, 22), "COLMAP capture coverage: sparse points + registered camera path", fill=(20, 20, 20))
|
||||
draw.text((margin, height - 44), f"Axes: {axis_name(axis_x)} / {axis_name(axis_y)} Cameras: {len(camera_poses)} Preview points: {len(sparse_points)}", fill=(70, 70, 70))
|
||||
if camera_xy:
|
||||
sx, sy = camera_xy[0]
|
||||
ex, ey = camera_xy[-1]
|
||||
draw.text((sx + 10, sy + 10), "start", fill=(140, 30, 30))
|
||||
draw.text((ex + 10, ey + 10), "end", fill=(20, 105, 55))
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.save(output)
|
||||
return output
|
||||
|
||||
|
||||
def axis_name(axis: int) -> str:
|
||||
return ("x", "y", "z")[axis]
|
||||
|
||||
|
||||
def robust_axis_bounds(values: list[float]) -> tuple[float, float]:
|
||||
if not values:
|
||||
return 0.0, 1.0
|
||||
low = percentile(values, 0.02)
|
||||
high = percentile(values, 0.98)
|
||||
if high <= low:
|
||||
return min(values), max(values)
|
||||
return low, high
|
||||
|
||||
|
||||
def percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(float(value) for value in values)
|
||||
index = min(max(int(round((len(ordered) - 1) * fraction)), 0), len(ordered) - 1)
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def clamp_int(value: float, low: int, high: int) -> int:
|
||||
return max(low, min(high, int(round(value))))
|
||||
|
||||
|
||||
def distance(a: list[float], b: list[float]) -> float:
|
||||
return math.sqrt(sum((a[axis] - b[axis]) ** 2 for axis in range(3)))
|
||||
|
||||
|
||||
def round_list(values: list[float], digits: int = 6) -> list[float]:
|
||||
return [round(float(value), digits) for value in values]
|
||||
|
||||
|
||||
def read_database_stats(database: Path) -> dict[str, Any]:
|
||||
if not database.is_file():
|
||||
return {}
|
||||
try:
|
||||
connection = sqlite3.connect(database.as_posix())
|
||||
try:
|
||||
image_count = table_count(connection, "images")
|
||||
keypoint_rows = connection.execute("SELECT rows FROM keypoints").fetchall()
|
||||
match_rows = connection.execute("SELECT rows FROM matches").fetchall()
|
||||
geometry_rows = connection.execute("SELECT rows FROM two_view_geometries").fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
except sqlite3.Error:
|
||||
return {}
|
||||
|
||||
keypoint_counts = [int(row[0]) for row in keypoint_rows]
|
||||
match_counts = [int(row[0]) for row in match_rows if int(row[0]) > 0]
|
||||
geometry_counts = [int(row[0]) for row in geometry_rows if int(row[0]) > 0]
|
||||
return {
|
||||
"image_count": image_count,
|
||||
"keypoint_total": int(sum(keypoint_counts)),
|
||||
"keypoint_mean": round(float(sum(keypoint_counts) / max(len(keypoint_counts), 1)), 2),
|
||||
"matched_pair_count": len(match_counts),
|
||||
"matched_pair_mean": round(float(sum(match_counts) / max(len(match_counts), 1)), 2) if match_counts else 0.0,
|
||||
"verified_pair_count": len(geometry_counts),
|
||||
"verified_pair_mean": round(float(sum(geometry_counts) / max(len(geometry_counts), 1)), 2) if geometry_counts else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def table_count(connection: sqlite3.Connection, table: str) -> int:
|
||||
return int(connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
|
||||
|
||||
|
||||
def capture_recommendations(
|
||||
*,
|
||||
passed: bool,
|
||||
registered_ratio: float,
|
||||
sparse_points: int,
|
||||
database_stats: dict[str, Any],
|
||||
warnings: list[str],
|
||||
) -> list[str]:
|
||||
if passed:
|
||||
return ["COLMAP registered enough frames for a pose-initialized reconstruction path."]
|
||||
recommendations = []
|
||||
warning_text = "\n".join(warnings).lower()
|
||||
verified_pairs = int(database_stats.get("verified_pair_count", 0) or 0)
|
||||
if "no sparse model" in warning_text or "mapper" in warning_text:
|
||||
recommendations.append("Capture slower video with stronger sideways parallax; avoid pure forward motion or mostly rotational sweeps.")
|
||||
if verified_pairs < 3:
|
||||
recommendations.append("Increase overlap between adjacent views and keep textured surfaces in frame so COLMAP can verify image pairs.")
|
||||
if registered_ratio < 0.7:
|
||||
recommendations.append("Collect 30-80 sharp frames around the same room volume before attempting commercial-grade reconstruction.")
|
||||
if sparse_points < 500:
|
||||
recommendations.append("Avoid motion blur, reflective glass, blank walls, and very close foreground occluders during capture.")
|
||||
return recommendations
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
246
vendor/gs_playground/scripts/real2sim_colmap_delaunay_mesh.py
vendored
Normal file
246
vendor/gs_playground/scripts/real2sim_colmap_delaunay_mesh.py
vendored
Normal file
@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run COLMAP Delaunay meshing from an existing dense workspace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ColmapCommandError(RuntimeError):
|
||||
def __init__(self, command: list[str], returncode: int) -> None:
|
||||
super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}")
|
||||
self.command = command
|
||||
self.returncode = returncode
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run COLMAP Delaunay meshing from an existing dense workspace")
|
||||
parser.add_argument("--dense-workspace", type=Path, required=True, help="COLMAP dense workspace containing images/stereo/sparse")
|
||||
parser.add_argument("--output-ply", type=Path, required=True, help="Output Delaunay mesh PLY")
|
||||
parser.add_argument("--report", type=Path, required=True, help="Output Delaunay report JSON")
|
||||
parser.add_argument("--colmap-bin", default="colmap")
|
||||
parser.add_argument("--fused-ply", type=Path, help="Existing fused point cloud; defaults to dense-workspace/fused.ply")
|
||||
parser.add_argument("--ensure-vis", action="store_true", help="Run stereo_fusion into the workspace when fused.ply.vis is missing")
|
||||
parser.add_argument("--geom-consistency", action="store_true", default=True)
|
||||
parser.add_argument("--no-geom-consistency", action="store_false", dest="geom_consistency")
|
||||
parser.add_argument("--stereo-fusion-min-num-pixels", type=int, default=5)
|
||||
parser.add_argument("--stereo-fusion-max-reproj-error", type=float, default=2.0)
|
||||
parser.add_argument("--stereo-fusion-max-depth-error", type=float, default=0.01)
|
||||
parser.add_argument("--max-proj-dist", type=float, default=12.0)
|
||||
parser.add_argument("--max-depth-dist", type=float, default=0.035)
|
||||
parser.add_argument("--visibility-sigma", type=float, default=3.0)
|
||||
parser.add_argument("--quality-regularization", type=float, default=1.0)
|
||||
parser.add_argument("--max-side-length-factor", type=float, default=20.0)
|
||||
parser.add_argument("--max-side-length-percentile", type=float, default=95.0)
|
||||
parser.add_argument("--min-mesh-faces", type=int, default=100000)
|
||||
parser.add_argument("--require-pass", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_delaunay_meshing(args)
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if args.require_pass and not report.get("passed", False):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def run_delaunay_meshing(args: argparse.Namespace) -> dict[str, Any]:
|
||||
colmap = resolve_colmap(args.colmap_bin)
|
||||
dense_workspace = args.dense_workspace.resolve()
|
||||
output_ply = args.output_ply.resolve()
|
||||
fused_ply = (args.fused_ply or dense_workspace / "fused.ply").resolve()
|
||||
workspace_fused = dense_workspace / "fused.ply"
|
||||
workspace_vis = dense_workspace / "fused.ply.vis"
|
||||
commands: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
if not dense_workspace.is_dir():
|
||||
warnings.append(f"dense workspace is missing: {dense_workspace}")
|
||||
else:
|
||||
output_ply.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
prepare_workspace_fused(
|
||||
colmap=colmap,
|
||||
dense_workspace=dense_workspace,
|
||||
fused_ply=fused_ply,
|
||||
workspace_fused=workspace_fused,
|
||||
workspace_vis=workspace_vis,
|
||||
args=args,
|
||||
commands=commands,
|
||||
)
|
||||
run_logged(build_delaunay_command(colmap, dense_workspace, output_ply, args), commands)
|
||||
except ColmapCommandError as exc:
|
||||
warnings.append(str(exc))
|
||||
|
||||
mesh_counts = ply_counts(output_ply) if output_ply.is_file() else {}
|
||||
mesh_face_count = int(mesh_counts.get("face_count") or 0)
|
||||
passed = mesh_face_count >= int(args.min_mesh_faces)
|
||||
if mesh_face_count and not passed:
|
||||
warnings.append(f"Delaunay mesh face count {mesh_face_count} < {int(args.min_mesh_faces)}")
|
||||
if not mesh_face_count and not any("COLMAP command failed" in warning for warning in warnings):
|
||||
warnings.append("COLMAP Delaunay meshing produced no mesh")
|
||||
|
||||
return {
|
||||
"format": "gs-playground-real2sim-colmap-delaunay-report-v1",
|
||||
"dense_workspace": dense_workspace.as_posix(),
|
||||
"fused_ply": workspace_fused.as_posix() if workspace_fused.is_file() else None,
|
||||
"fused_visibility": workspace_vis.as_posix() if workspace_vis.is_file() else None,
|
||||
"output_ply": output_ply.as_posix() if output_ply.is_file() else None,
|
||||
"mesh_vertex_count": int(mesh_counts.get("vertex_count") or 0),
|
||||
"mesh_face_count": mesh_face_count,
|
||||
"min_mesh_faces": int(args.min_mesh_faces),
|
||||
"ensure_vis": bool(args.ensure_vis),
|
||||
"geom_consistency": bool(args.geom_consistency),
|
||||
"delaunay_options": {
|
||||
"max_proj_dist": float(args.max_proj_dist),
|
||||
"max_depth_dist": float(args.max_depth_dist),
|
||||
"visibility_sigma": float(args.visibility_sigma),
|
||||
"quality_regularization": float(args.quality_regularization),
|
||||
"max_side_length_factor": float(args.max_side_length_factor),
|
||||
"max_side_length_percentile": float(args.max_side_length_percentile),
|
||||
},
|
||||
"passed": bool(passed),
|
||||
"warnings": warnings,
|
||||
"commands": commands,
|
||||
}
|
||||
|
||||
|
||||
def prepare_workspace_fused(
|
||||
*,
|
||||
colmap: str,
|
||||
dense_workspace: Path,
|
||||
fused_ply: Path,
|
||||
workspace_fused: Path,
|
||||
workspace_vis: Path,
|
||||
args: argparse.Namespace,
|
||||
commands: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if workspace_fused.resolve() != fused_ply.resolve() and fused_ply.is_file():
|
||||
workspace_fused.unlink(missing_ok=True)
|
||||
try:
|
||||
workspace_fused.symlink_to(fused_ply)
|
||||
except OSError:
|
||||
shutil.copy2(fused_ply, workspace_fused)
|
||||
if workspace_fused.is_file() and workspace_vis.is_file():
|
||||
return
|
||||
if not args.ensure_vis:
|
||||
raise ColmapCommandError(["missing", workspace_vis.as_posix()], 2)
|
||||
workspace_fused.unlink(missing_ok=True)
|
||||
run_logged(build_stereo_fusion_command(colmap, dense_workspace, workspace_fused, args), commands)
|
||||
|
||||
|
||||
def build_stereo_fusion_command(
|
||||
colmap: str,
|
||||
dense_workspace: Path,
|
||||
workspace_fused: Path,
|
||||
args: argparse.Namespace,
|
||||
) -> list[str]:
|
||||
command = [
|
||||
colmap,
|
||||
"stereo_fusion",
|
||||
"--workspace_path",
|
||||
dense_workspace.as_posix(),
|
||||
"--workspace_format",
|
||||
"COLMAP",
|
||||
"--input_type",
|
||||
"geometric" if args.geom_consistency else "photometric",
|
||||
"--output_type",
|
||||
"PLY",
|
||||
"--output_path",
|
||||
workspace_fused.as_posix(),
|
||||
]
|
||||
append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.min_num_pixels"], str(max(int(args.stereo_fusion_min_num_pixels), 1)))
|
||||
append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.max_reproj_error"], str(max(float(args.stereo_fusion_max_reproj_error), 0.1)))
|
||||
append_option_if_supported(colmap, "stereo_fusion", command, ["--StereoFusion.max_depth_error"], str(max(float(args.stereo_fusion_max_depth_error), 0.0001)))
|
||||
return command
|
||||
|
||||
|
||||
def build_delaunay_command(colmap: str, dense_workspace: Path, output_ply: Path, args: argparse.Namespace) -> list[str]:
|
||||
command = [
|
||||
colmap,
|
||||
"delaunay_mesher",
|
||||
"--input_path",
|
||||
dense_workspace.as_posix(),
|
||||
"--input_type",
|
||||
"dense",
|
||||
"--output_path",
|
||||
output_ply.as_posix(),
|
||||
]
|
||||
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_proj_dist"], str(max(float(args.max_proj_dist), 0.1)))
|
||||
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_depth_dist"], str(max(float(args.max_depth_dist), 0.0001)))
|
||||
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.visibility_sigma"], str(max(float(args.visibility_sigma), 0.1)))
|
||||
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.quality_regularization"], str(max(float(args.quality_regularization), 0.0)))
|
||||
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_side_length_factor"], str(max(float(args.max_side_length_factor), 1.0)))
|
||||
append_option_if_supported(colmap, "delaunay_mesher", command, ["--DelaunayMeshing.max_side_length_percentile"], str(min(max(float(args.max_side_length_percentile), 1.0), 100.0)))
|
||||
return command
|
||||
|
||||
|
||||
def run_logged(command: list[str], commands: list[dict[str, Any]], *, required: bool = True) -> None:
|
||||
completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
commands.append(
|
||||
{
|
||||
"argv": list(command),
|
||||
"returncode": completed.returncode,
|
||||
"output_tail": "\n".join(completed.stdout.splitlines()[-100:]),
|
||||
}
|
||||
)
|
||||
if completed.returncode != 0 and required:
|
||||
sys.stderr.write(completed.stdout)
|
||||
raise ColmapCommandError(command, completed.returncode)
|
||||
|
||||
|
||||
def resolve_colmap(colmap_bin: str) -> str:
|
||||
path = shutil.which(colmap_bin)
|
||||
if path is not None:
|
||||
return path
|
||||
candidate = Path(colmap_bin)
|
||||
if candidate.is_file():
|
||||
return candidate.resolve().as_posix()
|
||||
raise SystemExit(f"COLMAP binary not found: {colmap_bin}")
|
||||
|
||||
|
||||
def append_option_if_supported(
|
||||
colmap: str,
|
||||
command: str,
|
||||
argv: list[str],
|
||||
option_names: list[str],
|
||||
value: str,
|
||||
) -> None:
|
||||
option = colmap_option(colmap, command, option_names)
|
||||
if option is not None:
|
||||
argv.extend([option, value])
|
||||
|
||||
|
||||
def colmap_option(colmap: str, command: str, names: list[str]) -> str | None:
|
||||
completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
help_text = completed.stdout
|
||||
for name in names:
|
||||
if name in help_text:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def ply_counts(path: Path) -> dict[str, int]:
|
||||
counts = {"vertex_count": 0, "face_count": 0}
|
||||
if not path.is_file():
|
||||
return counts
|
||||
with path.open("rb") as handle:
|
||||
for raw in handle:
|
||||
line = raw.decode("ascii", errors="ignore").strip()
|
||||
if line.startswith("element vertex "):
|
||||
counts["vertex_count"] = int(line.split()[-1])
|
||||
elif line.startswith("element face "):
|
||||
counts["face_count"] = int(line.split()[-1])
|
||||
elif line == "end_header":
|
||||
break
|
||||
return counts
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
317
vendor/gs_playground/scripts/real2sim_colmap_dense_reconstruct.py
vendored
Normal file
317
vendor/gs_playground/scripts/real2sim_colmap_dense_reconstruct.py
vendored
Normal file
@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run COLMAP dense MVS from a passing Real2Sim SfM report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ColmapCommandError(RuntimeError):
|
||||
def __init__(self, command: list[str], returncode: int) -> None:
|
||||
super().__init__(f"COLMAP command failed with exit code {returncode}: {' '.join(command[:3])}")
|
||||
self.command = command
|
||||
self.returncode = returncode
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run COLMAP dense MVS from a Real2Sim SfM report")
|
||||
parser.add_argument("--sfm-report", type=Path, required=True, help="SfM report produced by real2sim_colmap_capture_report.py")
|
||||
parser.add_argument("--output-dir", type=Path, required=True, help="Dense artifact output directory")
|
||||
parser.add_argument("--report", type=Path, required=True, help="Output dense reconstruction report JSON")
|
||||
parser.add_argument("--colmap-bin", default="colmap")
|
||||
parser.add_argument("--max-image-size", type=int, default=960)
|
||||
parser.add_argument("--min-fused-points", type=int, default=100000)
|
||||
parser.add_argument("--geom-consistency", action="store_true", default=True)
|
||||
parser.add_argument("--no-geom-consistency", action="store_false", dest="geom_consistency")
|
||||
parser.add_argument("--gpu-index", default="")
|
||||
parser.add_argument("--patch-match-window-radius", type=int, default=5)
|
||||
parser.add_argument("--patch-match-window-step", type=int, default=1)
|
||||
parser.add_argument("--patch-match-num-samples", type=int, default=15)
|
||||
parser.add_argument("--stereo-fusion-min-num-pixels", type=int, default=5)
|
||||
parser.add_argument("--stereo-fusion-max-reproj-error", type=float, default=2.0)
|
||||
parser.add_argument("--stereo-fusion-max-depth-error", type=float, default=0.01)
|
||||
parser.add_argument("--run-poisson", action="store_true")
|
||||
parser.add_argument("--poisson-depth", type=int, default=10)
|
||||
parser.add_argument("--require-pass", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_dense_reconstruction(args)
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if args.require_pass and not report.get("passed", False):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def run_dense_reconstruction(args: argparse.Namespace) -> dict[str, Any]:
|
||||
sfm = json.loads(args.sfm_report.read_text(encoding="utf-8"))
|
||||
colmap = resolve_colmap(args.colmap_bin)
|
||||
output_dir = args.output_dir.resolve()
|
||||
dense_workspace = output_dir / "workspace"
|
||||
fused_ply = output_dir / "fused.ply"
|
||||
mesh_ply = output_dir / "mesh_poisson.ply"
|
||||
commands: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
if output_dir.exists():
|
||||
shutil.rmtree(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
frames_dir = Path(str(sfm.get("frames_dir", ""))).resolve()
|
||||
model_dir = Path(str(sfm.get("model_dir", ""))).resolve()
|
||||
if not sfm.get("passed", False):
|
||||
warnings.append("source SfM report did not pass capture gates")
|
||||
if not frames_dir.is_dir():
|
||||
warnings.append(f"SfM frames_dir is missing: {frames_dir}")
|
||||
if not model_dir.is_dir():
|
||||
warnings.append(f"SfM model_dir is missing: {model_dir}")
|
||||
|
||||
if not warnings:
|
||||
try:
|
||||
run_logged(build_image_undistorter_command(colmap, frames_dir, model_dir, dense_workspace, args), commands)
|
||||
run_logged(build_patch_match_command(colmap, dense_workspace, args), commands)
|
||||
run_logged(build_stereo_fusion_command(colmap, dense_workspace, fused_ply, args), commands)
|
||||
if args.run_poisson and fused_ply.is_file():
|
||||
run_logged(build_poisson_command(colmap, fused_ply, mesh_ply, args), commands, required=False)
|
||||
except ColmapCommandError as exc:
|
||||
warnings.append(str(exc))
|
||||
|
||||
fused_counts = ply_counts(fused_ply) if fused_ply.is_file() else {}
|
||||
mesh_counts = ply_counts(mesh_ply) if mesh_ply.is_file() else {}
|
||||
fused_point_count = int(fused_counts.get("vertex_count") or 0)
|
||||
passed = fused_point_count >= int(args.min_fused_points)
|
||||
if fused_point_count and not passed:
|
||||
warnings.append(f"fused point count {fused_point_count} < {int(args.min_fused_points)}")
|
||||
if not fused_point_count and not any("COLMAP command failed" in warning for warning in warnings):
|
||||
warnings.append("COLMAP dense reconstruction produced no fused point cloud")
|
||||
|
||||
return {
|
||||
"format": "gs-playground-real2sim-colmap-dense-report-v1",
|
||||
"sfm_report": args.sfm_report.resolve().as_posix(),
|
||||
"frames_dir": frames_dir.as_posix(),
|
||||
"model_dir": model_dir.as_posix(),
|
||||
"output_dir": output_dir.as_posix(),
|
||||
"dense_workspace": dense_workspace.as_posix(),
|
||||
"fused_ply": fused_ply.as_posix() if fused_ply.is_file() else None,
|
||||
"mesh_ply": mesh_ply.as_posix() if mesh_ply.is_file() else None,
|
||||
"max_image_size": int(args.max_image_size),
|
||||
"geom_consistency": bool(args.geom_consistency),
|
||||
"gpu_index": str(args.gpu_index),
|
||||
"fused_point_count": fused_point_count,
|
||||
"mesh_vertex_count": int(mesh_counts.get("vertex_count") or 0),
|
||||
"mesh_face_count": int(mesh_counts.get("face_count") or 0),
|
||||
"min_fused_points": int(args.min_fused_points),
|
||||
"passed": bool(passed),
|
||||
"warnings": warnings,
|
||||
"commands": commands,
|
||||
}
|
||||
|
||||
|
||||
def build_image_undistorter_command(
|
||||
colmap: str,
|
||||
frames_dir: Path,
|
||||
model_dir: Path,
|
||||
dense_workspace: Path,
|
||||
args: argparse.Namespace,
|
||||
) -> list[str]:
|
||||
command = [
|
||||
colmap,
|
||||
"image_undistorter",
|
||||
"--image_path",
|
||||
frames_dir.as_posix(),
|
||||
"--input_path",
|
||||
model_dir.as_posix(),
|
||||
"--output_path",
|
||||
dense_workspace.as_posix(),
|
||||
"--output_type",
|
||||
"COLMAP",
|
||||
]
|
||||
if int(args.max_image_size) > 0:
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"image_undistorter",
|
||||
command,
|
||||
["--max_image_size", "--ImageUndistorter.max_image_size"],
|
||||
str(int(args.max_image_size)),
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
def build_patch_match_command(colmap: str, dense_workspace: Path, args: argparse.Namespace) -> list[str]:
|
||||
command = [
|
||||
colmap,
|
||||
"patch_match_stereo",
|
||||
"--workspace_path",
|
||||
dense_workspace.as_posix(),
|
||||
"--workspace_format",
|
||||
"COLMAP",
|
||||
]
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"patch_match_stereo",
|
||||
command,
|
||||
["--PatchMatchStereo.geom_consistency"],
|
||||
"true" if args.geom_consistency else "false",
|
||||
)
|
||||
if str(args.gpu_index).strip():
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"patch_match_stereo",
|
||||
command,
|
||||
["--PatchMatchStereo.gpu_index"],
|
||||
str(args.gpu_index).strip(),
|
||||
)
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"patch_match_stereo",
|
||||
command,
|
||||
["--PatchMatchStereo.window_radius"],
|
||||
str(max(int(args.patch_match_window_radius), 1)),
|
||||
)
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"patch_match_stereo",
|
||||
command,
|
||||
["--PatchMatchStereo.window_step"],
|
||||
str(max(int(args.patch_match_window_step), 1)),
|
||||
)
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"patch_match_stereo",
|
||||
command,
|
||||
["--PatchMatchStereo.num_samples"],
|
||||
str(max(int(args.patch_match_num_samples), 1)),
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
def build_stereo_fusion_command(
|
||||
colmap: str,
|
||||
dense_workspace: Path,
|
||||
fused_ply: Path,
|
||||
args: argparse.Namespace,
|
||||
) -> list[str]:
|
||||
command = [
|
||||
colmap,
|
||||
"stereo_fusion",
|
||||
"--workspace_path",
|
||||
dense_workspace.as_posix(),
|
||||
"--workspace_format",
|
||||
"COLMAP",
|
||||
"--input_type",
|
||||
"geometric" if args.geom_consistency else "photometric",
|
||||
"--output_path",
|
||||
fused_ply.as_posix(),
|
||||
]
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"stereo_fusion",
|
||||
command,
|
||||
["--StereoFusion.min_num_pixels"],
|
||||
str(max(int(args.stereo_fusion_min_num_pixels), 1)),
|
||||
)
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"stereo_fusion",
|
||||
command,
|
||||
["--StereoFusion.max_reproj_error"],
|
||||
str(max(float(args.stereo_fusion_max_reproj_error), 0.1)),
|
||||
)
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"stereo_fusion",
|
||||
command,
|
||||
["--StereoFusion.max_depth_error"],
|
||||
str(max(float(args.stereo_fusion_max_depth_error), 0.0001)),
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
def build_poisson_command(colmap: str, fused_ply: Path, mesh_ply: Path, args: argparse.Namespace) -> list[str]:
|
||||
command = [
|
||||
colmap,
|
||||
"poisson_mesher",
|
||||
"--input_path",
|
||||
fused_ply.as_posix(),
|
||||
"--output_path",
|
||||
mesh_ply.as_posix(),
|
||||
]
|
||||
append_option_if_supported(
|
||||
colmap,
|
||||
"poisson_mesher",
|
||||
command,
|
||||
["--PoissonMeshing.depth"],
|
||||
str(max(int(args.poisson_depth), 5)),
|
||||
)
|
||||
return command
|
||||
|
||||
|
||||
def run_logged(command: list[str], commands: list[dict[str, Any]], *, required: bool = True) -> None:
|
||||
completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
commands.append(
|
||||
{
|
||||
"argv": list(command),
|
||||
"returncode": completed.returncode,
|
||||
"output_tail": "\n".join(completed.stdout.splitlines()[-100:]),
|
||||
}
|
||||
)
|
||||
if completed.returncode != 0 and required:
|
||||
sys.stderr.write(completed.stdout)
|
||||
raise ColmapCommandError(command, completed.returncode)
|
||||
|
||||
|
||||
def resolve_colmap(colmap_bin: str) -> str:
|
||||
path = shutil.which(colmap_bin)
|
||||
if path is not None:
|
||||
return path
|
||||
candidate = Path(colmap_bin)
|
||||
if candidate.is_file():
|
||||
return candidate.resolve().as_posix()
|
||||
raise SystemExit(f"COLMAP binary not found: {colmap_bin}")
|
||||
|
||||
|
||||
def colmap_option(colmap: str, command: str, names: list[str]) -> str | None:
|
||||
completed = subprocess.run([colmap, command, "-h"], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
|
||||
help_text = completed.stdout
|
||||
for name in names:
|
||||
if name in help_text:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def append_option_if_supported(
|
||||
colmap: str,
|
||||
command: str,
|
||||
argv: list[str],
|
||||
option_names: list[str],
|
||||
value: str,
|
||||
) -> None:
|
||||
option = colmap_option(colmap, command, option_names)
|
||||
if option is not None:
|
||||
argv.extend([option, value])
|
||||
|
||||
|
||||
def ply_counts(path: Path) -> dict[str, int]:
|
||||
counts = {"vertex_count": 0, "face_count": 0}
|
||||
if not path.is_file():
|
||||
return counts
|
||||
with path.open("rb") as handle:
|
||||
for raw in handle:
|
||||
line = raw.decode("ascii", errors="ignore").strip()
|
||||
if line.startswith("element vertex "):
|
||||
counts["vertex_count"] = int(line.split()[-1])
|
||||
elif line.startswith("element face "):
|
||||
counts["face_count"] = int(line.split()[-1])
|
||||
elif line == "end_header":
|
||||
break
|
||||
return counts
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1198
vendor/gs_playground/scripts/real2sim_colmap_texture_bake_obj.py
vendored
Normal file
1198
vendor/gs_playground/scripts/real2sim_colmap_texture_bake_obj.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
68
vendor/gs_playground/scripts/real2sim_download_sam3d_checkpoints.sh
vendored
Executable file
68
vendor/gs_playground/scripts/real2sim_download_sam3d_checkpoints.sh
vendored
Executable file
@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download SAM-3D Objects checkpoints into the layout expected by Real2Sim.
|
||||
#
|
||||
# Official paper-equivalent source:
|
||||
# REAL2SIM_SAM3D_CHECKPOINT_REPO=facebook/sam-3d-objects
|
||||
#
|
||||
# Public mirror fallback used only when the official gated repo is unavailable:
|
||||
# REAL2SIM_SAM3D_CHECKPOINT_REPO=licesma/sam-3d-objects-weights
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "${REAL2SIM_SAM3D_ROOT:-}" ]]; then
|
||||
echo "REAL2SIM_SAM3D_ROOT is required. Source scripts/real2sim_wuying_paper_env.sh first." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
repo="${REAL2SIM_SAM3D_CHECKPOINT_REPO:-facebook/sam-3d-objects}"
|
||||
max_workers="${REAL2SIM_SAM3D_DOWNLOAD_WORKERS:-1}"
|
||||
dest="${REAL2SIM_SAM3D_CHECKPOINT_DIR:-${REAL2SIM_SAM3D_ROOT}/checkpoints/hf}"
|
||||
hf_bin="${REAL2SIM_HF_BIN:-hf}"
|
||||
force="${REAL2SIM_SAM3D_CHECKPOINT_FORCE:-0}"
|
||||
|
||||
if [[ -e "${dest}/pipeline.yaml" && "${force}" != "1" ]]; then
|
||||
echo "SAM-3D checkpoints already exist at ${dest}. Set REAL2SIM_SAM3D_CHECKPOINT_FORCE=1 to replace." >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "${REAL2SIM_SAM3D_ROOT}/checkpoints"
|
||||
download_dir="$(mktemp -d "${REAL2SIM_SAM3D_ROOT}/checkpoints/.sam3d-download.XXXXXX")"
|
||||
cleanup() {
|
||||
rm -rf "${download_dir}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"${hf_bin}" download \
|
||||
"${repo}" \
|
||||
--repo-type model \
|
||||
--local-dir "${download_dir}" \
|
||||
--max-workers "${max_workers}"
|
||||
|
||||
source_dir=""
|
||||
if [[ -f "${download_dir}/checkpoints/pipeline.yaml" ]]; then
|
||||
source_dir="${download_dir}/checkpoints"
|
||||
elif [[ -f "${download_dir}/hf/pipeline.yaml" ]]; then
|
||||
source_dir="${download_dir}/hf"
|
||||
else
|
||||
echo "Downloaded ${repo}, but no checkpoints/pipeline.yaml or hf/pipeline.yaml was found." >&2
|
||||
find "${download_dir}" -maxdepth 3 -type f | sort >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmp_dest="${dest}.tmp.$$"
|
||||
rm -rf "${tmp_dest}"
|
||||
mkdir -p "$(dirname "${dest}")"
|
||||
cp -a "${source_dir}" "${tmp_dest}"
|
||||
if [[ -e "${dest}" || -L "${dest}" ]]; then
|
||||
rm -rf "${dest}"
|
||||
fi
|
||||
mv "${tmp_dest}" "${dest}"
|
||||
|
||||
cat > "${dest}.SOURCE.txt" <<EOF
|
||||
repo=${repo}
|
||||
downloaded_at=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
destination=${dest}
|
||||
official_repo=facebook/sam-3d-objects
|
||||
EOF
|
||||
|
||||
echo "SAM-3D checkpoints are ready at ${dest}"
|
||||
281
vendor/gs_playground/scripts/real2sim_export_pose_init_dataset.py
vendored
Normal file
281
vendor/gs_playground/scripts/real2sim_export_pose_init_dataset.py
vendored
Normal file
@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export a COLMAP pose-initialized training dataset from a capture report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
COLMAP_TEXT_FILES = ("cameras.txt", "images.txt", "points3D.txt")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Export registered COLMAP frames as a pose-init 3DGS training dataset")
|
||||
parser.add_argument("--sfm-report", type=Path, required=True, help="COLMAP capture report JSON")
|
||||
parser.add_argument("--output-dir", type=Path, required=True, help="Output pose-init dataset directory")
|
||||
parser.add_argument("--allow-failed-sfm", action="store_true", help="Export even when sfm_report.passed is false")
|
||||
args = parser.parse_args()
|
||||
|
||||
summary = export_pose_init_dataset(
|
||||
sfm_report=args.sfm_report,
|
||||
output_dir=args.output_dir,
|
||||
allow_failed_sfm=bool(args.allow_failed_sfm),
|
||||
)
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def export_pose_init_dataset(*, sfm_report: Path, output_dir: Path, allow_failed_sfm: bool = False) -> dict[str, Any]:
|
||||
report = read_json(sfm_report)
|
||||
if not report.get("passed") and not allow_failed_sfm:
|
||||
raise SystemExit("SFM report did not pass; rerun with --allow-failed-sfm to export anyway")
|
||||
|
||||
frames_dir = Path(str(report.get("frames_dir", "")))
|
||||
if not frames_dir.is_dir():
|
||||
raise SystemExit(f"SFM report frames_dir does not exist: {frames_dir}")
|
||||
text_model = Path(str(report.get("best_model_text_dir", "")))
|
||||
if not text_model.is_dir():
|
||||
raise SystemExit(f"SFM report best_model_text_dir does not exist: {text_model}")
|
||||
|
||||
camera_models = {int(item["camera_id"]): item for item in report.get("camera_models", [])}
|
||||
camera_poses = list(report.get("camera_poses", []))
|
||||
if not camera_models:
|
||||
raise SystemExit("SFM report contains no camera models")
|
||||
if not camera_poses:
|
||||
raise SystemExit("SFM report contains no registered camera poses")
|
||||
|
||||
if output_dir.exists():
|
||||
shutil.rmtree(output_dir)
|
||||
image_dir = output_dir / "images"
|
||||
colmap_text_dir = output_dir / "colmap_text"
|
||||
image_dir.mkdir(parents=True, exist_ok=True)
|
||||
colmap_text_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
frames: list[dict[str, Any]] = []
|
||||
copied_images = 0
|
||||
for pose in sorted(camera_poses, key=lambda item: str(item.get("image_name", ""))):
|
||||
image_name = str(pose.get("image_name", ""))
|
||||
source_image = frames_dir / image_name
|
||||
if not source_image.is_file():
|
||||
raise SystemExit(f"registered COLMAP frame is missing: {source_image}")
|
||||
dest_image = image_dir / source_image.name
|
||||
shutil.copy2(source_image, dest_image)
|
||||
copied_images += 1
|
||||
|
||||
camera_id = int(pose["camera_id"])
|
||||
camera = camera_models.get(camera_id)
|
||||
if camera is None:
|
||||
raise SystemExit(f"pose references unknown camera_id {camera_id}: {image_name}")
|
||||
intrinsics = camera_intrinsics(camera)
|
||||
colmap_c2w = colmap_camera_to_world(pose["qvec_wxyz"], pose["tvec"])
|
||||
nerfstudio_c2w = colmap_c2w_to_nerfstudio(colmap_c2w)
|
||||
frames.append(
|
||||
{
|
||||
"file_path": f"images/{dest_image.name}",
|
||||
"transform_matrix": round_matrix(nerfstudio_c2w),
|
||||
"colmap_transform_matrix": round_matrix(colmap_c2w),
|
||||
"fl_x": intrinsics["fl_x"],
|
||||
"fl_y": intrinsics["fl_y"],
|
||||
"cx": intrinsics["cx"],
|
||||
"cy": intrinsics["cy"],
|
||||
"w": intrinsics["w"],
|
||||
"h": intrinsics["h"],
|
||||
"camera_model": intrinsics["camera_model"],
|
||||
"real2sim_pose": {
|
||||
"image_id": pose.get("image_id"),
|
||||
"camera_id": camera_id,
|
||||
"qvec_wxyz": pose.get("qvec_wxyz"),
|
||||
"tvec": pose.get("tvec"),
|
||||
"position_colmap_world": pose.get("position"),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
sparse_ply = write_sparse_points_ply(text_model / "points3D.txt", output_dir / "colmap_sparse_points.ply")
|
||||
copied_colmap_files = copy_colmap_text_model(text_model, colmap_text_dir)
|
||||
transforms = {
|
||||
"format": "gs-playground-real2sim-pose-init-transforms-v1",
|
||||
"camera_convention": "nerfstudio/OpenGL camera-to-world; raw COLMAP camera-to-world is preserved per frame",
|
||||
"source_sfm_report": sfm_report.resolve().as_posix(),
|
||||
"frame_count": len(frames),
|
||||
"registered_image_count": int(report.get("registered_image_count", len(frames))),
|
||||
"sparse_point_count": int(report.get("sparse_point_count", 0) or 0),
|
||||
"camera_path": report.get("camera_path", {}),
|
||||
"sparse_bounds": report.get("sparse_bounds", {}),
|
||||
"database_stats": report.get("database_stats", {}),
|
||||
"frames": frames,
|
||||
}
|
||||
(output_dir / "transforms.json").write_text(json.dumps(transforms, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
write_readme(output_dir, transforms, sparse_ply=sparse_ply)
|
||||
|
||||
summary = {
|
||||
"format": "gs-playground-real2sim-pose-init-dataset-v1",
|
||||
"root": output_dir.resolve().as_posix(),
|
||||
"transforms": "transforms.json",
|
||||
"images": "images",
|
||||
"image_count": copied_images,
|
||||
"colmap_text": "colmap_text",
|
||||
"colmap_text_files": copied_colmap_files,
|
||||
"sparse_points_ply": sparse_ply.name if sparse_ply is not None else None,
|
||||
"sparse_point_count": int(report.get("sparse_point_count", 0) or 0),
|
||||
"registered_ratio": report.get("registered_ratio"),
|
||||
"ready_for_training": bool(report.get("passed") and copied_images > 0 and sparse_ply is not None),
|
||||
}
|
||||
(output_dir / "pose_init_dataset.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
return summary
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise SystemExit(f"expected JSON object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def camera_intrinsics(camera: dict[str, Any]) -> dict[str, Any]:
|
||||
model = str(camera.get("model", "")).upper()
|
||||
width = int(camera["width"])
|
||||
height = int(camera["height"])
|
||||
params = [float(value) for value in camera.get("params", [])]
|
||||
if model == "SIMPLE_PINHOLE":
|
||||
f, cx, cy = params[:3]
|
||||
fx = fy = f
|
||||
elif model == "PINHOLE":
|
||||
fx, fy, cx, cy = params[:4]
|
||||
elif model in {"SIMPLE_RADIAL", "RADIAL"}:
|
||||
f, cx, cy = params[:3]
|
||||
fx = fy = f
|
||||
elif model in {"OPENCV", "OPENCV_FISHEYE", "FULL_OPENCV"}:
|
||||
fx, fy, cx, cy = params[:4]
|
||||
else:
|
||||
raise SystemExit(f"unsupported COLMAP camera model for pose-init export: {model}")
|
||||
return {
|
||||
"camera_model": model,
|
||||
"w": width,
|
||||
"h": height,
|
||||
"fl_x": round(float(fx), 6),
|
||||
"fl_y": round(float(fy), 6),
|
||||
"cx": round(float(cx), 6),
|
||||
"cy": round(float(cy), 6),
|
||||
}
|
||||
|
||||
|
||||
def colmap_camera_to_world(qvec: list[float], tvec: list[float]) -> list[list[float]]:
|
||||
rotation = qvec_to_rotmat([float(value) for value in qvec])
|
||||
t = [float(value) for value in tvec]
|
||||
rt = transpose(rotation)
|
||||
center = [-sum(rt[row][col] * t[col] for col in range(3)) for row in range(3)]
|
||||
return [
|
||||
[rt[0][0], rt[0][1], rt[0][2], center[0]],
|
||||
[rt[1][0], rt[1][1], rt[1][2], center[1]],
|
||||
[rt[2][0], rt[2][1], rt[2][2], center[2]],
|
||||
[0.0, 0.0, 0.0, 1.0],
|
||||
]
|
||||
|
||||
|
||||
def colmap_c2w_to_nerfstudio(matrix: list[list[float]]) -> list[list[float]]:
|
||||
converted = [list(row) for row in matrix]
|
||||
for row in range(3):
|
||||
converted[row][1] *= -1.0
|
||||
converted[row][2] *= -1.0
|
||||
return converted
|
||||
|
||||
|
||||
def qvec_to_rotmat(qvec: list[float]) -> list[list[float]]:
|
||||
w, x, y, z = qvec
|
||||
return [
|
||||
[1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * z * w, 2 * x * z + 2 * y * w],
|
||||
[2 * x * y + 2 * z * w, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * x * w],
|
||||
[2 * x * z - 2 * y * w, 2 * y * z + 2 * x * w, 1 - 2 * x * x - 2 * y * y],
|
||||
]
|
||||
|
||||
|
||||
def transpose(matrix: list[list[float]]) -> list[list[float]]:
|
||||
return [[matrix[col][row] for col in range(3)] for row in range(3)]
|
||||
|
||||
|
||||
def round_matrix(matrix: list[list[float]], digits: int = 8) -> list[list[float]]:
|
||||
return [[round(float(value), digits) for value in row] for row in matrix]
|
||||
|
||||
|
||||
def write_sparse_points_ply(points_txt: Path, output: Path) -> Path | None:
|
||||
points: list[tuple[float, float, float, int, int, int]] = []
|
||||
if not points_txt.is_file():
|
||||
return None
|
||||
for line in points_txt.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
x, y, z = (float(parts[index]) for index in range(1, 4))
|
||||
r, g, b = (clamp_color(float(parts[index])) for index in range(4, 7))
|
||||
points.append((x, y, z, r, g, b))
|
||||
if not points:
|
||||
return None
|
||||
with output.open("w", encoding="ascii") as handle:
|
||||
handle.write("ply\n")
|
||||
handle.write("format ascii 1.0\n")
|
||||
handle.write(f"element vertex {len(points)}\n")
|
||||
handle.write("property float x\n")
|
||||
handle.write("property float y\n")
|
||||
handle.write("property float z\n")
|
||||
handle.write("property uchar red\n")
|
||||
handle.write("property uchar green\n")
|
||||
handle.write("property uchar blue\n")
|
||||
handle.write("end_header\n")
|
||||
for x, y, z, r, g, b in points:
|
||||
handle.write(f"{x:.8f} {y:.8f} {z:.8f} {r} {g} {b}\n")
|
||||
return output
|
||||
|
||||
|
||||
def clamp_color(value: float) -> int:
|
||||
return max(0, min(255, int(round(value))))
|
||||
|
||||
|
||||
def copy_colmap_text_model(text_model: Path, output: Path) -> list[str]:
|
||||
copied: list[str] = []
|
||||
for name in COLMAP_TEXT_FILES:
|
||||
source = text_model / name
|
||||
if source.is_file():
|
||||
shutil.copy2(source, output / name)
|
||||
copied.append(name)
|
||||
return copied
|
||||
|
||||
|
||||
def write_readme(output_dir: Path, transforms: dict[str, Any], *, sparse_ply: Path | None) -> None:
|
||||
lines = [
|
||||
"# Real2Sim Pose-Init Dataset",
|
||||
"",
|
||||
"This directory is generated from a COLMAP capture report that passed the Real2Sim capture gate.",
|
||||
"It is intended as the bridge from fast AnySplat preview reconstruction to a pose-initialized 3DGS training path.",
|
||||
"",
|
||||
"Contents:",
|
||||
"",
|
||||
"- `images/`: registered video frames copied from the SfM frame set.",
|
||||
"- `transforms.json`: Nerfstudio/OpenGL-style camera-to-world transforms plus raw COLMAP pose metadata.",
|
||||
"- `colmap_text/`: the selected COLMAP text model (`cameras.txt`, `images.txt`, `points3D.txt`).",
|
||||
"- `colmap_sparse_points.ply`: colored sparse point cloud exported from `points3D.txt`.",
|
||||
"",
|
||||
"Summary:",
|
||||
"",
|
||||
f"- frames: {transforms.get('frame_count')}",
|
||||
f"- registered images: {transforms.get('registered_image_count')}",
|
||||
f"- sparse points: {transforms.get('sparse_point_count')}",
|
||||
f"- sparse PLY: {sparse_ply.name if sparse_ply is not None else 'missing'}",
|
||||
"",
|
||||
"A downstream trainer should initialize from these poses and sparse points instead of random camera/point initialization.",
|
||||
"",
|
||||
]
|
||||
(output_dir / "README.md").write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
115
vendor/gs_playground/scripts/real2sim_external/README.md
vendored
Normal file
115
vendor/gs_playground/scripts/real2sim_external/README.md
vendored
Normal file
@ -0,0 +1,115 @@
|
||||
# Real2Sim External Model Adapters
|
||||
|
||||
These scripts connect the GS-Playground Real2Sim stage contract to public model
|
||||
implementations. They are optional: the built-in backend still runs without any
|
||||
of these dependencies.
|
||||
|
||||
## Quality Upgrade Path
|
||||
|
||||
1. `groundingdino_hf_detect.py`
|
||||
- Purpose: replace heuristic connected components with open-vocabulary boxes.
|
||||
- Source: official GroundingDINO / Grounded-SAM-2 Hugging Face path.
|
||||
- Main dependency: `transformers`, `Pillow`, `torch`.
|
||||
- Default model: `IDEA-Research/grounding-dino-tiny`.
|
||||
|
||||
2. `sam2_segment.py`
|
||||
- Purpose: replace coarse masks with SAM2 box-prompted masks.
|
||||
- Source: official `facebookresearch/sam2`.
|
||||
- Required env:
|
||||
- `REAL2SIM_SAM2_CHECKPOINT=/path/to/sam2.1_hiera_large.pt`
|
||||
- `REAL2SIM_SAM2_MODEL_CFG=configs/sam2.1/sam2.1_hiera_l.yaml`
|
||||
|
||||
3. `instance_refine.py`
|
||||
- Purpose: match the paper's visual-only instance de-duplication stage after
|
||||
SAM/SAM2.
|
||||
- Method: composite detection/segmentation confidence, mask IoU, mask
|
||||
inclusion, and boundary overlap.
|
||||
|
||||
4. `lama_inpaint.py`
|
||||
- Purpose: replace median-fill background removal with sequential LaMa
|
||||
inpainting.
|
||||
- Runtime used here: `simple-lama-inpainting`, a lightweight LaMa wrapper
|
||||
listed by the official LaMa repository.
|
||||
- Method: one-object-at-a-time mask expansion and sequential background
|
||||
update. Re-detection of newly exposed regions must be supplied by the
|
||||
configured detection/segmentation loop for fully strict paper parity.
|
||||
The strict config uses `REAL2SIM_GROUNDING_DINO_COMMAND` and
|
||||
`REAL2SIM_SAM2_COMMAND` for this nested loop.
|
||||
|
||||
5. `depth_anything_depth.py`
|
||||
- Purpose: replace flat pseudo-depth detail with a monocular Depth Anything
|
||||
V2 depth prior before building the fallback 3DGS assets.
|
||||
- Main dependency: `transformers`, `Pillow`, `torch`.
|
||||
- Default model: `depth-anything/Depth-Anything-V2-Small-hf`.
|
||||
|
||||
6. `anysplat_scene.py`
|
||||
- Purpose: replace pseudo-depth scene Gaussians with AnySplat scene Gaussians.
|
||||
- Source: official AnySplat repo and Hugging Face model `lhjiang/anysplat`.
|
||||
- Output: official `export_ply` scene Gaussians, decoder depth, camera
|
||||
metadata, and a depth-derived scene mesh.
|
||||
- Required env:
|
||||
- `REAL2SIM_ANYSPLAT_ROOT=/path/to/AnySplat`
|
||||
- Best input: multiple nearby views or a short video, not a single repeated
|
||||
frame.
|
||||
|
||||
7. `sam3d_objects_reconstruct.py`
|
||||
- Purpose: replace object box meshes and object pseudo-Gaussians with SAM 3D
|
||||
Objects outputs.
|
||||
- Source: official Meta `facebookresearch/sam-3d-objects`.
|
||||
- Output: object Gaussian PLY from the SAM-3D `save_ply` output, an
|
||||
exportable object mesh, and pose/scale metadata. The script fails if no
|
||||
mesh is returned unless `--allow-fallback-mesh` is passed for debugging.
|
||||
- Required env:
|
||||
- `REAL2SIM_SAM3D_ROOT=/path/to/sam-3d-objects`
|
||||
- `REAL2SIM_SAM3D_CONFIG=/path/to/checkpoints/hf/pipeline.yaml`
|
||||
- Checkpoints are gated on Hugging Face and require accepted access. The
|
||||
upstream setup currently recommends an NVIDIA GPU with at least 32 GB VRAM.
|
||||
- Use `--cameras {cameras_json} --match-camera-preprocess` when the scene
|
||||
was reconstructed by AnySplat, so SAM-3D receives the same
|
||||
aspect-preserving center crop and mask geometry as the scene camera.
|
||||
- Use `--canonicalize-gaussian-to-mesh` to rotate the object Gaussian PLY
|
||||
into the SAM-3D mesh canonical frame using only geometric extent axes. It
|
||||
updates positions, normals, and splat quaternions without source-image
|
||||
photometric scoring.
|
||||
- Use `--remove-video-overlay` only for screenshots/video thumbnails with a
|
||||
central play button; it inpaints detected bright, low-saturation UI pixels
|
||||
near the image center before SAM-3D inference.
|
||||
|
||||
8. `depth_mask_pose_align.py`
|
||||
- Purpose: align SAM-3D objects back into the AnySplat scene using rendered
|
||||
object Gaussian PLY depth/mask, scene depth, and camera intrinsics.
|
||||
Mesh rendering is used only as a fallback when the object Gaussian asset is
|
||||
unavailable or invalid.
|
||||
- Output: an updated `{aligned_assets_json}` manifest with refined
|
||||
`pose.position`, `pose.quat_wxyz`, and `pose.scale` values plus alignment
|
||||
metadata including the selected `render_source`, candidate axis search
|
||||
label, mask IoU, centroid error, depth error, and quality summary.
|
||||
If the object Gaussian directory contains SAM-3D metadata showing
|
||||
`gaussian_canonicalization.status=ok`, the aligner keeps the canonical
|
||||
rotation and uses only a mask/depth position-scale seed instead of
|
||||
source-view axis search.
|
||||
|
||||
9. Speedy-Splat pruning
|
||||
- Purpose: match the paper's 3DGS pruning stage.
|
||||
- Configure `pruning.mode=speedy_splat` and set
|
||||
`pruning.external_command` or `REAL2SIM_SPEEDY_SPLAT_COMMAND`.
|
||||
- The pipeline fails if the command is missing in Speedy-Splat mode.
|
||||
- The adapter keeps Speedy-Splat's official score computation, but applies
|
||||
an exact-count tie-safe prune mask so all-zero or heavily tied object
|
||||
scores cannot delete every Gaussian in a canonical local object asset.
|
||||
|
||||
## Example
|
||||
|
||||
```bash
|
||||
python -m real2sim run \
|
||||
--backend hybrid \
|
||||
--config configs/real2sim.external.example.json \
|
||||
--input media/teaser.png \
|
||||
--output /tmp/gs_real2sim_hybrid \
|
||||
--prompt "robot. person. table. chair. object."
|
||||
```
|
||||
|
||||
Use `--backend external` only after AnySplat and SAM 3D Objects are installed,
|
||||
their checkpoints are available, and the machine meets the SAM-3D memory
|
||||
requirements. Use `python -m real2sim run ... --dry-run --backend hybrid` to
|
||||
check which stage commands are configured before launching a long model run.
|
||||
223
vendor/gs_playground/scripts/real2sim_external/anysplat_scene.py
vendored
Normal file
223
vendor/gs_playground/scripts/real2sim_external/anysplat_scene.py
vendored
Normal file
@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""AnySplat scene reconstruction adapter for Real2Sim.
|
||||
|
||||
This adapter uses the official AnySplat model/API and exports a real 3DGS PLY
|
||||
through AnySplat's `export_ply` helper. Video inputs can provide real nearby
|
||||
frames through `--frames-dir`; single-image inputs are kept as a single view
|
||||
instead of pretending that duplicated frames are independent observations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.contracts import write_json
|
||||
from real2sim.image_io import write_gray
|
||||
|
||||
|
||||
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".ppm"}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run AnySplat scene reconstruction for Real2Sim")
|
||||
parser.add_argument("--image", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--ply", type=Path, required=True)
|
||||
parser.add_argument("--mesh", type=Path, required=True)
|
||||
parser.add_argument("--cameras", type=Path, required=True)
|
||||
parser.add_argument("--depth", type=Path, required=True)
|
||||
parser.add_argument("--source-preview", type=Path, default=None, help="Optional source-camera RGB render preview PNG")
|
||||
parser.add_argument("--frames-dir", type=Path, default=None, help="Optional extracted video frames directory")
|
||||
parser.add_argument("--anysplat-root", type=Path, default=os.environ.get("REAL2SIM_ANYSPLAT_ROOT"))
|
||||
parser.add_argument("--model-id", default=os.environ.get("REAL2SIM_ANYSPLAT_MODEL_ID", "lhjiang/anysplat"))
|
||||
parser.add_argument("--max-views", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MAX_VIEWS", "8")))
|
||||
parser.add_argument("--mesh-grid", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MESH_GRID", "48")))
|
||||
parser.add_argument("--shift-and-scale", action="store_true", help="Normalize exported Gaussian positions with AnySplat exporter")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.anysplat_root is None:
|
||||
raise SystemExit("AnySplat root is required via --anysplat-root or REAL2SIM_ANYSPLAT_ROOT")
|
||||
sys.path.insert(0, args.anysplat_root.as_posix())
|
||||
|
||||
import torch
|
||||
from src.model.model.anysplat import AnySplat
|
||||
from src.model.ply_export import export_ply
|
||||
from src.utils.image import process_image
|
||||
from real2sim.builtin import write_depth_mesh_obj
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
model = AnySplat.from_pretrained(args.model_id).to(device)
|
||||
model.eval()
|
||||
for param in model.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
image_paths = collect_image_paths(args.image, args.frames_dir, max_views=args.max_views)
|
||||
images = torch.stack([process_image(path.as_posix()) for path in image_paths], dim=0).unsqueeze(0).to(device)
|
||||
b, views, _, height, width = images.shape
|
||||
with torch.inference_mode():
|
||||
gaussians, pose = model.inference((images + 1) * 0.5)
|
||||
|
||||
export_ply(
|
||||
gaussians.means[0],
|
||||
gaussians.scales[0],
|
||||
gaussians.rotations[0],
|
||||
gaussians.harmonics[0],
|
||||
gaussians.opacities[0],
|
||||
args.ply,
|
||||
shift_and_scale=args.shift_and_scale,
|
||||
save_sh_dc_only=True,
|
||||
)
|
||||
|
||||
extrinsic = pose["extrinsic"]
|
||||
intrinsic = pose["intrinsic"]
|
||||
near = torch.ones(b, views, device=device) * 0.1
|
||||
far = torch.ones(b, views, device=device) * 100.0
|
||||
with torch.inference_mode():
|
||||
rendered = model.decoder.forward(gaussians, extrinsic, intrinsic.float(), near, far, (height, width), "depth")
|
||||
|
||||
preview_metrics = None
|
||||
if args.source_preview is not None:
|
||||
source_rgb_tensor = ((images[0, 0].detach().cpu() + 1.0) * 0.5).clamp(0.0, 1.0)
|
||||
rendered_rgb_tensor = rendered.color[0, 0].detach().cpu().clamp(0.0, 1.0)
|
||||
preview_metrics = save_source_preview(args.source_preview, rendered_rgb_tensor, source_rgb_tensor)
|
||||
|
||||
depth = extract_first_depth_map(rendered.depth)
|
||||
normalized_depth = normalize_depth(depth)
|
||||
write_gray(args.depth, normalized_depth)
|
||||
|
||||
intrinsics = denormalize_intrinsics(intrinsic[0, 0].detach().cpu().numpy(), width, height)
|
||||
scene_rgb = (((images[0, 0].detach().cpu().permute(1, 2, 0).numpy()) + 1.0) * 0.5).clip(0.0, 1.0)
|
||||
write_depth_mesh_obj(args.mesh, scene_rgb, normalized_depth, intrinsics, grid=max(8, int(args.mesh_grid)))
|
||||
|
||||
write_json(
|
||||
args.cameras,
|
||||
{
|
||||
"backend": "anysplat",
|
||||
"model_id": args.model_id,
|
||||
"image_paths": [path.as_posix() for path in image_paths],
|
||||
"image_shape": [height, width],
|
||||
"intrinsics": intrinsics,
|
||||
"source_preview": args.source_preview.as_posix() if args.source_preview is not None else None,
|
||||
"source_preview_metrics": preview_metrics,
|
||||
"pred_context_pose": to_jsonable(pose),
|
||||
},
|
||||
)
|
||||
write_json(
|
||||
args.output_dir / "anysplat_metadata.json",
|
||||
{
|
||||
"backend": "anysplat",
|
||||
"model_id": args.model_id,
|
||||
"num_views": len(image_paths),
|
||||
"max_views": args.max_views,
|
||||
"mesh_grid": max(8, int(args.mesh_grid)),
|
||||
"shift_and_scale": args.shift_and_scale,
|
||||
"source_preview": args.source_preview.as_posix() if args.source_preview is not None else None,
|
||||
"source_preview_metrics": preview_metrics,
|
||||
},
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def save_source_preview(output: Path, rendered_rgb, source_rgb) -> dict[str, float]:
|
||||
from PIL import Image
|
||||
|
||||
rendered = tensor_rgb_to_numpy(rendered_rgb)
|
||||
source = tensor_rgb_to_numpy(source_rgb)
|
||||
mae = float(np.mean(np.abs(rendered.astype(np.float32) / 255.0 - source.astype(np.float32) / 255.0)))
|
||||
mse = float(np.mean((rendered.astype(np.float32) / 255.0 - source.astype(np.float32) / 255.0) ** 2))
|
||||
psnr = float(99.0 if mse <= 1e-12 else -10.0 * np.log10(mse))
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.fromarray(rendered).save(output)
|
||||
return {"mae": mae, "mse": mse, "psnr": psnr}
|
||||
|
||||
|
||||
def tensor_rgb_to_numpy(rgb) -> np.ndarray:
|
||||
if hasattr(rgb, "detach"):
|
||||
rgb = rgb.detach().cpu().numpy()
|
||||
arr = np.asarray(rgb, dtype=np.float32)
|
||||
if arr.ndim == 3 and arr.shape[0] == 3:
|
||||
arr = np.transpose(arr, (1, 2, 0))
|
||||
if arr.ndim != 3 or arr.shape[-1] != 3:
|
||||
raise SystemExit(f"RGB preview tensor must have shape 3xHxW or HxWx3, got {arr.shape}")
|
||||
return (arr.clip(0.0, 1.0) * 255.0).round().astype(np.uint8)
|
||||
|
||||
|
||||
def collect_image_paths(image: Path, frames_dir: Path | None, *, max_views: int) -> list[Path]:
|
||||
if frames_dir is not None and str(frames_dir) not in {"", ".", "None", "__none__"} and frames_dir.is_dir():
|
||||
frames = sorted(path for path in frames_dir.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES)
|
||||
if frames:
|
||||
return sample_evenly(frames, max_views=max(1, max_views))
|
||||
return [image]
|
||||
|
||||
|
||||
def sample_evenly(paths: list[Path], *, max_views: int) -> list[Path]:
|
||||
if len(paths) <= max_views:
|
||||
return paths
|
||||
if max_views == 1:
|
||||
return [paths[0]]
|
||||
indices = np.linspace(0, len(paths) - 1, num=max_views).round().astype(int)
|
||||
return [paths[int(idx)] for idx in sorted(set(indices))]
|
||||
|
||||
|
||||
def normalize_depth(depth: np.ndarray) -> np.ndarray:
|
||||
finite = np.asarray(depth, dtype=np.float32)
|
||||
valid = np.isfinite(finite)
|
||||
if not valid.any():
|
||||
return np.zeros_like(finite, dtype=np.float32)
|
||||
lo = float(np.quantile(finite[valid], 0.02))
|
||||
hi = float(np.quantile(finite[valid], 0.98))
|
||||
if hi <= lo:
|
||||
return np.zeros_like(finite, dtype=np.float32)
|
||||
return np.clip((finite - lo) / (hi - lo), 0.0, 1.0).astype(np.float32)
|
||||
|
||||
|
||||
def extract_first_depth_map(depth) -> np.ndarray:
|
||||
if hasattr(depth, "detach"):
|
||||
depth = depth.detach().cpu().numpy()
|
||||
arr = np.asarray(depth, dtype=np.float32)
|
||||
if arr.ndim == 5 and arr.shape[-1] == 1:
|
||||
arr = arr[..., 0]
|
||||
if arr.ndim == 4:
|
||||
arr = arr[0, 0]
|
||||
elif arr.ndim == 3:
|
||||
arr = arr[0]
|
||||
elif arr.ndim != 2:
|
||||
raise SystemExit(f"AnySplat decoder depth must be 2D, 3D, or 4D, got shape {arr.shape}")
|
||||
if arr.ndim != 2:
|
||||
raise SystemExit(f"AnySplat decoder depth did not reduce to a 2D map, got shape {arr.shape}")
|
||||
return arr.astype(np.float32, copy=False)
|
||||
|
||||
|
||||
def denormalize_intrinsics(intrinsic: np.ndarray, width: int, height: int) -> dict[str, float | int]:
|
||||
intr = np.asarray(intrinsic, dtype=np.float32)
|
||||
return {
|
||||
"width": int(width),
|
||||
"height": int(height),
|
||||
"fx": float(intr[0, 0] * width),
|
||||
"fy": float(intr[1, 1] * height),
|
||||
"cx": float(intr[0, 2] * width),
|
||||
"cy": float(intr[1, 2] * height),
|
||||
}
|
||||
|
||||
|
||||
def to_jsonable(value):
|
||||
if isinstance(value, dict):
|
||||
return {str(k): to_jsonable(v) for k, v in value.items()}
|
||||
if hasattr(value, "detach"):
|
||||
return value.detach().cpu().tolist()
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [to_jsonable(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
85
vendor/gs_playground/scripts/real2sim_external/depth_anything_depth.py
vendored
Normal file
85
vendor/gs_playground/scripts/real2sim_external/depth_anything_depth.py
vendored
Normal file
@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Depth Anything V2 adapter for the Real2Sim hybrid backend.
|
||||
|
||||
The adapter writes a normalized PGM/PNG depth map that the built-in asset
|
||||
generator can fuse with its geometry prior. The default model is the small
|
||||
Transformers checkpoint so it can run on the Wuying 12 GB GPU.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.image_io import write_gray
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run Depth Anything V2 monocular depth for Real2Sim")
|
||||
parser.add_argument("--image", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default=os.environ.get("REAL2SIM_DEPTH_MODEL", "depth-anything/Depth-Anything-V2-Small-hf"),
|
||||
help="Hugging Face depth-estimation model id",
|
||||
)
|
||||
parser.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda"])
|
||||
parser.add_argument("--invert", dest="invert", action="store_true", help="Invert relative depth before writing")
|
||||
parser.add_argument("--no-invert", dest="invert", action="store_false")
|
||||
parser.set_defaults(invert=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
depth = estimate_depth(args.image, args.model, args.device)
|
||||
if args.invert:
|
||||
depth = 1.0 - depth
|
||||
write_gray(args.output, depth)
|
||||
return 0
|
||||
|
||||
|
||||
def estimate_depth(image_path: Path, model_id: str, device: str) -> np.ndarray:
|
||||
try:
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import pipeline
|
||||
except Exception as exc: # pragma: no cover - external runtime guard
|
||||
raise SystemExit(
|
||||
"Depth Anything requires torch, pillow, and transformers. "
|
||||
"Install them in the Real2Sim environment before enabling depth_estimation."
|
||||
) from exc
|
||||
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
device_index = -1
|
||||
if device == "cuda" or (device == "auto" and torch.cuda.is_available()):
|
||||
device_index = 0
|
||||
|
||||
pipe = pipeline(task="depth-estimation", model=model_id, device=device_index)
|
||||
result = pipe(image)
|
||||
raw_depth = result.get("predicted_depth")
|
||||
if raw_depth is not None:
|
||||
depth = raw_depth.detach().float().cpu().numpy()
|
||||
depth = np.squeeze(depth)
|
||||
else:
|
||||
depth = np.asarray(result["depth"], dtype=np.float32)
|
||||
if depth.ndim == 3:
|
||||
depth = depth[:, :, 0]
|
||||
return normalize_depth(depth)
|
||||
|
||||
|
||||
def normalize_depth(depth: np.ndarray) -> np.ndarray:
|
||||
depth = depth.astype(np.float32)
|
||||
lo = float(np.percentile(depth, 1.0))
|
||||
hi = float(np.percentile(depth, 99.0))
|
||||
if hi <= lo:
|
||||
return np.zeros_like(depth, dtype=np.float32)
|
||||
return np.clip((depth - lo) / (hi - lo), 0.0, 1.0).astype(np.float32)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1180
vendor/gs_playground/scripts/real2sim_external/depth_mask_pose_align.py
vendored
Normal file
1180
vendor/gs_playground/scripts/real2sim_external/depth_mask_pose_align.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
146
vendor/gs_playground/scripts/real2sim_external/groundingdino_hf_detect.py
vendored
Normal file
146
vendor/gs_playground/scripts/real2sim_external/groundingdino_hf_detect.py
vendored
Normal file
@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""GroundingDINO detection adapter for Real2Sim.
|
||||
|
||||
This uses the Hugging Face GroundingDINO interface used by the official
|
||||
Grounded-SAM-2 demos. It writes the Real2Sim detections.json contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.contracts import write_json
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run GroundingDINO via Hugging Face and write detections.json")
|
||||
parser.add_argument("--image", type=Path, required=True)
|
||||
parser.add_argument("--prompt", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--model-id", default="IDEA-Research/grounding-dino-tiny")
|
||||
parser.add_argument("--box-threshold", type=float, default=0.25)
|
||||
parser.add_argument("--text-threshold", type=float, default=0.25)
|
||||
parser.add_argument("--nms-threshold", type=float, default=0.6)
|
||||
parser.add_argument("--max-detections", type=int, default=24)
|
||||
parser.add_argument("--device", default="auto", choices=["auto", "cuda", "cpu"])
|
||||
args = parser.parse_args()
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoModelForZeroShotObjectDetection, AutoProcessor
|
||||
|
||||
device = "cuda" if args.device == "auto" and torch.cuda.is_available() else args.device
|
||||
if device == "auto":
|
||||
device = "cpu"
|
||||
|
||||
image = Image.open(args.image).convert("RGB")
|
||||
processor = AutoProcessor.from_pretrained(args.model_id)
|
||||
model = AutoModelForZeroShotObjectDetection.from_pretrained(args.model_id).to(device)
|
||||
|
||||
detections = []
|
||||
queries = prompt_queries(args.prompt)
|
||||
for prompt_index, prompt in enumerate(queries):
|
||||
inputs = processor(images=image, text=prompt, return_tensors="pt").to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
outputs = model(**inputs)
|
||||
|
||||
result = processor.post_process_grounded_object_detection(
|
||||
outputs,
|
||||
inputs.input_ids,
|
||||
threshold=args.box_threshold,
|
||||
text_threshold=args.text_threshold,
|
||||
target_sizes=[image.size[::-1]],
|
||||
)[0]
|
||||
|
||||
boxes = result.get("boxes", [])
|
||||
scores = result.get("scores", [])
|
||||
labels = result.get("labels", [])
|
||||
for idx, box in enumerate(boxes):
|
||||
label = labels[idx] if idx < len(labels) else prompt.rstrip(".")
|
||||
score = float(scores[idx].detach().cpu().item()) if idx < len(scores) else None
|
||||
bbox = [float(v) for v in box.detach().cpu().tolist()]
|
||||
detections.append(
|
||||
{
|
||||
"id": f"det_{prompt_index:03d}_{idx:03d}",
|
||||
"label": str(label).strip().rstrip(".") or prompt.rstrip("."),
|
||||
"score": score,
|
||||
"bbox_xyxy": bbox,
|
||||
"prompt_query": prompt,
|
||||
"prompt_index": prompt_index,
|
||||
}
|
||||
)
|
||||
|
||||
detections = nms_detections(detections, args.nms_threshold)[: max(0, args.max_detections)]
|
||||
write_json(
|
||||
args.output,
|
||||
{
|
||||
"detections": detections,
|
||||
"metadata": {
|
||||
"backend": "groundingdino_hf",
|
||||
"model_id": args.model_id,
|
||||
"prompt": args.prompt,
|
||||
"prompt_queries": queries,
|
||||
"detection_scheme": "prompt_wise_independent",
|
||||
"box_threshold": args.box_threshold,
|
||||
"text_threshold": args.text_threshold,
|
||||
"nms_threshold": args.nms_threshold,
|
||||
"max_detections": args.max_detections,
|
||||
},
|
||||
},
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def normalize_prompt(prompt: str) -> str:
|
||||
text = " ".join(prompt.strip().lower().split())
|
||||
if not text:
|
||||
text = "object"
|
||||
return text if text.endswith(".") else f"{text}."
|
||||
|
||||
|
||||
def prompt_queries(prompt: str) -> list[str]:
|
||||
parts = [part.strip() for part in re.split(r"[.,;\n]+", prompt) if part.strip()]
|
||||
if not parts:
|
||||
parts = ["object"]
|
||||
return [normalize_prompt(part) for part in parts]
|
||||
|
||||
|
||||
def nms_detections(detections: list[dict], threshold: float) -> list[dict]:
|
||||
pending = sorted(detections, key=lambda item: float(item.get("score") or 0.0), reverse=True)
|
||||
kept: list[dict] = []
|
||||
while pending:
|
||||
current = pending.pop(0)
|
||||
kept.append(current)
|
||||
pending = [
|
||||
candidate
|
||||
for candidate in pending
|
||||
if current.get("label") != candidate.get("label")
|
||||
or iou(current["bbox_xyxy"], candidate["bbox_xyxy"]) < threshold
|
||||
]
|
||||
for idx, item in enumerate(kept):
|
||||
item["id"] = f"det_{idx:03d}"
|
||||
return kept
|
||||
|
||||
|
||||
def iou(a: list[float], b: list[float]) -> float:
|
||||
ax0, ay0, ax1, ay1 = a
|
||||
bx0, by0, bx1, by1 = b
|
||||
ix0, iy0 = max(ax0, bx0), max(ay0, by0)
|
||||
ix1, iy1 = min(ax1, bx1), min(ay1, by1)
|
||||
iw, ih = max(0.0, ix1 - ix0), max(0.0, iy1 - iy0)
|
||||
inter = iw * ih
|
||||
area_a = max(0.0, ax1 - ax0) * max(0.0, ay1 - ay0)
|
||||
area_b = max(0.0, bx1 - bx0) * max(0.0, by1 - by0)
|
||||
union = area_a + area_b - inter
|
||||
return inter / union if union > 0 else 0.0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
328
vendor/gs_playground/scripts/real2sim_external/instance_refine.py
vendored
Normal file
328
vendor/gs_playground/scripts/real2sim_external/instance_refine.py
vendored
Normal file
@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Paper-style visual instance refinement for Real2Sim.
|
||||
|
||||
The GS-Playground paper avoids semantic-similarity de-duplication after
|
||||
open-vocabulary detection. This stage uses only visual mask criteria and a
|
||||
composite detection/segmentation confidence score.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.contracts import read_json, write_json
|
||||
from real2sim.image_io import read_gray, read_rgb, write_gray
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Refine SAM/SAM2 instances using visual de-duplication")
|
||||
parser.add_argument("--instances", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--image", type=Path, default=None, help="Optional source RGB image for accessory mask recovery")
|
||||
parser.add_argument("--mask-iou-threshold", type=float, default=0.75)
|
||||
parser.add_argument("--inclusion-threshold", type=float, default=0.92)
|
||||
parser.add_argument("--boundary-overlap-threshold", type=float, default=0.85)
|
||||
parser.add_argument("--merge-saturated-accessories", action="store_true")
|
||||
parser.add_argument("--accessory-search-margin-ratio", type=float, default=0.25)
|
||||
parser.add_argument("--accessory-saturation-threshold", type=float, default=0.45)
|
||||
parser.add_argument("--accessory-contact-passes", type=int, default=4)
|
||||
parser.add_argument("--accessory-max-area-growth", type=float, default=0.15)
|
||||
parser.add_argument("--accessory-min-component-area-ratio", type=float, default=0.001)
|
||||
parser.add_argument("--accessory-max-components", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
|
||||
raw = read_json(args.instances)
|
||||
entries = raw.get("instances", raw) if isinstance(raw, dict) else raw
|
||||
image = read_rgb(args.image) if args.image is not None and args.image.exists() else None
|
||||
refined = refine_instance_entries(
|
||||
entries,
|
||||
args.instances.parent,
|
||||
image=image,
|
||||
merge_saturated_accessories=args.merge_saturated_accessories,
|
||||
mask_iou_threshold=args.mask_iou_threshold,
|
||||
inclusion_threshold=args.inclusion_threshold,
|
||||
boundary_overlap_threshold=args.boundary_overlap_threshold,
|
||||
accessory_search_margin_ratio=args.accessory_search_margin_ratio,
|
||||
accessory_saturation_threshold=args.accessory_saturation_threshold,
|
||||
accessory_contact_passes=args.accessory_contact_passes,
|
||||
accessory_max_area_growth=args.accessory_max_area_growth,
|
||||
accessory_min_component_area_ratio=args.accessory_min_component_area_ratio,
|
||||
accessory_max_components=args.accessory_max_components,
|
||||
)
|
||||
metadata = dict(raw.get("metadata", {})) if isinstance(raw, dict) else {}
|
||||
metadata["instance_refinement"] = {
|
||||
"backend": "visual_mask_dedup",
|
||||
"input_instances": len(entries),
|
||||
"output_instances": len(refined),
|
||||
"mask_iou_threshold": args.mask_iou_threshold,
|
||||
"inclusion_threshold": args.inclusion_threshold,
|
||||
"boundary_overlap_threshold": args.boundary_overlap_threshold,
|
||||
"merge_saturated_accessories": args.merge_saturated_accessories,
|
||||
"accessory_search_margin_ratio": args.accessory_search_margin_ratio,
|
||||
"accessory_saturation_threshold": args.accessory_saturation_threshold,
|
||||
"accessory_contact_passes": args.accessory_contact_passes,
|
||||
"accessory_max_area_growth": args.accessory_max_area_growth,
|
||||
"accessory_min_component_area_ratio": args.accessory_min_component_area_ratio,
|
||||
"accessory_max_components": args.accessory_max_components,
|
||||
"accessory_pixels_added": sum(int(entry.get("accessory_pixels_added", 0)) for entry in refined),
|
||||
}
|
||||
write_json(args.output, {"instances": refined, "metadata": metadata})
|
||||
return 0
|
||||
|
||||
|
||||
def refine_instance_entries(
|
||||
entries: list[dict[str, Any]],
|
||||
base_dir: Path,
|
||||
*,
|
||||
image: np.ndarray | None = None,
|
||||
merge_saturated_accessories: bool = False,
|
||||
mask_iou_threshold: float = 0.75,
|
||||
inclusion_threshold: float = 0.92,
|
||||
boundary_overlap_threshold: float = 0.85,
|
||||
accessory_search_margin_ratio: float = 0.25,
|
||||
accessory_saturation_threshold: float = 0.45,
|
||||
accessory_contact_passes: int = 4,
|
||||
accessory_max_area_growth: float = 0.15,
|
||||
accessory_min_component_area_ratio: float = 0.001,
|
||||
accessory_max_components: int = 8,
|
||||
) -> list[dict[str, Any]]:
|
||||
prepared = []
|
||||
for index, entry in enumerate(entries):
|
||||
mask_path = entry.get("mask_path", entry.get("mask"))
|
||||
if not mask_path:
|
||||
continue
|
||||
path = Path(str(mask_path))
|
||||
path = path if path.is_absolute() else (base_dir / path).resolve()
|
||||
if not path.exists():
|
||||
continue
|
||||
mask = read_gray(path) > 0.5
|
||||
if merge_saturated_accessories and image is not None and image.shape[:2] == mask.shape:
|
||||
mask, merge_event = merge_saturated_accessories_into_mask(
|
||||
mask,
|
||||
image,
|
||||
bbox_xyxy=entry.get("bbox_xyxy"),
|
||||
search_margin_ratio=accessory_search_margin_ratio,
|
||||
saturation_threshold=accessory_saturation_threshold,
|
||||
contact_passes=accessory_contact_passes,
|
||||
max_area_growth=accessory_max_area_growth,
|
||||
min_component_area_ratio=accessory_min_component_area_ratio,
|
||||
max_components=accessory_max_components,
|
||||
)
|
||||
if merge_event["added_pixels"] > 0:
|
||||
write_gray(path, mask.astype(np.float32))
|
||||
entry = dict(entry)
|
||||
entry["accessory_pixels_added"] = merge_event["added_pixels"]
|
||||
entry["accessory_components_added"] = merge_event["components_added"]
|
||||
score = composite_score(entry)
|
||||
item = dict(entry)
|
||||
item["composite_score"] = score
|
||||
item["_input_index"] = index
|
||||
prepared.append((item, mask))
|
||||
|
||||
prepared.sort(key=lambda pair: (-float(pair[0]["composite_score"]), int(pair[0]["_input_index"])))
|
||||
kept: list[tuple[dict[str, Any], np.ndarray]] = []
|
||||
for item, mask in prepared:
|
||||
duplicate = False
|
||||
for kept_item, kept_mask in kept:
|
||||
if is_visual_duplicate(
|
||||
mask,
|
||||
kept_mask,
|
||||
mask_iou_threshold=mask_iou_threshold,
|
||||
inclusion_threshold=inclusion_threshold,
|
||||
boundary_overlap_threshold=boundary_overlap_threshold,
|
||||
):
|
||||
duplicate = True
|
||||
break
|
||||
if not duplicate:
|
||||
kept.append((item, mask))
|
||||
|
||||
kept.sort(key=lambda pair: int(pair[0]["_input_index"]))
|
||||
output = []
|
||||
for idx, (item, _) in enumerate(kept):
|
||||
cleaned = dict(item)
|
||||
cleaned.pop("_input_index", None)
|
||||
cleaned["id"] = str(cleaned.get("id", cleaned.get("instance_id", f"inst_{idx:03d}")))
|
||||
output.append(cleaned)
|
||||
return output
|
||||
|
||||
|
||||
def composite_score(entry: dict[str, Any]) -> float:
|
||||
detection = entry.get("detection_score", entry.get("score", 0.0))
|
||||
segmentation = entry.get("segmentation_score", entry.get("mask_score", detection))
|
||||
detection = 0.0 if detection is None else float(detection)
|
||||
segmentation = 0.0 if segmentation is None else float(segmentation)
|
||||
return 0.5 * detection + 0.5 * segmentation
|
||||
|
||||
|
||||
def is_visual_duplicate(
|
||||
mask: np.ndarray,
|
||||
reference: np.ndarray,
|
||||
*,
|
||||
mask_iou_threshold: float,
|
||||
inclusion_threshold: float,
|
||||
boundary_overlap_threshold: float,
|
||||
) -> bool:
|
||||
if mask.shape != reference.shape:
|
||||
return False
|
||||
intersection = int((mask & reference).sum())
|
||||
area = int(mask.sum())
|
||||
reference_area = int(reference.sum())
|
||||
if area == 0 or reference_area == 0:
|
||||
return False
|
||||
union = area + reference_area - intersection
|
||||
if union > 0 and intersection / union >= mask_iou_threshold:
|
||||
return True
|
||||
smaller = mask if area <= reference_area else reference
|
||||
larger = reference if area <= reference_area else mask
|
||||
inclusion = int((smaller & larger).sum()) / float(int(smaller.sum()))
|
||||
boundary_overlap = int((boundary(smaller) & larger).sum()) / max(1, int(boundary(smaller).sum()))
|
||||
return inclusion >= inclusion_threshold and boundary_overlap >= boundary_overlap_threshold
|
||||
|
||||
|
||||
def boundary(mask: np.ndarray) -> np.ndarray:
|
||||
eroded = mask.copy()
|
||||
padded = np.pad(mask, 1, mode="constant", constant_values=False)
|
||||
eroded &= padded[:-2, 1:-1]
|
||||
eroded &= padded[2:, 1:-1]
|
||||
eroded &= padded[1:-1, :-2]
|
||||
eroded &= padded[1:-1, 2:]
|
||||
return mask & ~eroded
|
||||
|
||||
|
||||
def merge_saturated_accessories_into_mask(
|
||||
mask: np.ndarray,
|
||||
image: np.ndarray,
|
||||
*,
|
||||
bbox_xyxy: Any | None = None,
|
||||
search_margin_ratio: float = 0.25,
|
||||
saturation_threshold: float = 0.45,
|
||||
contact_passes: int = 4,
|
||||
max_area_growth: float = 0.15,
|
||||
min_component_area_ratio: float = 0.001,
|
||||
max_components: int = 8,
|
||||
) -> tuple[np.ndarray, dict[str, int]]:
|
||||
base = np.asarray(mask, dtype=bool)
|
||||
if not base.any():
|
||||
return base, {"added_pixels": 0, "components_added": 0}
|
||||
|
||||
search = expanded_search_region(base, bbox_xyxy, margin_ratio=search_margin_ratio)
|
||||
rgb = np.asarray(image, dtype=np.float32)
|
||||
if rgb.max(initial=0.0) > 1.5:
|
||||
rgb = rgb / 255.0
|
||||
saturation = rgb_saturation(rgb)
|
||||
candidates = (saturation >= saturation_threshold) & search & ~base
|
||||
if not candidates.any():
|
||||
return base, {"added_pixels": 0, "components_added": 0}
|
||||
|
||||
contact = dilate(base, max(0, int(contact_passes)))
|
||||
max_added = max(1, int(base.sum() * max(0.0, max_area_growth)))
|
||||
min_component_area = max(1, int(np.ceil(base.sum() * max(0.0, min_component_area_ratio))))
|
||||
merged = base.copy()
|
||||
added_pixels = 0
|
||||
components_added = 0
|
||||
components = connected_components(candidates)
|
||||
components.sort(key=lambda comp: int(comp.sum()), reverse=True)
|
||||
for component in components:
|
||||
if max_components > 0 and components_added >= max_components:
|
||||
break
|
||||
area = int(component.sum())
|
||||
if area < min_component_area or added_pixels + area > max_added:
|
||||
continue
|
||||
if not (component & contact).any():
|
||||
continue
|
||||
merged |= component
|
||||
contact = dilate(merged, max(0, int(contact_passes)))
|
||||
added_pixels += area
|
||||
components_added += 1
|
||||
return merged, {"added_pixels": added_pixels, "components_added": components_added}
|
||||
|
||||
|
||||
def expanded_search_region(mask: np.ndarray, bbox_xyxy: Any | None, *, margin_ratio: float) -> np.ndarray:
|
||||
h, w = mask.shape
|
||||
y0, y1, x0, x1 = mask_bounds(mask)
|
||||
if bbox_xyxy is not None:
|
||||
try:
|
||||
bx0, by0, bx1, by1 = [float(v) for v in bbox_xyxy]
|
||||
x0 = min(x0, int(np.floor(bx0)))
|
||||
y0 = min(y0, int(np.floor(by0)))
|
||||
x1 = max(x1, int(np.ceil(bx1)))
|
||||
y1 = max(y1, int(np.ceil(by1)))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
box_w = max(1, x1 - x0 + 1)
|
||||
box_h = max(1, y1 - y0 + 1)
|
||||
margin_x = max(1, int(round(box_w * max(0.0, margin_ratio))))
|
||||
margin_y = max(1, int(round(box_h * max(0.0, margin_ratio))))
|
||||
x0 = max(0, x0 - margin_x)
|
||||
x1 = min(w - 1, x1 + margin_x)
|
||||
y0 = max(0, y0 - margin_y)
|
||||
y1 = min(h - 1, y1 + margin_y)
|
||||
region = np.zeros((h, w), dtype=bool)
|
||||
region[y0 : y1 + 1, x0 : x1 + 1] = True
|
||||
return region
|
||||
|
||||
|
||||
def mask_bounds(mask: np.ndarray) -> tuple[int, int, int, int]:
|
||||
ys, xs = np.where(mask)
|
||||
if ys.size == 0:
|
||||
return 0, 0, 0, 0
|
||||
return int(ys.min()), int(ys.max()), int(xs.min()), int(xs.max())
|
||||
|
||||
|
||||
def rgb_saturation(rgb: np.ndarray) -> np.ndarray:
|
||||
clipped = np.clip(rgb[:, :, :3], 0.0, 1.0)
|
||||
maxc = clipped.max(axis=2)
|
||||
minc = clipped.min(axis=2)
|
||||
return (maxc - minc) / np.maximum(maxc, 1e-6)
|
||||
|
||||
|
||||
def connected_components(mask: np.ndarray) -> list[np.ndarray]:
|
||||
h, w = mask.shape
|
||||
visited = np.zeros_like(mask, dtype=bool)
|
||||
components: list[np.ndarray] = []
|
||||
ys, xs = np.where(mask)
|
||||
for sy, sx in zip(ys.tolist(), xs.tolist()):
|
||||
if visited[sy, sx] or not mask[sy, sx]:
|
||||
continue
|
||||
stack = [(sy, sx)]
|
||||
visited[sy, sx] = True
|
||||
pixels = []
|
||||
while stack:
|
||||
y, x = stack.pop()
|
||||
pixels.append((y, x))
|
||||
for ny in range(y - 1, y + 2):
|
||||
for nx in range(x - 1, x + 2):
|
||||
if ny == y and nx == x:
|
||||
continue
|
||||
if 0 <= ny < h and 0 <= nx < w and mask[ny, nx] and not visited[ny, nx]:
|
||||
visited[ny, nx] = True
|
||||
stack.append((ny, nx))
|
||||
component = np.zeros_like(mask, dtype=bool)
|
||||
for y, x in pixels:
|
||||
component[y, x] = True
|
||||
components.append(component)
|
||||
return components
|
||||
|
||||
|
||||
def dilate(mask: np.ndarray, passes: int) -> np.ndarray:
|
||||
out = np.asarray(mask, dtype=bool).copy()
|
||||
for _ in range(max(0, passes)):
|
||||
padded = np.pad(out, 1, mode="constant", constant_values=False)
|
||||
expanded = out.copy()
|
||||
for dy in range(3):
|
||||
for dx in range(3):
|
||||
expanded |= padded[dy : dy + out.shape[0], dx : dx + out.shape[1]]
|
||||
out = expanded
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
225
vendor/gs_playground/scripts/real2sim_external/lama_inpaint.py
vendored
Normal file
225
vendor/gs_playground/scripts/real2sim_external/lama_inpaint.py
vendored
Normal file
@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential LaMa inpainting adapter for Real2Sim.
|
||||
|
||||
Preferred runtime is the lightweight simple-lama-inpainting package, which is
|
||||
listed by the official LaMa repository as a third-party app. This adapter keeps
|
||||
the Real2Sim contract stable: image + instances.json -> background image.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.commands import format_command
|
||||
from real2sim.contracts import load_instances, write_json
|
||||
from real2sim.image_io import read_gray, read_rgb, write_gray, write_rgb
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run LaMa object removal from Real2Sim instance masks")
|
||||
parser.add_argument("--image", type=Path, required=True)
|
||||
parser.add_argument("--instances", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--dilate", type=int, default=7)
|
||||
parser.add_argument("--prompt", default="object")
|
||||
parser.add_argument("--work-dir", type=Path, default=None)
|
||||
parser.add_argument("--redetect-command", default="")
|
||||
parser.add_argument("--segment-command", default="")
|
||||
parser.add_argument("--redetect-command-env", default="")
|
||||
parser.add_argument("--segment-command-env", default="")
|
||||
parser.add_argument("--require-redetect-loop", action="store_true")
|
||||
parser.add_argument("--max-recovery-passes", type=int, default=1)
|
||||
parser.add_argument("--max-area-growth", type=float, default=0.5)
|
||||
args = parser.parse_args()
|
||||
|
||||
instances = load_instances(args.instances)
|
||||
image = read_rgb(args.image)
|
||||
redetect_command = args.redetect_command or (os.environ.get(args.redetect_command_env, "") if args.redetect_command_env else "")
|
||||
segment_command = args.segment_command or (os.environ.get(args.segment_command_env, "") if args.segment_command_env else "")
|
||||
require_recovery_commands(redetect_command, segment_command, strict=args.require_redetect_loop)
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
from simple_lama_inpainting import SimpleLama
|
||||
except Exception as exc:
|
||||
raise SystemExit(
|
||||
"simple-lama-inpainting is not installed. Install it in the external "
|
||||
"model environment or use the built-in backend."
|
||||
) from exc
|
||||
|
||||
lama = SimpleLama()
|
||||
current = image
|
||||
steps = []
|
||||
recovery_events = []
|
||||
work_dir = args.work_dir or args.output.parent
|
||||
instance_masks = {
|
||||
instance.instance_id: read_gray(instance.mask_path) > 0.5
|
||||
for instance in instances
|
||||
if instance.mask_path and instance.mask_path.exists()
|
||||
}
|
||||
ordered = sorted(
|
||||
[instance for instance in instances if instance.mask_path],
|
||||
key=lambda inst: (-(inst.score if inst.score is not None else 0.0), inst.instance_id),
|
||||
)
|
||||
for instance in ordered:
|
||||
mask = read_gray(instance.mask_path) > 0.5
|
||||
if mask.shape != current.shape[:2]:
|
||||
continue
|
||||
expanded = dilate(mask, args.dilate)
|
||||
pil_image = Image.fromarray((np.clip(current, 0, 1) * 255).astype(np.uint8))
|
||||
pil_mask = Image.fromarray((expanded.astype(np.uint8) * 255), mode="L")
|
||||
result = lama(pil_image, pil_mask)
|
||||
current = np.asarray(result.convert("RGB"), dtype=np.float32) / 255.0
|
||||
if redetect_command and segment_command:
|
||||
recovery_events.extend(
|
||||
recover_occluded_regions(
|
||||
current,
|
||||
instances=instances,
|
||||
instance_masks=instance_masks,
|
||||
instances_json=args.instances,
|
||||
prompt=args.prompt,
|
||||
work_dir=work_dir,
|
||||
pass_index=len(steps),
|
||||
redetect_command=redetect_command,
|
||||
segment_command=segment_command,
|
||||
max_area_growth=args.max_area_growth,
|
||||
)
|
||||
)
|
||||
steps.append(
|
||||
{
|
||||
"instance_id": instance.instance_id,
|
||||
"label": instance.label,
|
||||
"mask_pixels": int(mask.sum()),
|
||||
"expanded_mask_pixels": int(expanded.sum()),
|
||||
}
|
||||
)
|
||||
write_rgb(args.output, current)
|
||||
write_json(
|
||||
args.output.with_suffix(args.output.suffix + ".json"),
|
||||
{
|
||||
"backend": "simple_lama_sequential",
|
||||
"strategy": "one_object_at_a_time_mask_expansion",
|
||||
"steps": steps,
|
||||
"recovery_events": recovery_events,
|
||||
"redetect_loop": bool(redetect_command and segment_command),
|
||||
},
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def require_recovery_commands(redetect_command: str, segment_command: str, *, strict: bool) -> None:
|
||||
if strict and (not redetect_command or not segment_command):
|
||||
raise SystemExit(
|
||||
"paper-strict sequential inpainting requires both a re-detection command "
|
||||
"and a segmentation command for newly exposed region recovery"
|
||||
)
|
||||
|
||||
|
||||
def recover_occluded_regions(
|
||||
image: np.ndarray,
|
||||
*,
|
||||
instances,
|
||||
instance_masks: dict[str, np.ndarray],
|
||||
instances_json: Path,
|
||||
prompt: str,
|
||||
work_dir: Path,
|
||||
pass_index: int,
|
||||
redetect_command,
|
||||
segment_command,
|
||||
max_area_growth: float,
|
||||
) -> list[dict]:
|
||||
recovery_dir = work_dir / "03_inpainting" / "recovery" / f"pass_{pass_index:03d}"
|
||||
recovery_dir.mkdir(parents=True, exist_ok=True)
|
||||
image_path = recovery_dir / "image.ppm"
|
||||
detections_json = recovery_dir / "detections.json"
|
||||
masks_dir = recovery_dir / "masks"
|
||||
recovered_instances_json = recovery_dir / "instances.json"
|
||||
write_rgb(image_path, image)
|
||||
|
||||
base_context = {
|
||||
"python": sys.executable,
|
||||
"image": image_path,
|
||||
"input": image_path,
|
||||
"prompt": prompt,
|
||||
"detections_json": detections_json,
|
||||
"instances_json": recovered_instances_json,
|
||||
"masks_dir": masks_dir,
|
||||
}
|
||||
run_nested_command(redetect_command, base_context)
|
||||
run_nested_command(segment_command, base_context)
|
||||
|
||||
existing = {instance.instance_id: instance for instance in instances}
|
||||
events = []
|
||||
for candidate in load_instances(recovered_instances_json):
|
||||
if candidate.mask_path is None or not candidate.mask_path.exists():
|
||||
continue
|
||||
candidate_mask = read_gray(candidate.mask_path) > 0.5
|
||||
for instance in instances:
|
||||
if instance.label != candidate.label or instance.mask_path is None:
|
||||
continue
|
||||
base_mask = instance_masks.get(instance.instance_id)
|
||||
if base_mask is None:
|
||||
continue
|
||||
merged = merge_recovered_mask(base_mask, candidate_mask, same_label=True, max_area_growth=max_area_growth)
|
||||
if int(merged.sum()) == int(base_mask.sum()):
|
||||
continue
|
||||
write_gray(instance.mask_path, merged.astype(np.float32))
|
||||
instance_masks[instance.instance_id] = merged
|
||||
events.append(
|
||||
{
|
||||
"target_instance_id": instance.instance_id,
|
||||
"candidate_instance_id": candidate.instance_id,
|
||||
"label": instance.label,
|
||||
"old_mask_pixels": int(base_mask.sum()),
|
||||
"new_mask_pixels": int(merged.sum()),
|
||||
}
|
||||
)
|
||||
break
|
||||
return events
|
||||
|
||||
|
||||
def run_nested_command(command, context: dict) -> None:
|
||||
argv = format_command(command, context)
|
||||
completed = subprocess.run(argv, cwd=REPO_ROOT, check=False)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"nested recovery command failed with exit code {completed.returncode}: {argv}")
|
||||
|
||||
|
||||
def merge_recovered_mask(base: np.ndarray, candidate: np.ndarray, *, same_label: bool, max_area_growth: float) -> np.ndarray:
|
||||
if not same_label or base.shape != candidate.shape:
|
||||
return base
|
||||
new_region = candidate & ~base
|
||||
if not new_region.any():
|
||||
return base
|
||||
if int(new_region.sum()) > int(base.sum()) * max_area_growth:
|
||||
return base
|
||||
if not (dilate(base, 1) & new_region).any():
|
||||
return base
|
||||
return base | new_region
|
||||
|
||||
|
||||
def dilate(mask: np.ndarray, passes: int) -> np.ndarray:
|
||||
out = mask.copy()
|
||||
for _ in range(max(0, passes)):
|
||||
padded = np.pad(out, 1, mode="constant", constant_values=False)
|
||||
out = (
|
||||
padded[1:-1, 1:-1]
|
||||
| padded[:-2, 1:-1]
|
||||
| padded[2:, 1:-1]
|
||||
| padded[1:-1, :-2]
|
||||
| padded[1:-1, 2:]
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
129
vendor/gs_playground/scripts/real2sim_external/sam2_segment.py
vendored
Normal file
129
vendor/gs_playground/scripts/real2sim_external/sam2_segment.py
vendored
Normal file
@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SAM2 segmentation adapter for Real2Sim.
|
||||
|
||||
Consumes detections.json boxes and writes instances.json plus per-instance PGM
|
||||
masks. This uses Meta's official SAM2 image predictor API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.contracts import read_json, write_json
|
||||
from real2sim.image_io import write_gray
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run SAM2 masks from GroundingDINO boxes")
|
||||
parser.add_argument("--image", type=Path, required=True)
|
||||
parser.add_argument("--detections", type=Path, required=True)
|
||||
parser.add_argument("--masks-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--checkpoint", default=os.environ.get("REAL2SIM_SAM2_CHECKPOINT", ""))
|
||||
parser.add_argument("--model-cfg", default=os.environ.get("REAL2SIM_SAM2_MODEL_CFG", "configs/sam2.1/sam2.1_hiera_l.yaml"))
|
||||
parser.add_argument("--device", default="auto", choices=["auto", "cuda", "cpu"])
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.checkpoint:
|
||||
raise SystemExit("SAM2 checkpoint is required via --checkpoint or REAL2SIM_SAM2_CHECKPOINT")
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from sam2.build_sam import build_sam2
|
||||
from sam2.sam2_image_predictor import SAM2ImagePredictor
|
||||
|
||||
detections_raw = read_json(args.detections)
|
||||
detections = detections_raw.get("detections", detections_raw) if isinstance(detections_raw, dict) else detections_raw
|
||||
if not detections:
|
||||
write_json(args.output, {"instances": [], "metadata": {"backend": "sam2"}})
|
||||
return 0
|
||||
|
||||
boxes = np.array([entry.get("bbox_xyxy", entry.get("bbox")) for entry in detections], dtype=np.float32)
|
||||
device = "cuda" if args.device == "auto" and torch.cuda.is_available() else args.device
|
||||
if device == "auto":
|
||||
device = "cpu"
|
||||
|
||||
image = np.array(Image.open(args.image).convert("RGB"))
|
||||
model = build_sam2(args.model_cfg, args.checkpoint, device=device)
|
||||
predictor = SAM2ImagePredictor(model)
|
||||
args.masks_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with torch.inference_mode():
|
||||
if device == "cuda":
|
||||
with torch.autocast("cuda", dtype=torch.bfloat16):
|
||||
masks, scores, _ = predict_masks(predictor, image, boxes)
|
||||
else:
|
||||
masks, scores, _ = predict_masks(predictor, image, boxes)
|
||||
|
||||
masks = normalize_masks(masks)
|
||||
instances = []
|
||||
for idx, (detection, mask) in enumerate(zip(detections, masks)):
|
||||
instance_id = str(detection.get("id", f"det_{idx:03d}"))
|
||||
mask_path = args.masks_dir / f"{safe_name(instance_id)}.pgm"
|
||||
write_gray(mask_path, mask.astype(np.float32))
|
||||
segmentation_score = None
|
||||
if scores is not None and idx < len(scores):
|
||||
segmentation_score = float(np.asarray(scores[idx]).reshape(-1)[0])
|
||||
instances.append(
|
||||
instance_record(
|
||||
detection,
|
||||
mask_path=mask_path,
|
||||
output_parent=args.output.parent,
|
||||
segmentation_score=segmentation_score,
|
||||
)
|
||||
)
|
||||
|
||||
write_json(args.output, {"instances": instances, "metadata": {"backend": "sam2", "model_cfg": args.model_cfg}})
|
||||
return 0
|
||||
|
||||
|
||||
def predict_masks(predictor, image, boxes):
|
||||
predictor.set_image(image)
|
||||
return predictor.predict(point_coords=None, point_labels=None, box=boxes, multimask_output=False)
|
||||
|
||||
|
||||
def normalize_masks(masks):
|
||||
import numpy as np
|
||||
|
||||
masks = np.asarray(masks)
|
||||
if masks.ndim == 4:
|
||||
masks = masks[:, 0]
|
||||
if masks.ndim == 2:
|
||||
masks = masks[None]
|
||||
return masks > 0
|
||||
|
||||
|
||||
def instance_record(detection: dict, *, mask_path: Path, output_parent: Path, segmentation_score: float | None) -> dict:
|
||||
instance_id = str(detection.get("id", "instance"))
|
||||
detection_score = detection.get("score")
|
||||
score = float(detection_score) if detection_score is not None else segmentation_score
|
||||
record = {
|
||||
"id": instance_id,
|
||||
"label": str(detection.get("label", instance_id)),
|
||||
"score": score,
|
||||
"bbox_xyxy": detection.get("bbox_xyxy", detection.get("bbox")),
|
||||
"mask_path": mask_path.relative_to(output_parent).as_posix(),
|
||||
}
|
||||
if detection_score is not None:
|
||||
record["detection_score"] = float(detection_score)
|
||||
if segmentation_score is not None:
|
||||
record["segmentation_score"] = float(segmentation_score)
|
||||
for key in ("prompt_index", "prompt_query"):
|
||||
if key in detection:
|
||||
record[key] = detection[key]
|
||||
return record
|
||||
|
||||
|
||||
def safe_name(value: str) -> str:
|
||||
return "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in value).strip("_") or "instance"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
897
vendor/gs_playground/scripts/real2sim_external/sam3d_objects_reconstruct.py
vendored
Normal file
897
vendor/gs_playground/scripts/real2sim_external/sam3d_objects_reconstruct.py
vendored
Normal file
@ -0,0 +1,897 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SAM 3D Objects adapter for one Real2Sim object.
|
||||
|
||||
This follows Meta's public quick-start shape:
|
||||
from inference import Inference, load_image, load_single_mask
|
||||
output = inference(image, mask)
|
||||
output["gs"].save_ply("splat.ply")
|
||||
|
||||
The script intentionally fails with a clear message if the gated checkpoints
|
||||
or SAM 3D environment are not available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.contracts import Pose, read_json, write_json
|
||||
from real2sim.image_io import read_gray, write_gray
|
||||
|
||||
|
||||
class PreparedSam3DInputs:
|
||||
def __init__(self, image_path: Path, mask_path: Path, mask: np.ndarray, image_size: tuple[int, int], metadata: dict):
|
||||
self.image_path = image_path
|
||||
self.mask_path = mask_path
|
||||
self.mask = mask
|
||||
self.image_size = image_size
|
||||
self.metadata = metadata
|
||||
|
||||
|
||||
class PlyVertexTable:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
header_lines: list[str],
|
||||
body_start: int,
|
||||
fmt: str,
|
||||
vertex_count: int,
|
||||
property_names: list[str],
|
||||
property_types: list[str],
|
||||
array: np.ndarray,
|
||||
):
|
||||
self.header_lines = header_lines
|
||||
self.body_start = body_start
|
||||
self.fmt = fmt
|
||||
self.vertex_count = vertex_count
|
||||
self.property_names = property_names
|
||||
self.property_types = property_types
|
||||
self.array = array
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run SAM 3D Objects for one detected object")
|
||||
parser.add_argument("--image", type=Path, required=True)
|
||||
parser.add_argument("--mask", type=Path, required=True)
|
||||
parser.add_argument("--instance", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--ply", type=Path, required=True)
|
||||
parser.add_argument("--mesh", type=Path, required=True)
|
||||
parser.add_argument("--pose", type=Path, required=True)
|
||||
parser.add_argument("--cameras", type=Path, default=None, help="AnySplat cameras JSON used to match image preprocessing")
|
||||
parser.add_argument("--match-camera-preprocess", action="store_true", help="Run SAM-3D on the same aspect-preserving center crop used by AnySplat")
|
||||
parser.add_argument("--object-centric-preprocess", action="store_true", help="Crop around the instance mask before SAM-3D preprocessing")
|
||||
parser.add_argument("--object-preprocess-size", type=int, default=0, help="Override SAM-3D object input size with a square image")
|
||||
parser.add_argument("--object-crop-margin", type=float, default=0.12, help="Relative padding around the instance mask for object-centric preprocessing")
|
||||
parser.add_argument("--remove-video-overlay", action="store_true", help="Inpaint a central video play overlay before SAM-3D inference")
|
||||
parser.add_argument("--canonicalize-gaussian-to-mesh", action="store_true", help="Rotate SAM-3D Gaussian PLY into the mesh canonical frame")
|
||||
parser.add_argument("--texture-baking", action="store_true", help="Use SAM-3D's textured GLB postprocess path for the exported preview mesh")
|
||||
parser.add_argument("--texture-render-backend", choices=("auto", "inria", "gsplat"), default="gsplat")
|
||||
parser.add_argument("--texture-baking-engine", choices=("nvdiffrast", "pytorch3d"), default="pytorch3d")
|
||||
parser.add_argument("--sam3d-root", type=Path, default=os.environ.get("REAL2SIM_SAM3D_ROOT"))
|
||||
parser.add_argument("--config", type=Path, default=os.environ.get("REAL2SIM_SAM3D_CONFIG"))
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--allow-fallback-mesh", action="store_true", help="Write a collision proxy if SAM 3D returns no exportable mesh")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.sam3d_root is None:
|
||||
raise SystemExit("SAM 3D root is required via --sam3d-root or REAL2SIM_SAM3D_ROOT")
|
||||
if args.config is None:
|
||||
args.config = args.sam3d_root / "checkpoints" / "hf" / "pipeline.yaml"
|
||||
if not args.config.exists():
|
||||
raise SystemExit(f"SAM 3D config/checkpoints are not available: {args.config}")
|
||||
|
||||
sys.path.insert(0, (args.sam3d_root / "notebook").as_posix())
|
||||
try:
|
||||
from inference import Inference, load_image
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"failed to import SAM 3D inference from {args.sam3d_root}: {exc}") from exc
|
||||
|
||||
import torch
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
image_path = args.image
|
||||
mask_np = (read_gray(args.mask) > 0.5).astype(np.uint8)
|
||||
preprocess_metadata = None
|
||||
overlay_metadata = None
|
||||
if args.match_camera_preprocess:
|
||||
if args.cameras is None:
|
||||
raise SystemExit("--match-camera-preprocess requires --cameras")
|
||||
target_shape = None
|
||||
if args.object_preprocess_size > 0:
|
||||
target_shape = (int(args.object_preprocess_size), int(args.object_preprocess_size))
|
||||
prepared = prepare_camera_preprocessed_inputs(
|
||||
args.image,
|
||||
args.mask,
|
||||
args.cameras,
|
||||
args.output_dir / "camera_preprocess",
|
||||
object_centric=args.object_centric_preprocess,
|
||||
target_shape=target_shape,
|
||||
object_margin_ratio=args.object_crop_margin,
|
||||
)
|
||||
image_path = prepared.image_path
|
||||
mask_np = prepared.mask.astype(np.uint8)
|
||||
preprocess_metadata = prepared.metadata
|
||||
if args.remove_video_overlay:
|
||||
image_path, overlay_metadata = remove_center_video_overlay(image_path, mask_np > 0, args.output_dir / "video_overlay_removal")
|
||||
image = load_image(image_path.as_posix())
|
||||
inference = Inference(args.config.as_posix(), compile=False)
|
||||
with torch.inference_mode():
|
||||
output = run_sam3d_inference(
|
||||
inference,
|
||||
image,
|
||||
mask_np,
|
||||
seed=args.seed,
|
||||
texture_baking=args.texture_baking,
|
||||
texture_render_backend=args.texture_render_backend,
|
||||
texture_baking_engine=args.texture_baking_engine,
|
||||
)
|
||||
|
||||
save_gaussian(output, args.ply)
|
||||
|
||||
wrote_mesh = export_mesh(output, args.mesh, prefer_textured=args.texture_baking)
|
||||
if not wrote_mesh and args.allow_fallback_mesh:
|
||||
write_bbox_mesh_from_mask(args.mesh, mask_np)
|
||||
wrote_mesh = True
|
||||
if not wrote_mesh:
|
||||
raise SystemExit(
|
||||
"SAM 3D output did not include an exportable mesh. "
|
||||
"Use --allow-fallback-mesh only for collision-proxy debugging. "
|
||||
f"Output keys: {sorted(output.keys())}"
|
||||
)
|
||||
canonicalization_metadata = None
|
||||
if args.canonicalize_gaussian_to_mesh:
|
||||
canonicalization_metadata = canonicalize_gaussian_to_mesh_frame(args.ply, args.mesh)
|
||||
|
||||
pose = Pose()
|
||||
layout = output.get("layout") or output.get("pose") or output
|
||||
if isinstance(layout, dict):
|
||||
pose = pose_from_layout(layout)
|
||||
write_json(args.pose, pose.to_json())
|
||||
|
||||
metadata = {
|
||||
"backend": "sam3d_objects",
|
||||
"source_instance": read_json(args.instance),
|
||||
"sam3d_config": args.config.as_posix(),
|
||||
"texture_baking": bool(args.texture_baking),
|
||||
"texture_render_backend": args.texture_render_backend,
|
||||
"texture_baking_engine": args.texture_baking_engine,
|
||||
}
|
||||
if preprocess_metadata is not None:
|
||||
metadata["camera_preprocess"] = preprocess_metadata
|
||||
if overlay_metadata is not None:
|
||||
metadata["video_overlay_removal"] = overlay_metadata
|
||||
if canonicalization_metadata is not None:
|
||||
metadata["gaussian_canonicalization"] = canonicalization_metadata
|
||||
write_json(args.output_dir / "sam3d_metadata.json", metadata)
|
||||
return 0
|
||||
|
||||
|
||||
def prepare_camera_preprocessed_inputs(
|
||||
image_path: Path,
|
||||
mask_path: Path,
|
||||
cameras_path: Path,
|
||||
output_dir: Path,
|
||||
*,
|
||||
object_centric: bool = False,
|
||||
target_shape: tuple[int, int] | None = None,
|
||||
object_margin_ratio: float = 0.12,
|
||||
) -> PreparedSam3DInputs:
|
||||
cameras = read_json(cameras_path)
|
||||
target_h, target_w = target_shape or camera_image_shape(cameras)
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
mask = read_gray(mask_path) > 0.5
|
||||
if object_centric and mask.any():
|
||||
processed_image, processed_mask, crop_meta = object_centric_resize_pad_image_and_mask(
|
||||
image,
|
||||
mask,
|
||||
(target_h, target_w),
|
||||
margin_ratio=object_margin_ratio,
|
||||
resample=Image.BICUBIC,
|
||||
)
|
||||
else:
|
||||
processed_image, crop_meta = cover_resize_center_crop_image(image, (target_h, target_w), resample=Image.BICUBIC)
|
||||
processed_mask = cover_resize_center_crop_mask(mask, (target_h, target_w), crop_meta)
|
||||
crop_meta = {"preprocess_mode": "camera_center_crop", **crop_meta}
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
processed_image_path = output_dir / "sam3d_camera_image.png"
|
||||
processed_mask_path = output_dir / "sam3d_camera_mask.pgm"
|
||||
processed_image.save(processed_image_path)
|
||||
write_gray(processed_mask_path, processed_mask.astype(np.float32))
|
||||
|
||||
metadata = {
|
||||
"source_image": image_path.as_posix(),
|
||||
"source_mask": mask_path.as_posix(),
|
||||
"cameras": cameras_path.as_posix(),
|
||||
"source_size": [int(image.height), int(image.width)],
|
||||
"target_size": [int(target_h), int(target_w)],
|
||||
**crop_meta,
|
||||
"image": processed_image_path.as_posix(),
|
||||
"mask": processed_mask_path.as_posix(),
|
||||
"mask_pixels": int(processed_mask.sum()),
|
||||
}
|
||||
return PreparedSam3DInputs(
|
||||
image_path=processed_image_path,
|
||||
mask_path=processed_mask_path,
|
||||
mask=processed_mask,
|
||||
image_size=(target_h, target_w),
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
def camera_image_shape(cameras: dict) -> tuple[int, int]:
|
||||
shape = cameras.get("image_shape") if isinstance(cameras, dict) else None
|
||||
if isinstance(shape, (list, tuple)) and len(shape) >= 2:
|
||||
return (int(shape[0]), int(shape[1]))
|
||||
intrinsics = cameras.get("intrinsics", {}) if isinstance(cameras, dict) else {}
|
||||
height = int(intrinsics.get("height", 448))
|
||||
width = int(intrinsics.get("width", 448))
|
||||
return (height, width)
|
||||
|
||||
|
||||
def cover_resize_center_crop_image(image, shape: tuple[int, int], *, resample) -> tuple:
|
||||
dst_h, dst_w = shape
|
||||
src_w, src_h = image.size
|
||||
scale = max(dst_w / float(src_w), dst_h / float(src_h))
|
||||
resized_w = max(dst_w, int(src_w * scale))
|
||||
resized_h = max(dst_h, int(src_h * scale))
|
||||
resized = image.resize((resized_w, resized_h), resample)
|
||||
left = max(0, (resized_w - dst_w) // 2)
|
||||
top = max(0, (resized_h - dst_h) // 2)
|
||||
crop = resized.crop((left, top, left + dst_w, top + dst_h))
|
||||
return crop, {
|
||||
"resized_size": [int(resized_h), int(resized_w)],
|
||||
"crop_xyxy": [int(left), int(top), int(left + dst_w), int(top + dst_h)],
|
||||
"scale": float(scale),
|
||||
}
|
||||
|
||||
|
||||
def cover_resize_center_crop_mask(mask: np.ndarray, shape: tuple[int, int], crop_meta: dict) -> np.ndarray:
|
||||
from PIL import Image
|
||||
|
||||
dst_h, dst_w = shape
|
||||
resized_h, resized_w = crop_meta["resized_size"]
|
||||
left, top, right, bottom = crop_meta["crop_xyxy"]
|
||||
mask_img = Image.fromarray(mask.astype(np.uint8) * 255)
|
||||
resized = mask_img.resize((int(resized_w), int(resized_h)), Image.NEAREST)
|
||||
cropped = resized.crop((int(left), int(top), int(right), int(bottom)))
|
||||
arr = np.asarray(cropped, dtype=np.uint8) > 127
|
||||
if arr.shape == (dst_h, dst_w):
|
||||
return arr
|
||||
out = np.zeros((dst_h, dst_w), dtype=bool)
|
||||
out[: arr.shape[0], : arr.shape[1]] = arr
|
||||
return out
|
||||
|
||||
|
||||
def object_centric_resize_pad_image_and_mask(image, mask: np.ndarray, shape: tuple[int, int], *, margin_ratio: float, resample) -> tuple:
|
||||
from PIL import Image
|
||||
|
||||
dst_h, dst_w = shape
|
||||
src_w, src_h = image.size
|
||||
x0, y0, x1, y1 = mask_bbox_xyxy(mask)
|
||||
box_w = max(1, x1 - x0)
|
||||
box_h = max(1, y1 - y0)
|
||||
margin = max(1, int(round(max(box_w, box_h) * max(float(margin_ratio), 0.0))))
|
||||
crop_x0 = max(0, x0 - margin)
|
||||
crop_y0 = max(0, y0 - margin)
|
||||
crop_x1 = min(src_w, x1 + margin)
|
||||
crop_y1 = min(src_h, y1 + margin)
|
||||
if crop_x1 <= crop_x0 or crop_y1 <= crop_y0:
|
||||
processed_image, crop_meta = cover_resize_center_crop_image(image, shape, resample=resample)
|
||||
processed_mask = cover_resize_center_crop_mask(mask, shape, crop_meta)
|
||||
return processed_image, processed_mask, {"preprocess_mode": "camera_center_crop_empty_object", **crop_meta}
|
||||
|
||||
crop = image.crop((crop_x0, crop_y0, crop_x1, crop_y1))
|
||||
crop_mask = mask[crop_y0:crop_y1, crop_x0:crop_x1]
|
||||
crop_w, crop_h = crop.size
|
||||
scale = min(dst_w / float(crop_w), dst_h / float(crop_h))
|
||||
resized_w = max(1, min(dst_w, int(round(crop_w * scale))))
|
||||
resized_h = max(1, min(dst_h, int(round(crop_h * scale))))
|
||||
offset_x = (dst_w - resized_w) // 2
|
||||
offset_y = (dst_h - resized_h) // 2
|
||||
|
||||
fill = local_background_color(crop, crop_mask)
|
||||
canvas = Image.new("RGB", (dst_w, dst_h), fill)
|
||||
resized = crop.resize((resized_w, resized_h), resample)
|
||||
canvas.paste(resized, (offset_x, offset_y))
|
||||
|
||||
mask_canvas = np.zeros((dst_h, dst_w), dtype=bool)
|
||||
resized_mask = Image.fromarray(crop_mask.astype(np.uint8) * 255).resize((resized_w, resized_h), Image.NEAREST)
|
||||
mask_arr = np.asarray(resized_mask, dtype=np.uint8) > 127
|
||||
mask_canvas[offset_y : offset_y + resized_h, offset_x : offset_x + resized_w] = mask_arr
|
||||
return canvas, mask_canvas, {
|
||||
"preprocess_mode": "object_centric",
|
||||
"object_bbox_xyxy": [int(x0), int(y0), int(x1), int(y1)],
|
||||
"source_crop_xyxy": [int(crop_x0), int(crop_y0), int(crop_x1), int(crop_y1)],
|
||||
"resized_size": [int(resized_h), int(resized_w)],
|
||||
"pad_xy": [int(offset_x), int(offset_y)],
|
||||
"scale": float(scale),
|
||||
"object_margin_ratio": float(margin_ratio),
|
||||
}
|
||||
|
||||
|
||||
def mask_bbox_xyxy(mask: np.ndarray) -> tuple[int, int, int, int]:
|
||||
ys, xs = np.where(np.asarray(mask, dtype=bool))
|
||||
if len(xs) == 0:
|
||||
return (0, 0, int(mask.shape[1]), int(mask.shape[0]))
|
||||
return (int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1)
|
||||
|
||||
|
||||
def local_background_color(image, mask: np.ndarray) -> tuple[int, int, int]:
|
||||
arr = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
bg = arr[~np.asarray(mask, dtype=bool)]
|
||||
values = bg if bg.size else arr.reshape(-1, 3)
|
||||
color = np.median(values.reshape(-1, 3), axis=0)
|
||||
return tuple(int(np.clip(round(float(channel)), 0, 255)) for channel in color)
|
||||
|
||||
|
||||
def remove_center_video_overlay(image_path: Path, object_mask: np.ndarray, output_dir: Path) -> tuple[Path, dict]:
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
arr = np.asarray(image, dtype=np.uint8)
|
||||
mask = np.asarray(object_mask, dtype=bool)
|
||||
if mask.shape != arr.shape[:2]:
|
||||
mask = resize_bool_mask(mask, arr.shape[:2])
|
||||
|
||||
overlay_mask = detect_center_video_overlay(arr, mask)
|
||||
overlay_pixels = int(overlay_mask.sum())
|
||||
metadata = {
|
||||
"source_image": image_path.as_posix(),
|
||||
"detected": overlay_pixels > 0,
|
||||
"overlay_pixels": overlay_pixels,
|
||||
}
|
||||
if overlay_pixels == 0:
|
||||
return image_path, metadata
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_dir / "sam3d_overlay_removed.png"
|
||||
try:
|
||||
import cv2
|
||||
|
||||
inpaint_mask = (overlay_mask.astype(np.uint8) * 255)
|
||||
cleaned = cv2.inpaint(arr, inpaint_mask, 5, cv2.INPAINT_TELEA)
|
||||
except Exception:
|
||||
cleaned = simple_mask_fill(arr, overlay_mask)
|
||||
metadata["fallback"] = "simple_mask_fill"
|
||||
Image.fromarray(cleaned.astype(np.uint8)).save(output_path)
|
||||
metadata["image"] = output_path.as_posix()
|
||||
return output_path, metadata
|
||||
|
||||
|
||||
def detect_center_video_overlay(image: np.ndarray, object_mask: np.ndarray) -> np.ndarray:
|
||||
height, width = image.shape[:2]
|
||||
yy, xx = np.ogrid[:height, :width]
|
||||
cx = width * 0.5
|
||||
cy = height * 0.5
|
||||
radius = max(8.0, min(height, width) * 0.11)
|
||||
disk = (xx - cx) ** 2 + (yy - cy) ** 2 <= radius ** 2
|
||||
rgb = image.astype(np.float32) / 255.0
|
||||
max_channel = rgb.max(axis=2)
|
||||
min_channel = rgb.min(axis=2)
|
||||
saturation = (max_channel - min_channel) / np.maximum(max_channel, 1e-6)
|
||||
bright_neutral = (max_channel > 0.72) & (saturation < 0.28)
|
||||
raw = disk & object_mask & bright_neutral
|
||||
if int(raw.sum()) < max(20, int(radius)):
|
||||
return np.zeros((height, width), dtype=bool)
|
||||
return dilate_bool_mask(raw, radius=3)
|
||||
|
||||
|
||||
def resize_bool_mask(mask: np.ndarray, shape: tuple[int, int]) -> np.ndarray:
|
||||
from PIL import Image
|
||||
|
||||
height, width = shape
|
||||
resized = Image.fromarray(mask.astype(np.uint8) * 255).resize((width, height), Image.NEAREST)
|
||||
return np.asarray(resized, dtype=np.uint8) > 127
|
||||
|
||||
|
||||
def dilate_bool_mask(mask: np.ndarray, *, radius: int) -> np.ndarray:
|
||||
try:
|
||||
import cv2
|
||||
|
||||
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius * 2 + 1, radius * 2 + 1))
|
||||
return cv2.dilate(mask.astype(np.uint8), kernel, iterations=1) > 0
|
||||
except Exception:
|
||||
padded = np.pad(mask, radius, mode="constant", constant_values=False)
|
||||
out = np.zeros_like(mask, dtype=bool)
|
||||
for dy in range(radius * 2 + 1):
|
||||
for dx in range(radius * 2 + 1):
|
||||
if (dx - radius) ** 2 + (dy - radius) ** 2 <= radius ** 2:
|
||||
out |= padded[dy : dy + mask.shape[0], dx : dx + mask.shape[1]]
|
||||
return out
|
||||
|
||||
|
||||
def simple_mask_fill(image: np.ndarray, mask: np.ndarray) -> np.ndarray:
|
||||
cleaned = image.copy()
|
||||
if not mask.any():
|
||||
return cleaned
|
||||
yy, xx = np.where(mask)
|
||||
y0, y1 = max(0, int(yy.min()) - 5), min(image.shape[0], int(yy.max()) + 6)
|
||||
x0, x1 = max(0, int(xx.min()) - 5), min(image.shape[1], int(xx.max()) + 6)
|
||||
patch = image[y0:y1, x0:x1]
|
||||
patch_mask = mask[y0:y1, x0:x1]
|
||||
fill_values = patch[~patch_mask]
|
||||
fill = np.median(fill_values, axis=0) if fill_values.size else np.median(image.reshape(-1, 3), axis=0)
|
||||
cleaned[mask] = fill.astype(np.uint8)
|
||||
return cleaned
|
||||
|
||||
|
||||
PLY_NUMPY_DTYPES = {
|
||||
"char": "i1",
|
||||
"int8": "i1",
|
||||
"uchar": "u1",
|
||||
"uint8": "u1",
|
||||
"short": "i2",
|
||||
"int16": "i2",
|
||||
"ushort": "u2",
|
||||
"uint16": "u2",
|
||||
"int": "i4",
|
||||
"int32": "i4",
|
||||
"uint": "u4",
|
||||
"uint32": "u4",
|
||||
"float": "f4",
|
||||
"float32": "f4",
|
||||
"double": "f8",
|
||||
"float64": "f8",
|
||||
}
|
||||
|
||||
|
||||
def canonicalize_gaussian_to_mesh_frame(gaussian_ply: Path, mesh_obj: Path) -> dict:
|
||||
table = load_ply_vertex_table(gaussian_ply)
|
||||
mesh_vertices = load_obj_vertices(mesh_obj)
|
||||
if table.array.size == 0 or mesh_vertices.size == 0:
|
||||
return {
|
||||
"enabled": True,
|
||||
"status": "skipped_empty_geometry",
|
||||
"gaussian": gaussian_ply.as_posix(),
|
||||
"mesh": mesh_obj.as_posix(),
|
||||
}
|
||||
|
||||
xyz_indices = indices_for(table.property_names, ("x", "y", "z"))
|
||||
points = table.array[:, xyz_indices]
|
||||
gaussian_extent = extent(points)
|
||||
mesh_extent = extent(mesh_vertices)
|
||||
rotation, mapping, signs = canonical_axis_rotation(gaussian_extent, mesh_extent)
|
||||
changed = not np.allclose(rotation, np.eye(3), atol=1e-6)
|
||||
if changed:
|
||||
apply_axis_rotation_to_ply_table(table, rotation)
|
||||
write_ply_vertex_table(gaussian_ply, table)
|
||||
|
||||
return {
|
||||
"enabled": True,
|
||||
"status": "ok",
|
||||
"changed": bool(changed),
|
||||
"gaussian": gaussian_ply.as_posix(),
|
||||
"mesh": mesh_obj.as_posix(),
|
||||
"gaussian_extent_before": vector_json(gaussian_extent),
|
||||
"mesh_extent": vector_json(mesh_extent),
|
||||
"axis_mapping": {
|
||||
"source": "extent_permutation",
|
||||
"gaussian_axes_to_mesh_axes": [int(value) for value in mapping],
|
||||
"mesh_axis_signs": [int(value) for value in signs],
|
||||
},
|
||||
"rotation_matrix": [[round(float(value), 7) for value in row] for row in rotation.tolist()],
|
||||
}
|
||||
|
||||
|
||||
def canonical_axis_rotation(gaussian_extent: np.ndarray, mesh_extent: np.ndarray) -> tuple[np.ndarray, list[int], list[int]]:
|
||||
gaussian_order = list(np.argsort(-np.asarray(gaussian_extent, dtype=np.float32)))
|
||||
mesh_order = list(np.argsort(-np.asarray(mesh_extent, dtype=np.float32)))
|
||||
mapping = [0, 1, 2]
|
||||
for mesh_axis, gaussian_axis in zip(mesh_order, gaussian_order):
|
||||
mapping[int(mesh_axis)] = int(gaussian_axis)
|
||||
signs = [1, 1, 1]
|
||||
rotation = rotation_from_axis_mapping(mapping, signs)
|
||||
if np.linalg.det(rotation) < 0.0:
|
||||
signs[int(mesh_order[-1])] = -1
|
||||
rotation = rotation_from_axis_mapping(mapping, signs)
|
||||
return rotation.astype(np.float32), mapping, signs
|
||||
|
||||
|
||||
def rotation_from_axis_mapping(mapping: list[int], signs: list[int]) -> np.ndarray:
|
||||
rotation = np.zeros((3, 3), dtype=np.float32)
|
||||
for mesh_axis, gaussian_axis in enumerate(mapping):
|
||||
rotation[mesh_axis, gaussian_axis] = float(signs[mesh_axis])
|
||||
return rotation
|
||||
|
||||
|
||||
def apply_axis_rotation_to_ply_table(table: PlyVertexTable, rotation: np.ndarray) -> None:
|
||||
names = table.property_names
|
||||
xyz_indices = indices_for(names, ("x", "y", "z"))
|
||||
table.array[:, xyz_indices] = table.array[:, xyz_indices] @ rotation.T
|
||||
if all(name in names for name in ("nx", "ny", "nz")):
|
||||
normal_indices = indices_for(names, ("nx", "ny", "nz"))
|
||||
table.array[:, normal_indices] = table.array[:, normal_indices] @ rotation.T
|
||||
if all(name in names for name in ("rot_0", "rot_1", "rot_2", "rot_3")):
|
||||
rot_indices = indices_for(names, ("rot_0", "rot_1", "rot_2", "rot_3"))
|
||||
transform_quat = quat_wxyz_from_rotation_matrix(rotation)
|
||||
for idx in range(table.array.shape[0]):
|
||||
table.array[idx, rot_indices] = quat_multiply(transform_quat, normalize_quat(table.array[idx, rot_indices]))
|
||||
|
||||
|
||||
def load_ply_vertex_table(path: Path) -> PlyVertexTable:
|
||||
data = path.read_bytes()
|
||||
header_marker = b"end_header"
|
||||
marker_index = data.find(header_marker)
|
||||
if marker_index < 0:
|
||||
raise SystemExit(f"PLY missing end_header: {path}")
|
||||
body_start = data.find(b"\n", marker_index)
|
||||
if body_start < 0:
|
||||
raise SystemExit(f"PLY header missing final newline: {path}")
|
||||
body_start += 1
|
||||
header_lines = data[:body_start].decode("ascii", errors="ignore").splitlines()
|
||||
fmt = "ascii"
|
||||
vertex_count = 0
|
||||
property_names: list[str] = []
|
||||
property_types: list[str] = []
|
||||
current_element = ""
|
||||
for line in header_lines:
|
||||
parts = line.strip().split()
|
||||
if len(parts) >= 3 and parts[0] == "format":
|
||||
fmt = parts[1]
|
||||
elif len(parts) >= 3 and parts[0] == "element":
|
||||
current_element = parts[1]
|
||||
if current_element == "vertex":
|
||||
vertex_count = int(parts[2])
|
||||
elif len(parts) >= 3 and parts[0] == "property" and current_element == "vertex":
|
||||
if parts[1] == "list":
|
||||
raise SystemExit(f"PLY vertex list properties are not supported for canonicalization: {path}")
|
||||
property_types.append(parts[1])
|
||||
property_names.append(parts[-1])
|
||||
if vertex_count <= 0:
|
||||
return PlyVertexTable(
|
||||
header_lines=header_lines,
|
||||
body_start=body_start,
|
||||
fmt=fmt,
|
||||
vertex_count=0,
|
||||
property_names=property_names,
|
||||
property_types=property_types,
|
||||
array=np.zeros((0, len(property_names)), dtype=np.float32),
|
||||
)
|
||||
if not all(name in property_names for name in ("x", "y", "z")):
|
||||
raise SystemExit(f"PLY must expose x/y/z vertex properties: {path}")
|
||||
body = data[body_start:]
|
||||
if fmt == "ascii":
|
||||
array = load_ascii_ply_vertex_array(body, vertex_count, len(property_names))
|
||||
elif fmt in {"binary_little_endian", "binary_big_endian"}:
|
||||
array = load_binary_ply_vertex_array(body, vertex_count, property_types, little_endian=fmt == "binary_little_endian")
|
||||
else:
|
||||
raise SystemExit(f"unsupported PLY format for canonicalization: {fmt}")
|
||||
return PlyVertexTable(
|
||||
header_lines=header_lines,
|
||||
body_start=body_start,
|
||||
fmt=fmt,
|
||||
vertex_count=vertex_count,
|
||||
property_names=property_names,
|
||||
property_types=property_types,
|
||||
array=array.astype(np.float32, copy=False),
|
||||
)
|
||||
|
||||
|
||||
def load_ascii_ply_vertex_array(body: bytes, vertex_count: int, property_count: int) -> np.ndarray:
|
||||
rows = []
|
||||
for line in body.decode("ascii", errors="ignore").splitlines():
|
||||
if len(rows) >= vertex_count:
|
||||
break
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
values = [float(value) for value in stripped.split()[:property_count]]
|
||||
if len(values) == property_count:
|
||||
rows.append(values)
|
||||
return np.asarray(rows, dtype=np.float32)
|
||||
|
||||
|
||||
def load_binary_ply_vertex_array(body: bytes, vertex_count: int, property_types: list[str], *, little_endian: bool) -> np.ndarray:
|
||||
byte_order = "<" if little_endian else ">"
|
||||
try:
|
||||
dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(property_types)])
|
||||
except KeyError as exc:
|
||||
raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc
|
||||
structured = np.frombuffer(body, dtype=dtype, count=vertex_count)
|
||||
columns = [structured[f"f{idx}"].astype(np.float32) for idx in range(len(property_types))]
|
||||
return np.stack(columns, axis=1) if columns else np.zeros((vertex_count, 0), dtype=np.float32)
|
||||
|
||||
|
||||
def write_ply_vertex_table(path: Path, table: PlyVertexTable) -> None:
|
||||
header = "\n".join(table.header_lines) + "\n"
|
||||
if table.fmt == "ascii":
|
||||
lines = [" ".join(f"{float(value):.9g}" for value in row) for row in table.array]
|
||||
path.write_text(header + "\n".join(lines) + "\n", encoding="ascii")
|
||||
return
|
||||
if table.fmt in {"binary_little_endian", "binary_big_endian"}:
|
||||
byte_order = "<" if table.fmt == "binary_little_endian" else ">"
|
||||
try:
|
||||
dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(table.property_types)])
|
||||
except KeyError as exc:
|
||||
raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc
|
||||
structured = np.empty(table.array.shape[0], dtype=dtype)
|
||||
for idx in range(table.array.shape[1]):
|
||||
structured[f"f{idx}"] = table.array[:, idx].astype(dtype[idx])
|
||||
path.write_bytes(header.encode("ascii") + structured.tobytes())
|
||||
return
|
||||
raise SystemExit(f"unsupported PLY format for canonicalization: {table.fmt}")
|
||||
|
||||
|
||||
def load_obj_vertices(path: Path) -> np.ndarray:
|
||||
vertices = []
|
||||
with path.open("r", encoding="utf-8", errors="ignore") as f:
|
||||
for line in f:
|
||||
if not line.startswith("v "):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) >= 4:
|
||||
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
||||
return np.asarray(vertices, dtype=np.float32)
|
||||
|
||||
|
||||
def indices_for(names: list[str], required: tuple[str, ...]) -> list[int]:
|
||||
return [names.index(name) for name in required]
|
||||
|
||||
|
||||
def extent(points: np.ndarray) -> np.ndarray:
|
||||
if points.size == 0:
|
||||
return np.zeros(3, dtype=np.float32)
|
||||
return np.asarray(points.max(axis=0) - points.min(axis=0), dtype=np.float32)
|
||||
|
||||
|
||||
def normalize_quat(raw: np.ndarray) -> np.ndarray:
|
||||
quat = np.asarray(raw, dtype=np.float32).reshape(4)
|
||||
norm = float(np.linalg.norm(quat))
|
||||
if norm <= 1e-8:
|
||||
return np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32)
|
||||
return quat / norm
|
||||
|
||||
|
||||
def quat_multiply(a: np.ndarray, b: np.ndarray) -> np.ndarray:
|
||||
aw, ax, ay, az = normalize_quat(a)
|
||||
bw, bx, by, bz = normalize_quat(b)
|
||||
return normalize_quat(
|
||||
np.asarray(
|
||||
[
|
||||
aw * bw - ax * bx - ay * by - az * bz,
|
||||
aw * bx + ax * bw + ay * bz - az * by,
|
||||
aw * by - ax * bz + ay * bw + az * bx,
|
||||
aw * bz + ax * by - ay * bx + az * bw,
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def quat_wxyz_from_rotation_matrix(rotation: np.ndarray) -> np.ndarray:
|
||||
matrix = np.asarray(rotation, dtype=np.float64).reshape(3, 3)
|
||||
trace = float(np.trace(matrix))
|
||||
if trace > 0.0:
|
||||
s = np.sqrt(trace + 1.0) * 2.0
|
||||
quat = np.asarray(
|
||||
[
|
||||
0.25 * s,
|
||||
(matrix[2, 1] - matrix[1, 2]) / s,
|
||||
(matrix[0, 2] - matrix[2, 0]) / s,
|
||||
(matrix[1, 0] - matrix[0, 1]) / s,
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
else:
|
||||
axis = int(np.argmax(np.diag(matrix)))
|
||||
if axis == 0:
|
||||
s = np.sqrt(1.0 + matrix[0, 0] - matrix[1, 1] - matrix[2, 2]) * 2.0
|
||||
quat = np.asarray(
|
||||
[
|
||||
(matrix[2, 1] - matrix[1, 2]) / s,
|
||||
0.25 * s,
|
||||
(matrix[0, 1] + matrix[1, 0]) / s,
|
||||
(matrix[0, 2] + matrix[2, 0]) / s,
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
elif axis == 1:
|
||||
s = np.sqrt(1.0 + matrix[1, 1] - matrix[0, 0] - matrix[2, 2]) * 2.0
|
||||
quat = np.asarray(
|
||||
[
|
||||
(matrix[0, 2] - matrix[2, 0]) / s,
|
||||
(matrix[0, 1] + matrix[1, 0]) / s,
|
||||
0.25 * s,
|
||||
(matrix[1, 2] + matrix[2, 1]) / s,
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
else:
|
||||
s = np.sqrt(1.0 + matrix[2, 2] - matrix[0, 0] - matrix[1, 1]) * 2.0
|
||||
quat = np.asarray(
|
||||
[
|
||||
(matrix[1, 0] - matrix[0, 1]) / s,
|
||||
(matrix[0, 2] + matrix[2, 0]) / s,
|
||||
(matrix[1, 2] + matrix[2, 1]) / s,
|
||||
0.25 * s,
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
return normalize_quat(quat.astype(np.float32))
|
||||
|
||||
|
||||
def vector_json(values: np.ndarray) -> list[float]:
|
||||
return [round(float(value), 7) for value in np.asarray(values, dtype=np.float32).reshape(-1).tolist()]
|
||||
|
||||
|
||||
def pose_from_layout(layout: dict) -> Pose:
|
||||
position = layout.get("position", layout.get("translation", [0.0, 0.0, 0.0]))
|
||||
scale = layout.get("scale", [1.0, 1.0, 1.0])
|
||||
quat = layout.get("quat_wxyz", layout.get("quaternion_wxyz", layout.get("rotation", [1.0, 0.0, 0.0, 0.0])))
|
||||
if "quaternion_xyzw" in layout and "quat_wxyz" not in layout and "quaternion_wxyz" not in layout and "rotation" not in layout:
|
||||
qx, qy, qz, qw = vector_from_value(layout["quaternion_xyzw"], 4, "quaternion_xyzw")
|
||||
quat = (qw, qx, qy, qz)
|
||||
return Pose(
|
||||
position=vector_from_value(position, 3, "position"),
|
||||
quat_wxyz=vector_from_value(quat, 4, "quat_wxyz"),
|
||||
scale=vector_from_value(scale, 3, "scale", broadcast_scalar=True),
|
||||
)
|
||||
|
||||
|
||||
def run_sam3d_inference(
|
||||
inference,
|
||||
image: np.ndarray,
|
||||
mask_np: np.ndarray,
|
||||
*,
|
||||
seed: int,
|
||||
texture_baking: bool,
|
||||
texture_render_backend: str = "gsplat",
|
||||
texture_baking_engine: str = "pytorch3d",
|
||||
) -> dict:
|
||||
if not texture_baking:
|
||||
return inference(image, mask_np, seed=seed)
|
||||
|
||||
patch_texture_baking_render_backend(texture_render_backend)
|
||||
pipeline = getattr(inference, "_pipeline", None)
|
||||
merge_mask_to_rgba = getattr(inference, "merge_mask_to_rgba", None)
|
||||
if pipeline is None or not hasattr(pipeline, "run") or merge_mask_to_rgba is None:
|
||||
raise SystemExit("--texture-baking requires SAM-3D notebook Inference._pipeline.run support")
|
||||
if hasattr(pipeline, "rendering_engine"):
|
||||
pipeline.rendering_engine = texture_baking_engine
|
||||
|
||||
rgba = merge_mask_to_rgba(image, mask_np)
|
||||
options = {
|
||||
"stage1_only": False,
|
||||
"with_mesh_postprocess": False,
|
||||
"with_texture_baking": True,
|
||||
"with_layout_postprocess": False,
|
||||
"use_vertex_color": False,
|
||||
"stage1_inference_steps": None,
|
||||
"pointmap": None,
|
||||
}
|
||||
return pipeline.run(rgba, None, seed, **supported_kwargs(pipeline.run, options))
|
||||
|
||||
|
||||
def patch_texture_baking_render_backend(backend: str) -> bool:
|
||||
if backend in {"", "auto", "inria"}:
|
||||
return False
|
||||
if backend != "gsplat":
|
||||
raise SystemExit(f"unsupported SAM-3D texture render backend: {backend}")
|
||||
|
||||
render_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.render_utils")
|
||||
postprocessing_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.postprocessing_utils")
|
||||
|
||||
def render_multiview_with_backend(sample, resolution=512, nviews=30):
|
||||
radius = 2
|
||||
fov = 40
|
||||
cameras = [render_utils.sphere_hammersley_sequence(index, nviews) for index in range(nviews)]
|
||||
yaws = [camera[0] for camera in cameras]
|
||||
pitches = [camera[1] for camera in cameras]
|
||||
extrinsics, intrinsics = render_utils.yaw_pitch_r_fov_to_extrinsics_intrinsics(yaws, pitches, radius, fov)
|
||||
result = render_utils.render_frames(
|
||||
sample,
|
||||
extrinsics,
|
||||
intrinsics,
|
||||
{"resolution": resolution, "bg_color": (0, 0, 0), "backend": "gsplat"},
|
||||
)
|
||||
return result["color"], extrinsics, intrinsics
|
||||
|
||||
postprocessing_utils.render_multiview = render_multiview_with_backend
|
||||
return True
|
||||
|
||||
|
||||
def supported_kwargs(callable_obj, options: dict[str, object]) -> dict[str, object]:
|
||||
signature = inspect.signature(callable_obj)
|
||||
if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values()):
|
||||
return options
|
||||
return {key: value for key, value in options.items() if key in signature.parameters}
|
||||
|
||||
|
||||
def save_gaussian(output: dict, path: Path) -> None:
|
||||
for key in ("gs", "gaussian", "gaussians"):
|
||||
if key not in output:
|
||||
continue
|
||||
value = first_value(output[key])
|
||||
if hasattr(value, "save_ply"):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
value.save_ply(path.as_posix())
|
||||
return
|
||||
raise SystemExit(f"SAM 3D output does not expose a Gaussian save_ply method; keys={sorted(output.keys())}")
|
||||
|
||||
|
||||
def export_mesh(output: dict, path: Path, *, prefer_textured: bool = False) -> bool:
|
||||
keys = ("glb", "mesh", "trimesh", "obj") if prefer_textured else ("mesh", "trimesh", "glb", "obj")
|
||||
for key in keys:
|
||||
if key not in output:
|
||||
continue
|
||||
value = first_value(output[key])
|
||||
if hasattr(value, "export"):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
value.export(path.as_posix())
|
||||
return True
|
||||
if isinstance(value, (str, os.PathLike)):
|
||||
source = Path(value)
|
||||
if source.is_file():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(source.read_bytes())
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def first_value(value):
|
||||
if isinstance(value, (list, tuple)):
|
||||
return value[0] if value else value
|
||||
return value
|
||||
|
||||
|
||||
def vector_from_value(value, size: int, field_name: str, *, broadcast_scalar: bool = False) -> tuple[float, ...]:
|
||||
if hasattr(value, "detach"):
|
||||
value = value.detach().cpu().numpy()
|
||||
arr = np.asarray(value, dtype=np.float64).squeeze()
|
||||
if arr.ndim == 0:
|
||||
if broadcast_scalar:
|
||||
return tuple(round(float(arr), 7) for _ in range(size))
|
||||
if size == 1:
|
||||
return (round(float(arr), 7),)
|
||||
flat = arr.reshape(-1)
|
||||
if flat.size != size:
|
||||
raise SystemExit(f"{field_name} must have {size} values, got shape {arr.shape}")
|
||||
return tuple(round(float(x), 7) for x in flat)
|
||||
|
||||
|
||||
def write_bbox_mesh_from_mask(path: Path, mask) -> None:
|
||||
import numpy as np
|
||||
|
||||
ys, xs = np.where(mask)
|
||||
if ys.size == 0:
|
||||
extent = np.array([0.1, 0.1, 0.1], dtype=np.float32)
|
||||
else:
|
||||
extent = np.array([max((xs.max() - xs.min()) / 512.0, 0.05), 0.08, max((ys.max() - ys.min()) / 512.0, 0.05)], dtype=np.float32)
|
||||
half = extent / 2.0
|
||||
vertices = [
|
||||
(-half[0], -half[1], -half[2]),
|
||||
(half[0], -half[1], -half[2]),
|
||||
(half[0], half[1], -half[2]),
|
||||
(-half[0], half[1], -half[2]),
|
||||
(-half[0], -half[1], half[2]),
|
||||
(half[0], -half[1], half[2]),
|
||||
(half[0], half[1], half[2]),
|
||||
(-half[0], half[1], half[2]),
|
||||
]
|
||||
faces = [(1, 2, 3), (1, 3, 4), (5, 8, 7), (5, 7, 6), (1, 5, 6), (1, 6, 2), (2, 6, 7), (2, 7, 3), (3, 7, 8), (3, 8, 4), (4, 8, 5), (4, 5, 1)]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="ascii") as f:
|
||||
f.write("o sam3d_collision_fallback\n")
|
||||
for v in vertices:
|
||||
f.write(f"v {v[0]:.7g} {v[1]:.7g} {v[2]:.7g}\n")
|
||||
for face in faces:
|
||||
f.write(f"f {face[0]} {face[1]} {face[2]}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
296
vendor/gs_playground/scripts/real2sim_external/speedy_splat_prune.py
vendored
Normal file
296
vendor/gs_playground/scripts/real2sim_external/speedy_splat_prune.py
vendored
Normal file
@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Speedy-Splat pruning adapter for Real2Sim 3DGS assets.
|
||||
|
||||
This adapter bridges Real2Sim's exported PLY assets into the official
|
||||
Speedy-Splat pruning implementation. It loads the input PLY with
|
||||
Speedy-Splat's GaussianModel, builds a camera from the AnySplat camera JSON,
|
||||
calls the official train.prune function, and writes the pruned PLY.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Prune a 3DGS PLY with official Speedy-Splat scoring")
|
||||
parser.add_argument("--input", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--image", type=Path, required=True, help="Source image used to build the pruning camera")
|
||||
parser.add_argument("--cameras", type=Path, required=True, help="AnySplat cameras.json artifact")
|
||||
parser.add_argument("--keep-ratio", type=float, default=float(os.environ.get("REAL2SIM_SPEEDY_KEEP_RATIO", "0.85")))
|
||||
parser.add_argument("--keep-count", type=int, default=None)
|
||||
parser.add_argument("--speedy-root", type=Path, default=os.environ.get("REAL2SIM_SPEEDY_SPLAT_ROOT"))
|
||||
parser.add_argument("--background", choices=("black", "white"), default=os.environ.get("REAL2SIM_SPEEDY_BACKGROUND", "black"))
|
||||
parser.add_argument("--metadata", type=Path, default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.speedy_root is None:
|
||||
raise SystemExit("Speedy-Splat root is required via --speedy-root or REAL2SIM_SPEEDY_SPLAT_ROOT")
|
||||
if not args.input.is_file():
|
||||
raise SystemExit(f"input PLY not found: {args.input}")
|
||||
if not args.image.is_file():
|
||||
raise SystemExit(f"source image not found: {args.image}")
|
||||
if not args.cameras.is_file():
|
||||
raise SystemExit(f"camera JSON not found: {args.cameras}")
|
||||
|
||||
sys.path.insert(0, args.speedy_root.as_posix())
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
from plyfile import PlyData
|
||||
from scene.cameras import Camera
|
||||
from scene.gaussian_model import GaussianModel
|
||||
from train import prune as speedy_prune
|
||||
|
||||
props = [prop.name for prop in PlyData.read(args.input)["vertex"].properties]
|
||||
sh_degree = infer_sh_degree(props)
|
||||
gaussians = GaussianModel(sh_degree)
|
||||
gaussians.load_ply(args.input.as_posix())
|
||||
gaussians.spatial_lr_scale = 1.0
|
||||
gaussians.training_setup(speedy_training_args())
|
||||
gaussians.max_radii2D = torch.zeros((gaussians.get_xyz.shape[0],), device="cuda")
|
||||
prune_policy = install_exact_count_pruning(torch, gaussians)
|
||||
|
||||
camera_payload = load_json(args.cameras)
|
||||
intrinsics = require_intrinsics(camera_payload)
|
||||
width = int(intrinsics["width"])
|
||||
height = int(intrinsics["height"])
|
||||
image_tensor = load_image_tensor(Image, torch, args.image, width, height)
|
||||
view_index = image_view_index(camera_payload, args.image)
|
||||
extrinsic = first_matrix(camera_payload.get("pred_context_pose", {}).get("extrinsic"), rows=4, cols=4, view_index=view_index)
|
||||
rotation, translation = speedy_camera_rt(extrinsic)
|
||||
camera = Camera(
|
||||
colmap_id=0,
|
||||
R=rotation,
|
||||
T=translation,
|
||||
FoVx=focal_to_fov(float(intrinsics["fx"]), width),
|
||||
FoVy=focal_to_fov(float(intrinsics["fy"]), height),
|
||||
image=image_tensor,
|
||||
gt_alpha_mask=None,
|
||||
image_name=args.image.stem,
|
||||
uid=0,
|
||||
)
|
||||
|
||||
prune_ratio = prune_ratio_from_keep(args.keep_ratio, args.keep_count, int(gaussians.get_xyz.shape[0]))
|
||||
background = torch.tensor([1.0, 1.0, 1.0] if args.background == "white" else [0.0, 0.0, 0.0], dtype=torch.float32, device="cuda")
|
||||
scene = SingleCameraScene([camera])
|
||||
pipe = SimpleNamespace(debug=False, convert_SHs_python=False, compute_cov3D_python=False)
|
||||
prune_metrics = run_speedy_prune(torch, speedy_prune, scene, gaussians, pipe, background, prune_ratio)
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
gaussians.save_ply(args.output.as_posix())
|
||||
metadata_path = args.metadata or args.output.with_suffix(".speedy_splat.json")
|
||||
write_json(
|
||||
metadata_path,
|
||||
{
|
||||
"backend": "speedy_splat",
|
||||
"speedy_root": args.speedy_root.as_posix(),
|
||||
"input": args.input.as_posix(),
|
||||
"output": args.output.as_posix(),
|
||||
"image": args.image.as_posix(),
|
||||
"cameras": args.cameras.as_posix(),
|
||||
"sh_degree": sh_degree,
|
||||
"keep_ratio": args.keep_ratio,
|
||||
"keep_count": args.keep_count,
|
||||
"camera_view_index": view_index,
|
||||
"prune_ratio": prune_ratio,
|
||||
"input_gaussians": int(len(PlyData.read(args.input)["vertex"])),
|
||||
"output_gaussians": int(gaussians.get_xyz.shape[0]),
|
||||
"prune_metrics": prune_metrics,
|
||||
"prune_policy": prune_policy,
|
||||
},
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
class SingleCameraScene:
|
||||
def __init__(self, cameras: list[Any]) -> None:
|
||||
self._cameras = cameras
|
||||
|
||||
def getTrainCameras(self) -> list[Any]:
|
||||
return self._cameras
|
||||
|
||||
|
||||
def run_speedy_prune(
|
||||
torch_module: Any,
|
||||
speedy_prune: Any,
|
||||
scene: Any,
|
||||
gaussians: Any,
|
||||
pipe: Any,
|
||||
background: Any,
|
||||
prune_ratio: float,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
metrics = speedy_prune(scene, gaussians, pipe, background, prune_ratio)
|
||||
synchronize_cuda(torch_module)
|
||||
return dict(metrics) if isinstance(metrics, dict) else {"status": "ok"}
|
||||
except RuntimeError as exc:
|
||||
trace = traceback.format_exc()
|
||||
if not is_cuda_event_timing_failure(exc, trace):
|
||||
raise
|
||||
synchronize_cuda(torch_module)
|
||||
return {
|
||||
"status": "ok_with_cuda_timing_warning",
|
||||
"timing_warning": first_line(str(exc)),
|
||||
}
|
||||
|
||||
|
||||
def install_exact_count_pruning(torch_module: Any, gaussians: Any) -> str:
|
||||
def prune_gaussians(percent: float, import_score: Any) -> None:
|
||||
prune_gaussians_exact_count(torch_module, gaussians, percent, import_score)
|
||||
|
||||
gaussians.prune_gaussians = prune_gaussians
|
||||
return "speedy_splat_exact_count_tie_safe"
|
||||
|
||||
|
||||
def prune_gaussians_exact_count(torch_module: Any, gaussians: Any, percent: float, import_score: Any) -> None:
|
||||
scores = import_score.reshape(-1)
|
||||
total = int(scores.shape[0])
|
||||
prune_count = exact_prune_count(total, percent)
|
||||
prune_mask = torch_module.zeros((total,), dtype=torch_module.bool, device=scores.device)
|
||||
if prune_count > 0:
|
||||
try:
|
||||
order = torch_module.argsort(scores, stable=True)
|
||||
except TypeError:
|
||||
order = torch_module.argsort(scores)
|
||||
prune_mask[order[:prune_count]] = True
|
||||
gaussians.prune_points(prune_mask)
|
||||
|
||||
|
||||
def exact_prune_count(total: int, percent: float) -> int:
|
||||
if total <= 1:
|
||||
return 0
|
||||
if percent <= 0.0:
|
||||
return 0
|
||||
return min(max(int(round(total * percent)), 0), total - 1)
|
||||
|
||||
|
||||
def is_cuda_event_timing_failure(exc: RuntimeError, trace: str) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "device not ready" in message and "elapsed_time" in trace
|
||||
|
||||
|
||||
def synchronize_cuda(torch_module: Any) -> None:
|
||||
cuda = getattr(torch_module, "cuda", None)
|
||||
synchronize = getattr(cuda, "synchronize", None)
|
||||
if callable(synchronize):
|
||||
synchronize()
|
||||
|
||||
|
||||
def first_line(value: str) -> str:
|
||||
return value.splitlines()[0] if value.splitlines() else value
|
||||
|
||||
|
||||
def speedy_training_args() -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
percent_dense=0.01,
|
||||
position_lr_init=0.00016,
|
||||
position_lr_final=0.0000016,
|
||||
position_lr_delay_mult=0.01,
|
||||
position_lr_max_steps=30_000,
|
||||
feature_lr=0.0025,
|
||||
opacity_lr=0.05,
|
||||
scaling_lr=0.005,
|
||||
rotation_lr=0.001,
|
||||
)
|
||||
|
||||
|
||||
def infer_sh_degree(property_names: list[str]) -> int:
|
||||
f_rest = [name for name in property_names if name.startswith("f_rest_")]
|
||||
coeffs = (len(f_rest) + 3) / 3
|
||||
degree = math.sqrt(coeffs) - 1
|
||||
rounded = int(round(degree))
|
||||
if rounded < 0 or not math.isclose(degree, rounded, rel_tol=0.0, abs_tol=1e-6):
|
||||
raise SystemExit(f"cannot infer spherical harmonics degree from {len(f_rest)} f_rest properties")
|
||||
return rounded
|
||||
|
||||
|
||||
def prune_ratio_from_keep(keep_ratio: float, keep_count: int | None, total: int) -> float:
|
||||
if total <= 0:
|
||||
raise SystemExit("input PLY has no gaussians")
|
||||
if keep_count is not None:
|
||||
if keep_count <= 0:
|
||||
raise SystemExit("--keep-count must be positive")
|
||||
keep_ratio = min(1.0, max(0.0, keep_count / total))
|
||||
if not 0.0 < keep_ratio <= 1.0:
|
||||
raise SystemExit("--keep-ratio must be in (0, 1]")
|
||||
return max(0.0, 1.0 - keep_ratio)
|
||||
|
||||
|
||||
def require_intrinsics(payload: dict[str, Any]) -> dict[str, float]:
|
||||
intrinsics = payload.get("intrinsics")
|
||||
required = ("width", "height", "fx", "fy")
|
||||
if not isinstance(intrinsics, dict) or any(key not in intrinsics for key in required):
|
||||
raise SystemExit(f"camera JSON must contain intrinsics with {', '.join(required)}")
|
||||
return intrinsics
|
||||
|
||||
|
||||
def focal_to_fov(focal: float, pixels: int) -> float:
|
||||
if focal <= 0:
|
||||
raise SystemExit("camera focal length must be positive")
|
||||
return 2.0 * math.atan(float(pixels) / (2.0 * focal))
|
||||
|
||||
|
||||
def image_view_index(payload: dict[str, Any], image_path: Path) -> int:
|
||||
image_paths = payload.get("image_paths")
|
||||
if not isinstance(image_paths, list):
|
||||
return 0
|
||||
target = image_path.expanduser().resolve()
|
||||
for index, raw_path in enumerate(image_paths):
|
||||
candidate = Path(str(raw_path)).expanduser().resolve()
|
||||
if candidate == target:
|
||||
return index
|
||||
return 0
|
||||
|
||||
|
||||
def first_matrix(value: Any, *, rows: int, cols: int, view_index: int = 0) -> np.ndarray:
|
||||
arr = np.asarray(value, dtype=np.float32)
|
||||
while arr.ndim > 2 and arr.shape[0] == 1:
|
||||
arr = arr[0]
|
||||
if arr.ndim == 3 and arr.shape[1:] == (rows, cols):
|
||||
index = min(max(int(view_index), 0), arr.shape[0] - 1)
|
||||
arr = arr[index]
|
||||
if arr.shape != (rows, cols):
|
||||
raise SystemExit(f"expected matrix shape {(rows, cols)}, got {arr.shape}")
|
||||
return arr
|
||||
|
||||
|
||||
def speedy_camera_rt(extrinsic: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
||||
# Speedy-Splat's Camera builds world_view_transform with R.T and T.
|
||||
return np.asarray(extrinsic[:3, :3], dtype=np.float32).T, np.asarray(extrinsic[:3, 3], dtype=np.float32)
|
||||
|
||||
|
||||
def load_image_tensor(image_module: Any, torch_module: Any, path: Path, width: int, height: int):
|
||||
image = image_module.open(path).convert("RGB").resize((width, height))
|
||||
arr = np.asarray(image, dtype=np.float32) / 255.0
|
||||
return torch_module.from_numpy(arr).permute(2, 0, 1).contiguous()
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
raise SystemExit(f"expected JSON object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
299
vendor/gs_playground/scripts/real2sim_external/synthetic_multiview_expand.py
vendored
Normal file
299
vendor/gs_playground/scripts/real2sim_external/synthetic_multiview_expand.py
vendored
Normal file
@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Expand one RGB reference image into synthetic multi-view frames.
|
||||
|
||||
This is an optional Real2Sim adapter. It calls an OpenAI-style image edit API
|
||||
with one reference image, saves the returned contact sheet, and slices it into
|
||||
ordered view images that AnySplat can consume through ``--frames-dir``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
VIEW_NAMES = [
|
||||
"01_front_main",
|
||||
"02_front_left_45",
|
||||
"03_left_90",
|
||||
"04_rear_left_135",
|
||||
"05_rear_back",
|
||||
"06_rear_right_225",
|
||||
"07_right_270",
|
||||
"08_top_oblique",
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate synthetic multi-view frames from one image")
|
||||
parser.add_argument("--image", type=Path, required=True, help="Reference RGB image")
|
||||
parser.add_argument("--output-dir", type=Path, required=True, help="Directory for generated metadata and contact sheet")
|
||||
parser.add_argument("--frames-dir", type=Path, required=True, help="Directory where sliced views are written")
|
||||
parser.add_argument("--prompt", default="", help="Image edit prompt. A real2sim prompt is built when omitted.")
|
||||
parser.add_argument("--mode", choices=["object", "scene"], default="object", help="Default prompt mode when --prompt is omitted")
|
||||
parser.add_argument("--api-url", default=os.environ.get("REAL2SIM_IMAGE_EDIT_API_URL", "https://chatapi.applyma.com/v1/images/edits"))
|
||||
parser.add_argument("--api-key-env", default="REAL2SIM_IMAGE_EDIT_API_KEY", help="Environment variable that contains the API key")
|
||||
parser.add_argument("--model", default=os.environ.get("REAL2SIM_IMAGE_EDIT_MODEL", "gpt-image-2"))
|
||||
parser.add_argument("--size", default=os.environ.get("REAL2SIM_IMAGE_EDIT_SIZE", "1536x1024"))
|
||||
parser.add_argument("--quality", default=os.environ.get("REAL2SIM_IMAGE_EDIT_QUALITY", "auto"))
|
||||
parser.add_argument("--response-format", default="b64_json")
|
||||
parser.add_argument("--cols", type=int, default=4)
|
||||
parser.add_argument("--rows", type=int, default=2)
|
||||
parser.add_argument(
|
||||
"--cell-crop-ratio",
|
||||
type=float,
|
||||
default=float(os.environ.get("REAL2SIM_IMAGE_EDIT_CELL_CROP_RATIO", "0.015")),
|
||||
help="Small per-cell edge crop used to remove generated contact-sheet gutters",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-label-bars",
|
||||
action="store_true",
|
||||
help="Keep generated panel label bars instead of automatically cropping dark bottom captions",
|
||||
)
|
||||
parser.add_argument("--timeout", type=float, default=float(os.environ.get("REAL2SIM_IMAGE_EDIT_TIMEOUT", "300")))
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.image.is_file():
|
||||
raise SystemExit(f"reference image not found: {args.image}")
|
||||
api_key = os.environ.get(args.api_key_env)
|
||||
if not api_key:
|
||||
raise SystemExit(f"missing API key: set {args.api_key_env}")
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
prompt = args.prompt.strip() or build_real2sim_multiview_prompt("the reference object", mode=args.mode)
|
||||
(args.output_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
|
||||
payload = request_image_edit(
|
||||
api_url=args.api_url,
|
||||
api_key=api_key,
|
||||
image=args.image,
|
||||
prompt=prompt,
|
||||
model=args.model,
|
||||
size=args.size,
|
||||
quality=args.quality,
|
||||
response_format=args.response_format,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
write_response_metadata(payload, args.output_dir / "response_meta.json")
|
||||
contact_sheet = args.output_dir / "contact_sheet.png"
|
||||
save_first_image_payload(payload, contact_sheet, timeout=args.timeout)
|
||||
views = split_contact_sheet(
|
||||
contact_sheet,
|
||||
args.frames_dir,
|
||||
cols=args.cols,
|
||||
rows=args.rows,
|
||||
cell_crop_ratio=args.cell_crop_ratio,
|
||||
remove_label_bars=not args.keep_label_bars,
|
||||
)
|
||||
manifest = {
|
||||
"backend": "synthetic_multiview_expand",
|
||||
"reference_image": args.image.as_posix(),
|
||||
"contact_sheet": contact_sheet.as_posix(),
|
||||
"frames_dir": args.frames_dir.as_posix(),
|
||||
"views": [path.as_posix() for path in views],
|
||||
"grid": {"cols": args.cols, "rows": args.rows},
|
||||
"cell_crop_ratio": args.cell_crop_ratio,
|
||||
"model": args.model,
|
||||
"size": args.size,
|
||||
"quality": args.quality,
|
||||
}
|
||||
(args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
def build_real2sim_multiview_prompt(subject: str, *, mode: str = "object") -> str:
|
||||
if mode == "scene":
|
||||
return (
|
||||
"Create a single photorealistic real2sim scene-completion multiview contact sheet from the reference image. "
|
||||
f"Scene identity to preserve: {subject}. Treat the reference as the canonical source view and preserve the exact visible layout, "
|
||||
"floor plane, walls, doors, windows, static furniture, fixtures, material colors, lighting direction, and object identities. "
|
||||
"Output exactly 8 equal portrait-photo panels in a clean 4 columns x 2 rows grid, filling one 1536x1024 image. "
|
||||
"Use thin white gutters between panels. No panel titles, labels, captions, numbers, UI icons, or watermarks. "
|
||||
"Views, left to right, top row then bottom row: source/front view, front-left 45 degree view, left side 90 degree view, "
|
||||
"rear-left 135 degree view, rear/back view, rear-right 225 degree view, right side 270 degree view, and top-oblique "
|
||||
"30 to 45 degree view. Infer unseen regions conservatively as physically plausible continuations of the visible scene, "
|
||||
"keeping camera height, focal length, scale, horizon, navigable ground continuity, occlusion relationships, and shadows consistent. "
|
||||
"Do not turn the scene into an isolated object turntable, do not invent a different room, and do not add people."
|
||||
)
|
||||
return (
|
||||
"Create a single photorealistic real2sim multiview capture contact sheet from the reference image. "
|
||||
f"Subject identity to preserve: {subject}. Keep the same object geometry, material, colors, labels, markings, and scale. "
|
||||
"Output exactly 8 equal portrait-photo panels in a clean 4 columns x 2 rows grid, filling one 1536x1024 image. "
|
||||
"Use thin white gutters between panels. No panel titles, labels, captions, numbers, UI icons, or watermarks. "
|
||||
"Views, left to right, top row then bottom row: front main view, front-left 45 degree view, left side 90 degree view, "
|
||||
"rear-left 135 degree view, rear/back view, rear-right 225 degree view, right side 270 degree view, and top-oblique "
|
||||
"30 to 45 degree view. Keep the object stationary on the same tabletop, consistent focal length, consistent soft lighting, "
|
||||
"sharp focus, no motion blur, no hands, no people blocking the object, no play button overlays. Keep the object fully inside "
|
||||
"every panel with 10 to 15 percent silhouette margin and preserve reconstruction-useful texture and contour details."
|
||||
)
|
||||
|
||||
|
||||
def request_image_edit(
|
||||
*,
|
||||
api_url: str,
|
||||
api_key: str,
|
||||
image: Path,
|
||||
prompt: str,
|
||||
model: str,
|
||||
size: str,
|
||||
quality: str,
|
||||
response_format: str,
|
||||
timeout: float,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
import requests
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"synthetic multiview expansion requires requests: {exc}") from exc
|
||||
|
||||
with image.open("rb") as image_file:
|
||||
response = requests.post(
|
||||
api_url,
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
data={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"size": size,
|
||||
"quality": quality,
|
||||
"response_format": response_format,
|
||||
"n": "1",
|
||||
},
|
||||
files=[("image", (image.name, image_file, mime_type_for_image(image)))],
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise SystemExit(f"image edit API failed with status {response.status_code}: {response.text[:1000]}")
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"image edit API returned non-JSON response: {exc}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise SystemExit("image edit API response must be a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def write_response_metadata(payload: dict[str, Any], path: Path) -> None:
|
||||
redacted = {key: value for key, value in payload.items() if key != "data"}
|
||||
if isinstance(payload.get("data"), list):
|
||||
redacted["data"] = [
|
||||
{key: value for key, value in item.items() if key != "b64_json"}
|
||||
for item in payload["data"]
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
path.write_text(json.dumps(redacted, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def save_first_image_payload(payload: dict[str, Any], output: Path, *, timeout: float) -> None:
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
||||
raise SystemExit("image edit API response missing data[0]")
|
||||
item = data[0]
|
||||
if item.get("b64_json"):
|
||||
output.write_bytes(base64.b64decode(str(item["b64_json"])))
|
||||
return
|
||||
if item.get("url"):
|
||||
try:
|
||||
import requests
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"downloading URL response requires requests: {exc}") from exc
|
||||
response = requests.get(str(item["url"]), timeout=timeout)
|
||||
if response.status_code >= 400:
|
||||
raise SystemExit(f"failed to download generated image: HTTP {response.status_code}")
|
||||
output.write_bytes(response.content)
|
||||
return
|
||||
raise SystemExit("image edit API response data[0] must contain b64_json or url")
|
||||
|
||||
|
||||
def split_contact_sheet(
|
||||
contact_sheet: Path,
|
||||
frames_dir: Path,
|
||||
*,
|
||||
cols: int = 4,
|
||||
rows: int = 2,
|
||||
cell_crop_ratio: float = 0.015,
|
||||
remove_label_bars: bool = True,
|
||||
) -> list[Path]:
|
||||
from PIL import Image
|
||||
|
||||
if cols <= 0 or rows <= 0:
|
||||
raise ValueError("contact sheet grid must have positive cols and rows")
|
||||
if not 0 <= cell_crop_ratio < 0.25:
|
||||
raise ValueError("cell_crop_ratio must be in [0, 0.25)")
|
||||
image = Image.open(contact_sheet).convert("RGB")
|
||||
width, height = image.size
|
||||
cell_w = width // cols
|
||||
cell_h = height // rows
|
||||
if cell_w <= 0 or cell_h <= 0:
|
||||
raise ValueError(f"contact sheet is too small for {cols}x{rows}: {width}x{height}")
|
||||
frames_dir.mkdir(parents=True, exist_ok=True)
|
||||
views: list[Path] = []
|
||||
for index in range(cols * rows):
|
||||
col = index % cols
|
||||
row = index // cols
|
||||
left = col * cell_w
|
||||
top = row * cell_h
|
||||
right = width if col == cols - 1 else (col + 1) * cell_w
|
||||
bottom = height if row == rows - 1 else (row + 1) * cell_h
|
||||
crop_x = int((right - left) * cell_crop_ratio)
|
||||
crop_y = int((bottom - top) * cell_crop_ratio)
|
||||
if right - left > (2 * crop_x + 8) and bottom - top > (2 * crop_y + 8):
|
||||
left += crop_x
|
||||
right -= crop_x
|
||||
top += crop_y
|
||||
bottom -= crop_y
|
||||
name = VIEW_NAMES[index] if index < len(VIEW_NAMES) else f"{index + 1:02d}_view"
|
||||
path = frames_dir / f"{name}.jpg"
|
||||
panel = image.crop((left, top, right, bottom))
|
||||
if remove_label_bars:
|
||||
panel = crop_dark_bottom_label_bar(panel)
|
||||
panel.save(path, quality=95)
|
||||
views.append(path)
|
||||
return views
|
||||
|
||||
|
||||
def crop_dark_bottom_label_bar(image: "Image.Image") -> "Image.Image":
|
||||
import numpy as np
|
||||
|
||||
gray = np.asarray(image.convert("L"), dtype=np.float32) / 255.0
|
||||
height, width = gray.shape
|
||||
if height < 80 or width < 80:
|
||||
return image
|
||||
|
||||
dark_fraction = np.mean(gray < 0.10, axis=1)
|
||||
row_mean = np.mean(gray, axis=1)
|
||||
start = height
|
||||
scan_limit = int(height * 0.55)
|
||||
for row in range(height - 1, scan_limit, -1):
|
||||
if dark_fraction[row] > 0.55 and row_mean[row] < 0.22:
|
||||
start = row
|
||||
continue
|
||||
if start < height:
|
||||
break
|
||||
|
||||
if start == height:
|
||||
return image
|
||||
|
||||
while start > scan_limit and dark_fraction[start - 1] > 0.35 and row_mean[start - 1] < 0.35:
|
||||
start -= 1
|
||||
|
||||
bar_height = height - start
|
||||
if bar_height < max(12, int(height * 0.035)):
|
||||
return image
|
||||
if start < int(height * 0.60):
|
||||
return image
|
||||
return image.crop((0, 0, width, start))
|
||||
|
||||
|
||||
def mime_type_for_image(path: Path) -> str:
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in {".jpg", ".jpeg"}:
|
||||
return "image/jpeg"
|
||||
if suffix == ".webp":
|
||||
return "image/webp"
|
||||
return "image/png"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
176
vendor/gs_playground/scripts/real2sim_movable_tasks_to_objects.py
vendored
Normal file
176
vendor/gs_playground/scripts/real2sim_movable_tasks_to_objects.py
vendored
Normal file
@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Materialize movable extraction tasks into provisional Real2Sim object assets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from real2sim.builtin import write_3dgs_ply, write_box_obj
|
||||
from real2sim.contracts import write_json
|
||||
from real2sim.image_io import read_gray, read_rgb
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Convert movable extraction task crops/masks into provisional object assets")
|
||||
parser.add_argument("--package", type=Path, required=True, help="Real2Sim package root")
|
||||
parser.add_argument("--tasks", type=Path, default=None, help="movable_extraction_tasks.json; defaults under package")
|
||||
parser.add_argument("--output-objects-json", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--max-objects", type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = materialize_movable_tasks(
|
||||
package_root=args.package,
|
||||
tasks_path=args.tasks or args.package / "artifacts" / "assets" / "movable_extraction_tasks.json",
|
||||
output_objects_json=args.output_objects_json,
|
||||
output_dir=args.output_dir,
|
||||
max_objects=max(1, int(args.max_objects)),
|
||||
)
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def materialize_movable_tasks(
|
||||
*,
|
||||
package_root: Path,
|
||||
tasks_path: Path,
|
||||
output_objects_json: Path,
|
||||
output_dir: Path,
|
||||
max_objects: int,
|
||||
) -> dict[str, Any]:
|
||||
tasks_data = json.loads(tasks_path.read_text(encoding="utf-8"))
|
||||
tasks = tasks_data.get("tasks", []) if isinstance(tasks_data, dict) else []
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
objects: list[dict[str, Any]] = []
|
||||
for index, task in enumerate(tasks[:max_objects]):
|
||||
if not isinstance(task, dict):
|
||||
continue
|
||||
asset = materialize_task_asset(package_root=package_root, output_dir=output_dir, task=task, index=index)
|
||||
if asset is not None:
|
||||
objects.append(asset)
|
||||
payload = {
|
||||
"format": "gs-playground-real2sim-provisional-objects-v1",
|
||||
"objects": objects,
|
||||
"quality": {
|
||||
"object_count": len(objects),
|
||||
"source_task_count": len(tasks),
|
||||
"provisional": True,
|
||||
},
|
||||
"commercial_gap_notes": [
|
||||
"These assets are deterministic provisional billboards/boxes from extraction crops.",
|
||||
"Replace them with SAM/SAM3D object reconstructions for commercial visual fidelity.",
|
||||
],
|
||||
}
|
||||
write_json(output_objects_json, payload)
|
||||
return {"objects_json": output_objects_json.as_posix(), **payload["quality"]}
|
||||
|
||||
|
||||
def materialize_task_asset(*, package_root: Path, output_dir: Path, task: dict[str, Any], index: int) -> dict[str, Any] | None:
|
||||
crop_value = task.get("crop")
|
||||
mask_value = task.get("mask")
|
||||
if not crop_value or not mask_value:
|
||||
return None
|
||||
crop_path = package_root / str(crop_value)
|
||||
mask_path = package_root / str(mask_value)
|
||||
rgb = read_rgb(crop_path)
|
||||
mask = read_gray(mask_path)
|
||||
local_mask = local_crop_mask(mask, task.get("bbox_xyxy"), rgb.shape[:2])
|
||||
if int(local_mask.sum()) < 4:
|
||||
local_mask = np.ones(rgb.shape[:2], dtype=bool)
|
||||
points, colors = crop_billboard_points(rgb, local_mask, max_points=6000)
|
||||
if points.size == 0:
|
||||
return None
|
||||
label = best_prompt_label(task)
|
||||
name = f"{index:02d}_{safe_token(label)}"
|
||||
asset_dir = output_dir / name
|
||||
gaussian = asset_dir / "gaussian.ply"
|
||||
mesh = asset_dir / "mesh.obj"
|
||||
extent = object_extent_from_mask(local_mask, rgb.shape[:2], label=label)
|
||||
write_3dgs_ply(gaussian, points, colors, point_scale=0.01, opacity_logit=4.0)
|
||||
write_box_obj(mesh, extent)
|
||||
pose = pose_from_task_bbox(task.get("bbox_xyxy"))
|
||||
return {
|
||||
"name": name,
|
||||
"label": label,
|
||||
"gaussian_ply": gaussian.as_posix(),
|
||||
"mesh_obj": mesh.as_posix(),
|
||||
"pose": pose,
|
||||
"dynamic": True,
|
||||
"source_instance_id": str(task.get("id") or name),
|
||||
"provisional": True,
|
||||
}
|
||||
|
||||
|
||||
def local_crop_mask(global_mask: np.ndarray, bbox: Any, crop_shape: tuple[int, int]) -> np.ndarray:
|
||||
if not isinstance(bbox, list) or len(bbox) != 4:
|
||||
return np.ones(crop_shape, dtype=bool)
|
||||
x0, y0, x1, y1 = [int(value) for value in bbox]
|
||||
cropped = global_mask[max(0, y0) : max(0, y1), max(0, x0) : max(0, x1)] > 0.5
|
||||
if cropped.shape != crop_shape:
|
||||
return np.ones(crop_shape, dtype=bool)
|
||||
return cropped
|
||||
|
||||
|
||||
def crop_billboard_points(rgb: np.ndarray, mask: np.ndarray, *, max_points: int) -> tuple[np.ndarray, np.ndarray]:
|
||||
ys, xs = np.where(mask)
|
||||
if ys.size == 0:
|
||||
return np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.float32)
|
||||
if ys.size > max_points:
|
||||
indices = np.linspace(0, ys.size - 1, max_points, dtype=np.int64)
|
||||
ys = ys[indices]
|
||||
xs = xs[indices]
|
||||
h, w = rgb.shape[:2]
|
||||
x = ((xs.astype(np.float32) / max(w - 1, 1)) - 0.5) * 0.38
|
||||
z = (0.5 - (ys.astype(np.float32) / max(h - 1, 1))) * 0.38
|
||||
y = np.zeros_like(x)
|
||||
points = np.stack([x, y, z], axis=1).astype(np.float32)
|
||||
colors = rgb[ys, xs, :3].astype(np.float32)
|
||||
return points, colors
|
||||
|
||||
|
||||
def object_extent_from_mask(mask: np.ndarray, shape: tuple[int, int], *, label: str) -> np.ndarray:
|
||||
ys, xs = np.where(mask)
|
||||
if ys.size == 0:
|
||||
return np.array([0.2, 0.08, 0.2], dtype=np.float32)
|
||||
h, w = shape
|
||||
width = max((float(xs.max() - xs.min() + 1) / max(w, 1)) * 0.38, 0.06)
|
||||
height = max((float(ys.max() - ys.min() + 1) / max(h, 1)) * 0.38, 0.06)
|
||||
depth = 0.08
|
||||
if "bottle" in label or "cup" in label:
|
||||
depth = max(width * 0.55, 0.06)
|
||||
return np.array([width, depth, height], dtype=np.float32)
|
||||
|
||||
|
||||
def pose_from_task_bbox(bbox: Any) -> dict[str, list[float]]:
|
||||
if not isinstance(bbox, list) or len(bbox) != 4:
|
||||
return {"position": [0.0, -0.25, 0.65], "quat_wxyz": [1.0, 0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}
|
||||
x0, y0, x1, y1 = [float(value) for value in bbox]
|
||||
cx = (x0 + x1) * 0.5
|
||||
cy = (y0 + y1) * 0.5
|
||||
position = [round((cx - 224.0) / 224.0 * 0.9, 6), -0.35, round((448.0 - cy) / 448.0 * 0.65 + 0.25, 6)]
|
||||
return {"position": position, "quat_wxyz": [1.0, 0.0, 0.0, 0.0], "scale": [1.0, 1.0, 1.0]}
|
||||
|
||||
|
||||
def best_prompt_label(task: dict[str, Any]) -> str:
|
||||
prompts = task.get("prompt_candidates")
|
||||
if isinstance(prompts, list) and prompts:
|
||||
return str(prompts[0])
|
||||
return "movable_object"
|
||||
|
||||
|
||||
def safe_token(value: str) -> str:
|
||||
return "".join(ch if ch.isalnum() else "_" for ch in value.lower()).strip("_") or "object"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
382
vendor/gs_playground/scripts/real2sim_obj_mesh_cleanup.py
vendored
Normal file
382
vendor/gs_playground/scripts/real2sim_obj_mesh_cleanup.py
vendored
Normal file
@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Filter OBJ mesh fragments before Real2Sim texture baking."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict, deque
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjFace:
|
||||
vertices: tuple[int, int, int]
|
||||
texcoords: tuple[int, int, int]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjMesh:
|
||||
vertices: np.ndarray
|
||||
colors: np.ndarray
|
||||
texcoords: np.ndarray
|
||||
faces: tuple[ObjFace, ...]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Filter small disconnected OBJ mesh fragments")
|
||||
parser.add_argument("--input-obj", type=Path, required=True)
|
||||
parser.add_argument("--output-obj", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--keep-largest", type=int, default=1, help="Keep this many largest connected components")
|
||||
parser.add_argument("--min-faces", type=int, default=0, help="Also keep components with at least this many faces")
|
||||
parser.add_argument("--min-area-ratio", type=float, default=0.0, help="Also keep components with area >= ratio * total area")
|
||||
parser.add_argument("--max-face-area-percentile", type=float, default=100.0, help="Drop triangles above this area percentile")
|
||||
parser.add_argument("--max-edge-length-percentile", type=float, default=100.0, help="Drop triangles whose longest edge is above this percentile")
|
||||
parser.add_argument("--min-centroid-x-percentile", type=float, default=0.0, help="Drop triangles whose centroid x is below this percentile")
|
||||
parser.add_argument("--max-centroid-x-percentile", type=float, default=100.0, help="Drop triangles whose centroid x is above this percentile")
|
||||
parser.add_argument("--min-centroid-y-percentile", type=float, default=0.0, help="Drop triangles whose centroid y is below this percentile")
|
||||
parser.add_argument("--max-centroid-y-percentile", type=float, default=100.0, help="Drop triangles whose centroid y is above this percentile")
|
||||
parser.add_argument("--min-centroid-z-percentile", type=float, default=0.0, help="Drop triangles whose centroid z is below this percentile")
|
||||
parser.add_argument("--max-centroid-z-percentile", type=float, default=100.0, help="Drop triangles whose centroid z is above this percentile")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = cleanup_obj_mesh(
|
||||
input_obj=args.input_obj,
|
||||
output_obj=args.output_obj,
|
||||
keep_largest=max(int(args.keep_largest), 0),
|
||||
min_faces=max(int(args.min_faces), 0),
|
||||
min_area_ratio=max(float(args.min_area_ratio), 0.0),
|
||||
max_face_area_percentile=float(args.max_face_area_percentile),
|
||||
max_edge_length_percentile=float(args.max_edge_length_percentile),
|
||||
centroid_percentile_bounds={
|
||||
"x": (float(args.min_centroid_x_percentile), float(args.max_centroid_x_percentile)),
|
||||
"y": (float(args.min_centroid_y_percentile), float(args.max_centroid_y_percentile)),
|
||||
"z": (float(args.min_centroid_z_percentile), float(args.max_centroid_z_percentile)),
|
||||
},
|
||||
)
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def cleanup_obj_mesh(
|
||||
*,
|
||||
input_obj: Path,
|
||||
output_obj: Path,
|
||||
keep_largest: int = 1,
|
||||
min_faces: int = 0,
|
||||
min_area_ratio: float = 0.0,
|
||||
max_face_area_percentile: float = 100.0,
|
||||
max_edge_length_percentile: float = 100.0,
|
||||
centroid_percentile_bounds: dict[str, tuple[float, float]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
mesh = read_obj_mesh(input_obj)
|
||||
components = connected_face_components(mesh)
|
||||
face_areas = triangle_areas(mesh)
|
||||
face_edges = triangle_max_edges(mesh)
|
||||
centroids = triangle_centroids(mesh)
|
||||
total_area = float(np.sum(face_areas))
|
||||
area_threshold = percentile_threshold(face_areas, max_face_area_percentile)
|
||||
edge_threshold = percentile_threshold(face_edges, max_edge_length_percentile)
|
||||
centroid_filters = centroid_axis_filters(centroids, centroid_percentile_bounds)
|
||||
ranked = sorted(
|
||||
(
|
||||
{
|
||||
"component": idx,
|
||||
"face_count": len(face_indices),
|
||||
"area": float(np.sum(face_areas[np.asarray(face_indices, dtype=np.int64)])),
|
||||
"faces": face_indices,
|
||||
}
|
||||
for idx, face_indices in enumerate(components)
|
||||
),
|
||||
key=lambda item: (item["face_count"], item["area"]),
|
||||
reverse=True,
|
||||
)
|
||||
keep_component_ids: set[int] = set()
|
||||
for item in ranked[:keep_largest]:
|
||||
keep_component_ids.add(int(item["component"]))
|
||||
for item in ranked:
|
||||
area_ratio = (float(item["area"]) / total_area) if total_area > 0 else 0.0
|
||||
if (min_faces > 0 and int(item["face_count"]) >= min_faces) or (
|
||||
min_area_ratio > 0.0 and area_ratio >= min_area_ratio
|
||||
):
|
||||
keep_component_ids.add(int(item["component"]))
|
||||
kept_faces_before_quality = [
|
||||
face_index
|
||||
for component_id, face_indices in enumerate(components)
|
||||
if component_id in keep_component_ids
|
||||
for face_index in face_indices
|
||||
]
|
||||
kept_faces = [
|
||||
face_index
|
||||
for face_index in kept_faces_before_quality
|
||||
if (area_threshold is None or face_areas[int(face_index)] <= area_threshold)
|
||||
and (edge_threshold is None or face_edges[int(face_index)] <= edge_threshold)
|
||||
and centroid_face_allowed(centroids[int(face_index)], centroid_filters)
|
||||
]
|
||||
cleaned = remap_mesh(mesh, sorted(kept_faces))
|
||||
output_obj.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_obj_mesh(cleaned, output_obj)
|
||||
kept_area = float(np.sum(face_areas[np.asarray(kept_faces, dtype=np.int64)])) if kept_faces else 0.0
|
||||
report = {
|
||||
"format": "gs-playground-real2sim-obj-mesh-cleanup-report-v1",
|
||||
"input_obj": input_obj.resolve().as_posix(),
|
||||
"output_obj": output_obj.resolve().as_posix(),
|
||||
"input_vertex_count": int(mesh.vertices.shape[0]),
|
||||
"input_texcoord_count": int(mesh.texcoords.shape[0]),
|
||||
"input_face_count": int(len(mesh.faces)),
|
||||
"component_count": int(len(components)),
|
||||
"kept_component_count": int(len(keep_component_ids)),
|
||||
"component_kept_face_count": int(len(kept_faces_before_quality)),
|
||||
"kept_face_count": int(len(kept_faces)),
|
||||
"removed_face_count": int(len(mesh.faces) - len(kept_faces)),
|
||||
"quality_removed_face_count": int(len(kept_faces_before_quality) - len(kept_faces)),
|
||||
"centroid_filter_removed_face_count": int(
|
||||
sum(
|
||||
1
|
||||
for face_index in kept_faces_before_quality
|
||||
if not centroid_face_allowed(centroids[int(face_index)], centroid_filters)
|
||||
)
|
||||
),
|
||||
"kept_area_ratio": round((kept_area / total_area) if total_area > 0 else 0.0, 6),
|
||||
"keep_largest": int(keep_largest),
|
||||
"min_faces": int(min_faces),
|
||||
"min_area_ratio": float(min_area_ratio),
|
||||
"max_face_area_percentile": float(max_face_area_percentile),
|
||||
"max_edge_length_percentile": float(max_edge_length_percentile),
|
||||
"face_area_threshold": float(area_threshold) if area_threshold is not None else None,
|
||||
"edge_length_threshold": float(edge_threshold) if edge_threshold is not None else None,
|
||||
"centroid_filters": centroid_filters,
|
||||
"largest_components": [
|
||||
{
|
||||
"component": int(item["component"]),
|
||||
"face_count": int(item["face_count"]),
|
||||
"area_ratio": round((float(item["area"]) / total_area) if total_area > 0 else 0.0, 6),
|
||||
}
|
||||
for item in ranked[:10]
|
||||
],
|
||||
}
|
||||
return report
|
||||
|
||||
|
||||
def connected_face_components(mesh: ObjMesh) -> list[list[int]]:
|
||||
vertex_to_faces: dict[int, list[int]] = defaultdict(list)
|
||||
for face_index, face in enumerate(mesh.faces):
|
||||
for vertex in face.vertices:
|
||||
vertex_to_faces[int(vertex)].append(face_index)
|
||||
visited = np.zeros(len(mesh.faces), dtype=bool)
|
||||
components: list[list[int]] = []
|
||||
for start in range(len(mesh.faces)):
|
||||
if visited[start]:
|
||||
continue
|
||||
queue: deque[int] = deque([start])
|
||||
visited[start] = True
|
||||
component: list[int] = []
|
||||
while queue:
|
||||
face_index = queue.popleft()
|
||||
component.append(face_index)
|
||||
for vertex in mesh.faces[face_index].vertices:
|
||||
for neighbor in vertex_to_faces[int(vertex)]:
|
||||
if not visited[neighbor]:
|
||||
visited[neighbor] = True
|
||||
queue.append(neighbor)
|
||||
components.append(component)
|
||||
return components
|
||||
|
||||
|
||||
def triangle_areas(mesh: ObjMesh) -> np.ndarray:
|
||||
if not mesh.faces:
|
||||
return np.zeros((0,), dtype=np.float64)
|
||||
indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64)
|
||||
points = mesh.vertices[indices].astype(np.float64)
|
||||
areas = np.linalg.norm(np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]), axis=1) * 0.5
|
||||
return areas
|
||||
|
||||
|
||||
def triangle_max_edges(mesh: ObjMesh) -> np.ndarray:
|
||||
if not mesh.faces:
|
||||
return np.zeros((0,), dtype=np.float64)
|
||||
indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64)
|
||||
points = mesh.vertices[indices].astype(np.float64)
|
||||
e01 = np.linalg.norm(points[:, 0] - points[:, 1], axis=1)
|
||||
e12 = np.linalg.norm(points[:, 1] - points[:, 2], axis=1)
|
||||
e20 = np.linalg.norm(points[:, 2] - points[:, 0], axis=1)
|
||||
return np.maximum(np.maximum(e01, e12), e20)
|
||||
|
||||
|
||||
def triangle_centroids(mesh: ObjMesh) -> np.ndarray:
|
||||
if not mesh.faces:
|
||||
return np.zeros((0, 3), dtype=np.float64)
|
||||
indices = np.asarray([face.vertices for face in mesh.faces], dtype=np.int64)
|
||||
points = mesh.vertices[indices].astype(np.float64)
|
||||
return points.mean(axis=1)
|
||||
|
||||
|
||||
def percentile_threshold(values: np.ndarray, percentile: float) -> float | None:
|
||||
if values.size == 0 or percentile >= 100.0:
|
||||
return None
|
||||
clamped = min(max(float(percentile), 0.0), 100.0)
|
||||
return float(np.percentile(values, clamped))
|
||||
|
||||
|
||||
def centroid_axis_filters(
|
||||
centroids: np.ndarray,
|
||||
percentile_bounds: dict[str, tuple[float, float]] | None,
|
||||
) -> dict[str, dict[str, float | None]]:
|
||||
axes = {"x": 0, "y": 1, "z": 2}
|
||||
filters: dict[str, dict[str, float | None]] = {}
|
||||
if centroids.size == 0:
|
||||
return {
|
||||
axis: {"min_percentile": None, "max_percentile": None, "min_value": None, "max_value": None}
|
||||
for axis in axes
|
||||
}
|
||||
percentile_bounds = percentile_bounds or {}
|
||||
for axis, index in axes.items():
|
||||
lower, upper = percentile_bounds.get(axis, (0.0, 100.0))
|
||||
lower = min(max(float(lower), 0.0), 100.0)
|
||||
upper = min(max(float(upper), 0.0), 100.0)
|
||||
if lower > upper:
|
||||
lower, upper = upper, lower
|
||||
values = centroids[:, index]
|
||||
min_value = float(np.percentile(values, lower)) if lower > 0.0 else None
|
||||
max_value = float(np.percentile(values, upper)) if upper < 100.0 else None
|
||||
filters[axis] = {
|
||||
"min_percentile": lower if min_value is not None else None,
|
||||
"max_percentile": upper if max_value is not None else None,
|
||||
"min_value": min_value,
|
||||
"max_value": max_value,
|
||||
}
|
||||
return filters
|
||||
|
||||
|
||||
def centroid_face_allowed(centroid: np.ndarray, filters: dict[str, dict[str, float | None]]) -> bool:
|
||||
for axis, index in {"x": 0, "y": 1, "z": 2}.items():
|
||||
bounds = filters.get(axis, {})
|
||||
min_value = bounds.get("min_value")
|
||||
max_value = bounds.get("max_value")
|
||||
value = float(centroid[index])
|
||||
if min_value is not None and value < float(min_value):
|
||||
return False
|
||||
if max_value is not None and value > float(max_value):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def remap_mesh(mesh: ObjMesh, face_indices: list[int]) -> ObjMesh:
|
||||
vertex_map: dict[int, int] = {}
|
||||
texcoord_map: dict[int, int] = {}
|
||||
vertices: list[np.ndarray] = []
|
||||
colors: list[np.ndarray] = []
|
||||
texcoords: list[np.ndarray] = []
|
||||
faces: list[ObjFace] = []
|
||||
for face_index in face_indices:
|
||||
face = mesh.faces[int(face_index)]
|
||||
new_vertices: list[int] = []
|
||||
new_texcoords: list[int] = []
|
||||
for vertex in face.vertices:
|
||||
if vertex not in vertex_map:
|
||||
vertex_map[vertex] = len(vertices)
|
||||
vertices.append(mesh.vertices[vertex])
|
||||
colors.append(mesh.colors[vertex])
|
||||
new_vertices.append(vertex_map[vertex])
|
||||
for texcoord in face.texcoords:
|
||||
if texcoord not in texcoord_map:
|
||||
texcoord_map[texcoord] = len(texcoords)
|
||||
texcoords.append(mesh.texcoords[texcoord])
|
||||
new_texcoords.append(texcoord_map[texcoord])
|
||||
faces.append(ObjFace(vertices=tuple(new_vertices), texcoords=tuple(new_texcoords))) # type: ignore[arg-type]
|
||||
return ObjMesh(
|
||||
vertices=np.asarray(vertices, dtype=np.float32),
|
||||
colors=np.asarray(colors, dtype=np.float32),
|
||||
texcoords=np.asarray(texcoords, dtype=np.float32),
|
||||
faces=tuple(faces),
|
||||
)
|
||||
|
||||
|
||||
def read_obj_mesh(path: Path) -> ObjMesh:
|
||||
vertices: list[tuple[float, float, float]] = []
|
||||
colors: list[tuple[float, float, float]] = []
|
||||
texcoords: list[tuple[float, float]] = []
|
||||
faces: list[ObjFace] = []
|
||||
for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
if raw.startswith("v "):
|
||||
parts = raw.split()
|
||||
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
||||
if len(parts) >= 7:
|
||||
colors.append((float(parts[4]), float(parts[5]), float(parts[6])))
|
||||
else:
|
||||
colors.append((0.5, 0.5, 0.5))
|
||||
elif raw.startswith("vt "):
|
||||
parts = raw.split()
|
||||
if len(parts) >= 3:
|
||||
texcoords.append((float(parts[1]), float(parts[2])))
|
||||
elif raw.startswith("f "):
|
||||
refs = [parse_face_ref(token) for token in raw.split()[1:]]
|
||||
if len(refs) < 3:
|
||||
continue
|
||||
for idx in range(1, len(refs) - 1):
|
||||
tri = (refs[0], refs[idx], refs[idx + 1])
|
||||
faces.append(
|
||||
ObjFace(
|
||||
vertices=tuple(item[0] for item in tri),
|
||||
texcoords=tuple(item[1] if item[1] is not None else item[0] for item in tri),
|
||||
)
|
||||
)
|
||||
if not texcoords:
|
||||
texcoords = [(0.0, 0.0) for _ in vertices]
|
||||
return ObjMesh(
|
||||
vertices=np.asarray(vertices, dtype=np.float32),
|
||||
colors=np.clip(np.asarray(colors, dtype=np.float32), 0.0, 1.0),
|
||||
texcoords=np.asarray(texcoords, dtype=np.float32),
|
||||
faces=tuple(faces),
|
||||
)
|
||||
|
||||
|
||||
def parse_face_ref(token: str) -> tuple[int, int | None]:
|
||||
parts = token.split("/")
|
||||
vertex = int(parts[0]) - 1
|
||||
texcoord = int(parts[1]) - 1 if len(parts) >= 2 and parts[1] else None
|
||||
return vertex, texcoord
|
||||
|
||||
|
||||
def write_obj_mesh(mesh: ObjMesh, output_obj: Path) -> None:
|
||||
material_name = output_obj.stem + "_mat"
|
||||
material_file = output_obj.with_suffix(".mtl")
|
||||
with material_file.open("w", encoding="utf-8") as dst:
|
||||
dst.write("# Generated Real2Sim cleaned OBJ material\n")
|
||||
dst.write(f"newmtl {material_name}\n")
|
||||
dst.write("Ka 1.000000 1.000000 1.000000\n")
|
||||
dst.write("Kd 1.000000 1.000000 1.000000\n")
|
||||
dst.write("Ks 0.030000 0.030000 0.030000\n")
|
||||
dst.write("Ns 8.000000\n")
|
||||
dst.write("d 1.000000\n")
|
||||
dst.write("illum 2\n")
|
||||
with output_obj.open("w", encoding="utf-8") as dst:
|
||||
dst.write("# Real2Sim cleaned scene mesh\n")
|
||||
dst.write(f"mtllib {material_file.name}\n")
|
||||
dst.write("o real2sim_cleaned_scene\n")
|
||||
for vertex, color in zip(mesh.vertices, mesh.colors):
|
||||
dst.write(
|
||||
f"v {vertex[0]:.9g} {vertex[1]:.9g} {vertex[2]:.9g} "
|
||||
f"{color[0]:.6f} {color[1]:.6f} {color[2]:.6f}\n"
|
||||
)
|
||||
for texcoord in mesh.texcoords:
|
||||
dst.write(f"vt {texcoord[0]:.7g} {texcoord[1]:.7g}\n")
|
||||
dst.write(f"usemtl {material_name}\n")
|
||||
for face in mesh.faces:
|
||||
dst.write(
|
||||
"f "
|
||||
f"{face.vertices[0] + 1}/{face.texcoords[0] + 1} "
|
||||
f"{face.vertices[1] + 1}/{face.texcoords[1] + 1} "
|
||||
f"{face.vertices[2] + 1}/{face.texcoords[2] + 1}\n"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
265
vendor/gs_playground/scripts/real2sim_obj_scene_completion.py
vendored
Normal file
265
vendor/gs_playground/scripts/real2sim_obj_scene_completion.py
vendored
Normal file
@ -0,0 +1,265 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Add coarse room/tabletop completion planes to a reconstructed OBJ scene.
|
||||
|
||||
The COLMAP/Delaunay mesh path often produces visually useful fragments but
|
||||
leaves large holes in walls, tabletops, or floors. This utility keeps the
|
||||
original OBJ intact and appends a conservative, quantile-bounded planar shell
|
||||
so MotrixSim has a continuous scene scaffold for inspection and collision.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Bounds:
|
||||
low: tuple[float, float, float]
|
||||
high: tuple[float, float, float]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Plane:
|
||||
name: str
|
||||
axis: int
|
||||
value: float
|
||||
u_axis: int
|
||||
v_axis: int
|
||||
u_min: float
|
||||
u_max: float
|
||||
v_min: float
|
||||
v_max: float
|
||||
material: str
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Append quantile-bounded scene completion planes to an OBJ")
|
||||
parser.add_argument("--input-obj", type=Path, required=True)
|
||||
parser.add_argument("--output-obj", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--low-quantile", type=float, default=0.05)
|
||||
parser.add_argument("--high-quantile", type=float, default=0.95)
|
||||
parser.add_argument("--margin-ratio", type=float, default=0.03)
|
||||
parser.add_argument("--up-axis", choices=("x", "y", "z"), default="z")
|
||||
parser.add_argument("--grid", type=int, default=24, help="Subdivisions per plane axis")
|
||||
parser.add_argument(
|
||||
"--planes",
|
||||
default="floor,ceiling,left,right,back,front",
|
||||
help="Comma-separated planes to append: floor,ceiling,left,right,back,front",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
vertices, texcoord_count = read_obj_counts(args.input_obj)
|
||||
if len(vertices) < 3:
|
||||
raise SystemExit(f"OBJ has too few vertices for scene completion: {args.input_obj}")
|
||||
low_q = clamp(args.low_quantile, 0.0, 0.49)
|
||||
high_q = clamp(args.high_quantile, 0.51, 1.0)
|
||||
if low_q >= high_q:
|
||||
raise SystemExit("--low-quantile must be below --high-quantile")
|
||||
bounds = quantile_bounds(vertices, low_q, high_q, margin_ratio=max(args.margin_ratio, 0.0))
|
||||
up_axis = "xyz".index(args.up_axis)
|
||||
plane_names = tuple(name.strip().lower() for name in args.planes.split(",") if name.strip())
|
||||
planes = build_planes(bounds, up_axis=up_axis, names=plane_names)
|
||||
if not planes:
|
||||
raise SystemExit("No completion planes selected")
|
||||
|
||||
args.output_obj.parent.mkdir(parents=True, exist_ok=True)
|
||||
mtl_path = args.output_obj.with_name(f"{args.output_obj.stem}_completion.mtl")
|
||||
append_completion(
|
||||
args.input_obj,
|
||||
args.output_obj,
|
||||
mtl_path,
|
||||
vertices_start=len(vertices),
|
||||
texcoords_start=texcoord_count,
|
||||
planes=planes,
|
||||
grid=max(args.grid, 1),
|
||||
)
|
||||
report = {
|
||||
"format": "gs-playground-real2sim-obj-scene-completion-report-v1",
|
||||
"input_obj": args.input_obj.as_posix(),
|
||||
"output_obj": args.output_obj.as_posix(),
|
||||
"material_library": mtl_path.name,
|
||||
"input_vertex_count": len(vertices),
|
||||
"input_texcoord_count": texcoord_count,
|
||||
"bounds_low": list(bounds.low),
|
||||
"bounds_high": list(bounds.high),
|
||||
"low_quantile": low_q,
|
||||
"high_quantile": high_q,
|
||||
"margin_ratio": max(args.margin_ratio, 0.0),
|
||||
"up_axis": args.up_axis,
|
||||
"grid": max(args.grid, 1),
|
||||
"plane_count": len(planes),
|
||||
"planes": [plane.name for plane in planes],
|
||||
"added_vertex_count": len(planes) * (max(args.grid, 1) + 1) ** 2,
|
||||
"added_texcoord_count": len(planes) * (max(args.grid, 1) + 1) ** 2,
|
||||
"added_face_count": len(planes) * max(args.grid, 1) * max(args.grid, 1) * 2,
|
||||
}
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def read_obj_counts(path: Path) -> tuple[list[tuple[float, float, float]], int]:
|
||||
vertices: list[tuple[float, float, float]] = []
|
||||
texcoord_count = 0
|
||||
with path.open("r", encoding="utf-8", errors="ignore") as handle:
|
||||
for line in handle:
|
||||
if line.startswith("vt "):
|
||||
texcoord_count += 1
|
||||
continue
|
||||
if not line.startswith("v "):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 4:
|
||||
continue
|
||||
try:
|
||||
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
||||
except ValueError:
|
||||
continue
|
||||
return vertices, texcoord_count
|
||||
|
||||
|
||||
def quantile_bounds(
|
||||
vertices: Iterable[tuple[float, float, float]],
|
||||
low_quantile: float,
|
||||
high_quantile: float,
|
||||
*,
|
||||
margin_ratio: float,
|
||||
) -> Bounds:
|
||||
columns = list(zip(*vertices))
|
||||
lows: list[float] = []
|
||||
highs: list[float] = []
|
||||
for values in columns:
|
||||
ordered = sorted(values)
|
||||
low = percentile(ordered, low_quantile)
|
||||
high = percentile(ordered, high_quantile)
|
||||
span = max(high - low, 1e-6)
|
||||
lows.append(low - span * margin_ratio)
|
||||
highs.append(high + span * margin_ratio)
|
||||
return Bounds(tuple(lows), tuple(highs))
|
||||
|
||||
|
||||
def percentile(values: list[float], q: float) -> float:
|
||||
if not values:
|
||||
raise ValueError("percentile requires values")
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
pos = q * (len(values) - 1)
|
||||
lo = int(pos)
|
||||
hi = min(lo + 1, len(values) - 1)
|
||||
frac = pos - lo
|
||||
return values[lo] * (1.0 - frac) + values[hi] * frac
|
||||
|
||||
|
||||
def build_planes(bounds: Bounds, *, up_axis: int, names: tuple[str, ...]) -> list[Plane]:
|
||||
axes = {0, 1, 2}
|
||||
horizontal = [axis for axis in axes if axis != up_axis]
|
||||
a0, a1 = horizontal
|
||||
low, high = bounds.low, bounds.high
|
||||
specs = {
|
||||
"floor": Plane("floor", up_axis, low[up_axis], a0, a1, low[a0], high[a0], low[a1], high[a1], "real2sim_completion_floor"),
|
||||
"ceiling": Plane("ceiling", up_axis, high[up_axis], a0, a1, low[a0], high[a0], low[a1], high[a1], "real2sim_completion_ceiling"),
|
||||
"left": Plane("left", a0, low[a0], a1, up_axis, low[a1], high[a1], low[up_axis], high[up_axis], "real2sim_completion_wall"),
|
||||
"right": Plane("right", a0, high[a0], a1, up_axis, low[a1], high[a1], low[up_axis], high[up_axis], "real2sim_completion_wall"),
|
||||
"back": Plane("back", a1, low[a1], a0, up_axis, low[a0], high[a0], low[up_axis], high[up_axis], "real2sim_completion_wall"),
|
||||
"front": Plane("front", a1, high[a1], a0, up_axis, low[a0], high[a0], low[up_axis], high[up_axis], "real2sim_completion_wall"),
|
||||
}
|
||||
return [specs[name] for name in names if name in specs]
|
||||
|
||||
|
||||
def append_completion(
|
||||
input_obj: Path,
|
||||
output_obj: Path,
|
||||
mtl_path: Path,
|
||||
*,
|
||||
vertices_start: int,
|
||||
texcoords_start: int,
|
||||
planes: list[Plane],
|
||||
grid: int,
|
||||
) -> None:
|
||||
source = input_obj.read_text(encoding="utf-8", errors="ignore")
|
||||
output = [source]
|
||||
if not source.endswith("\n"):
|
||||
output.append("\n")
|
||||
output.append(f"mtllib {mtl_path.name}\n")
|
||||
output.append("o real2sim_scene_completion\n")
|
||||
vertex_index = vertices_start
|
||||
for plane in planes:
|
||||
output.append(f"g real2sim_completion_{plane.name}\n")
|
||||
output.append(f"usemtl {plane.material}\n")
|
||||
vertex_start = vertex_index + 1
|
||||
texcoord_start = texcoords_start + 1
|
||||
for iv in range(grid + 1):
|
||||
v_fraction = iv / grid
|
||||
v = lerp(plane.v_min, plane.v_max, v_fraction)
|
||||
for iu in range(grid + 1):
|
||||
u_fraction = iu / grid
|
||||
u = lerp(plane.u_min, plane.u_max, u_fraction)
|
||||
coords = [0.0, 0.0, 0.0]
|
||||
coords[plane.axis] = plane.value
|
||||
coords[plane.u_axis] = u
|
||||
coords[plane.v_axis] = v
|
||||
output.append(f"v {coords[0]:.8f} {coords[1]:.8f} {coords[2]:.8f}\n")
|
||||
output.append(f"vt {u_fraction:.8f} {v_fraction:.8f}\n")
|
||||
for iv in range(grid):
|
||||
for iu in range(grid):
|
||||
a = vertex_start + iv * (grid + 1) + iu
|
||||
b = a + 1
|
||||
c = a + (grid + 1)
|
||||
d = c + 1
|
||||
ta = texcoord_start + iv * (grid + 1) + iu
|
||||
tb = ta + 1
|
||||
tc = ta + (grid + 1)
|
||||
td = tc + 1
|
||||
output.append(f"f {a}/{ta} {b}/{tb} {d}/{td}\n")
|
||||
output.append(f"f {a}/{ta} {d}/{td} {c}/{tc}\n")
|
||||
vertex_index += (grid + 1) * (grid + 1)
|
||||
texcoords_start += (grid + 1) * (grid + 1)
|
||||
output_obj.write_text("".join(output), encoding="utf-8")
|
||||
write_completion_mtl(mtl_path)
|
||||
|
||||
|
||||
def write_completion_mtl(path: Path) -> None:
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"# Generated Real2Sim scene completion materials",
|
||||
"newmtl real2sim_completion_floor",
|
||||
"Kd 0.420000 0.410000 0.390000",
|
||||
"Ka 0.120000 0.120000 0.120000",
|
||||
"Ks 0.020000 0.020000 0.020000",
|
||||
"Ns 8.000000",
|
||||
"",
|
||||
"newmtl real2sim_completion_wall",
|
||||
"Kd 0.680000 0.650000 0.590000",
|
||||
"Ka 0.150000 0.145000 0.135000",
|
||||
"Ks 0.015000 0.015000 0.015000",
|
||||
"Ns 6.000000",
|
||||
"",
|
||||
"newmtl real2sim_completion_ceiling",
|
||||
"Kd 0.720000 0.700000 0.660000",
|
||||
"Ka 0.160000 0.155000 0.145000",
|
||||
"Ks 0.010000 0.010000 0.010000",
|
||||
"Ns 4.000000",
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def lerp(a: float, b: float, t: float) -> float:
|
||||
return a * (1.0 - t) + b * t
|
||||
|
||||
|
||||
def clamp(value: float, low: float, high: float) -> float:
|
||||
return min(max(value, low), high)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
461
vendor/gs_playground/scripts/real2sim_obj_structure_report.py
vendored
Normal file
461
vendor/gs_playground/scripts/real2sim_obj_structure_report.py
vendored
Normal file
@ -0,0 +1,461 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract coarse room/scene structure cues from an OBJ mesh."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Analyze axis-aligned scene structure candidates in an OBJ mesh")
|
||||
parser.add_argument("--input-obj", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--up-axis", choices=("x", "y", "z"), default="z")
|
||||
parser.add_argument("--normal-threshold", type=float, default=0.78)
|
||||
parser.add_argument("--support-normal-threshold", type=float, default=0.82)
|
||||
args = parser.parse_args()
|
||||
|
||||
report = analyze_obj_structure(
|
||||
input_obj=args.input_obj,
|
||||
up_axis=args.up_axis,
|
||||
normal_threshold=max(min(float(args.normal_threshold), 1.0), 0.0),
|
||||
support_normal_threshold=max(min(float(args.support_normal_threshold), 1.0), 0.0),
|
||||
)
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def analyze_obj_structure(
|
||||
*,
|
||||
input_obj: Path,
|
||||
up_axis: str = "z",
|
||||
normal_threshold: float = 0.78,
|
||||
support_normal_threshold: float = 0.82,
|
||||
) -> dict[str, Any]:
|
||||
vertices, faces = read_obj_triangles(input_obj)
|
||||
if len(vertices) < 3 or len(faces) < 1:
|
||||
raise SystemExit(f"OBJ has no usable triangle mesh: {input_obj}")
|
||||
points = vertices[faces]
|
||||
normals = triangle_normals(points)
|
||||
areas = triangle_areas(points)
|
||||
centroids = points.mean(axis=1)
|
||||
total_area = float(np.sum(areas))
|
||||
up_index = "xyz".index(up_axis)
|
||||
axis_reports = [
|
||||
axis_structure_report(
|
||||
axis=index,
|
||||
name=name,
|
||||
normals=normals,
|
||||
centroids=centroids,
|
||||
areas=areas,
|
||||
total_area=total_area,
|
||||
threshold=normal_threshold,
|
||||
)
|
||||
for index, name in enumerate("xyz")
|
||||
]
|
||||
support = support_plane_candidates(
|
||||
up_axis=up_index,
|
||||
normals=normals,
|
||||
centroids=centroids,
|
||||
areas=areas,
|
||||
total_area=total_area,
|
||||
threshold=support_normal_threshold,
|
||||
)
|
||||
bounds = {
|
||||
"min": [float(value) for value in vertices.min(axis=0)],
|
||||
"max": [float(value) for value in vertices.max(axis=0)],
|
||||
"p05": [float(np.percentile(vertices[:, idx], 5)) for idx in range(3)],
|
||||
"p95": [float(np.percentile(vertices[:, idx], 95)) for idx in range(3)],
|
||||
}
|
||||
semantic_planes = semantic_plane_candidates(
|
||||
up_axis=up_index,
|
||||
normals=normals,
|
||||
centroids=centroids,
|
||||
areas=areas,
|
||||
bounds=bounds,
|
||||
total_area=total_area,
|
||||
threshold=normal_threshold,
|
||||
)
|
||||
semantic_planes = add_recovered_floor_candidate(
|
||||
semantic_planes,
|
||||
up_axis=up_index,
|
||||
normals=normals,
|
||||
centroids=centroids,
|
||||
areas=areas,
|
||||
bounds=bounds,
|
||||
total_area=total_area,
|
||||
threshold=support_normal_threshold,
|
||||
)
|
||||
structure_score = min(
|
||||
1.0,
|
||||
sum(float(axis["area_ratio"]) for axis in axis_reports) / 3.0
|
||||
+ min(float(support["support_area_ratio"]) * 0.5, 0.25),
|
||||
)
|
||||
semantic_score = semantic_structure_score(semantic_planes)
|
||||
return {
|
||||
"format": "gs-playground-real2sim-obj-structure-report-v1",
|
||||
"input_obj": input_obj.resolve().as_posix(),
|
||||
"vertex_count": int(vertices.shape[0]),
|
||||
"face_count": int(faces.shape[0]),
|
||||
"total_area": total_area,
|
||||
"up_axis": up_axis,
|
||||
"bounds": bounds,
|
||||
"axis_aligned_planes": axis_reports,
|
||||
"support_planes": support,
|
||||
"semantic_planes": semantic_planes,
|
||||
"structure_score": round(float(structure_score), 6),
|
||||
"commercial_readiness": {
|
||||
"structure_score": round(float(structure_score), 6),
|
||||
"semantic_structure_score": round(float(semantic_score), 6),
|
||||
"has_floor": any(plane["semantic_label"] == "floor" for plane in semantic_planes),
|
||||
"has_wall": any(plane["semantic_label"].startswith("wall_") for plane in semantic_planes),
|
||||
"has_support_surface": any(plane["semantic_label"] == "support_surface" for plane in semantic_planes),
|
||||
"remaining_blockers": [
|
||||
"semantic instance segmentation for movable assets",
|
||||
"watertight hole filling and room-boundary regularization",
|
||||
"collision/nav mesh generation from semantic planes",
|
||||
"physically meaningful material parameters",
|
||||
],
|
||||
},
|
||||
"commercial_gap_notes": [
|
||||
"This report now adds coarse semantic structure candidates, but it is not full room understanding.",
|
||||
"Commercial-grade Real2Sim still needs instance semantics, watertight reconstruction, collision/nav mesh derivation, and physical material inference.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def axis_structure_report(
|
||||
*,
|
||||
axis: int,
|
||||
name: str,
|
||||
normals: np.ndarray,
|
||||
centroids: np.ndarray,
|
||||
areas: np.ndarray,
|
||||
total_area: float,
|
||||
threshold: float,
|
||||
) -> dict[str, Any]:
|
||||
aligned = np.abs(normals[:, axis]) >= threshold
|
||||
area = float(np.sum(areas[aligned])) if np.any(aligned) else 0.0
|
||||
values = centroids[aligned, axis] if np.any(aligned) else np.asarray([], dtype=np.float64)
|
||||
return {
|
||||
"axis": name,
|
||||
"aligned_face_count": int(np.count_nonzero(aligned)),
|
||||
"area": area,
|
||||
"area_ratio": round((area / total_area) if total_area > 0 else 0.0, 6),
|
||||
"low_plane": plane_summary(values, areas[aligned], quantile=5),
|
||||
"high_plane": plane_summary(values, areas[aligned], quantile=95),
|
||||
}
|
||||
|
||||
|
||||
def support_plane_candidates(
|
||||
*,
|
||||
up_axis: int,
|
||||
normals: np.ndarray,
|
||||
centroids: np.ndarray,
|
||||
areas: np.ndarray,
|
||||
total_area: float,
|
||||
threshold: float,
|
||||
) -> dict[str, Any]:
|
||||
upward = np.abs(normals[:, up_axis]) >= threshold
|
||||
values = centroids[upward, up_axis] if np.any(upward) else np.asarray([], dtype=np.float64)
|
||||
support_areas = areas[upward] if np.any(upward) else np.asarray([], dtype=np.float64)
|
||||
low = plane_summary(values, support_areas, quantile=10)
|
||||
high = plane_summary(values, support_areas, quantile=75)
|
||||
support_area = float(np.sum(support_areas))
|
||||
return {
|
||||
"up_axis": "xyz"[up_axis],
|
||||
"support_face_count": int(np.count_nonzero(upward)),
|
||||
"support_area": support_area,
|
||||
"support_area_ratio": round((support_area / total_area) if total_area > 0 else 0.0, 6),
|
||||
"floor_like": low,
|
||||
"tabletop_or_shelf_like": high,
|
||||
}
|
||||
|
||||
|
||||
def semantic_plane_candidates(
|
||||
*,
|
||||
up_axis: int,
|
||||
normals: np.ndarray,
|
||||
centroids: np.ndarray,
|
||||
areas: np.ndarray,
|
||||
bounds: dict[str, list[float]],
|
||||
total_area: float,
|
||||
threshold: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
planes: list[dict[str, Any]] = []
|
||||
for axis, name in enumerate("xyz"):
|
||||
axis_span = max(float(bounds["p95"][axis] - bounds["p05"][axis]), 1e-6)
|
||||
clusters = clustered_axis_planes(
|
||||
axis=axis,
|
||||
normals=normals,
|
||||
centroids=centroids,
|
||||
areas=areas,
|
||||
threshold=threshold,
|
||||
cluster_tolerance=max(axis_span * 0.035, 0.04),
|
||||
)
|
||||
for cluster in clusters:
|
||||
label = semantic_label_for_plane(axis=axis, up_axis=up_axis, coordinate=cluster["coordinate"], bounds=bounds)
|
||||
if label == "vertical_partition" and cluster["area_ratio"] >= 0.05:
|
||||
label = "wall_visible"
|
||||
if label == "support_surface" and cluster["area_ratio"] < 0.01:
|
||||
continue
|
||||
if label.startswith("wall_") and cluster["area_ratio"] < 0.015:
|
||||
continue
|
||||
planes.append(
|
||||
{
|
||||
"semantic_label": label,
|
||||
"axis": name,
|
||||
"coordinate": round(float(cluster["coordinate"]), 6),
|
||||
"area": cluster["area"],
|
||||
"area_ratio": round(float(cluster["area_ratio"]), 6),
|
||||
"face_count": cluster["face_count"],
|
||||
"extent": cluster["extent"],
|
||||
"collision_proxy_hint": collision_proxy_hint(
|
||||
label=label,
|
||||
axis=axis,
|
||||
coordinate=float(cluster["coordinate"]),
|
||||
extent=cluster["extent"],
|
||||
bounds=bounds,
|
||||
),
|
||||
}
|
||||
)
|
||||
planes.sort(key=lambda item: (semantic_priority(item["semantic_label"]), -float(item["area_ratio"])))
|
||||
return planes[:16]
|
||||
|
||||
|
||||
def add_recovered_floor_candidate(
|
||||
planes: list[dict[str, Any]],
|
||||
*,
|
||||
up_axis: int,
|
||||
normals: np.ndarray,
|
||||
centroids: np.ndarray,
|
||||
areas: np.ndarray,
|
||||
bounds: dict[str, list[float]],
|
||||
total_area: float,
|
||||
threshold: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
if any(plane["semantic_label"] == "floor" for plane in planes):
|
||||
return planes
|
||||
candidate = recovered_floor_candidate(
|
||||
up_axis=up_axis,
|
||||
normals=normals,
|
||||
centroids=centroids,
|
||||
areas=areas,
|
||||
bounds=bounds,
|
||||
total_area=total_area,
|
||||
threshold=threshold,
|
||||
)
|
||||
if candidate is None:
|
||||
return planes
|
||||
updated = [candidate, *planes]
|
||||
updated.sort(key=lambda item: (semantic_priority(item["semantic_label"]), -float(item["area_ratio"])))
|
||||
return updated[:16]
|
||||
|
||||
|
||||
def recovered_floor_candidate(
|
||||
*,
|
||||
up_axis: int,
|
||||
normals: np.ndarray,
|
||||
centroids: np.ndarray,
|
||||
areas: np.ndarray,
|
||||
bounds: dict[str, list[float]],
|
||||
total_area: float,
|
||||
threshold: float,
|
||||
) -> dict[str, Any] | None:
|
||||
upward = np.abs(normals[:, up_axis]) >= threshold
|
||||
if not np.any(upward):
|
||||
return None
|
||||
up_values = centroids[upward, up_axis]
|
||||
up_areas = areas[upward]
|
||||
low = float(bounds["p05"][up_axis])
|
||||
high = float(bounds["p95"][up_axis])
|
||||
span = max(high - low, 1e-6)
|
||||
coordinate = float(np.percentile(up_values, 10))
|
||||
if coordinate > low + span * 0.18:
|
||||
return None
|
||||
near = np.abs(up_values - coordinate) <= max(span * 0.025, 0.035)
|
||||
if not np.any(near):
|
||||
return None
|
||||
near_points = centroids[upward][near]
|
||||
near_area = float(np.sum(up_areas[near]))
|
||||
area_ratio = near_area / total_area if total_area > 0 else 0.0
|
||||
if area_ratio < 0.012 or near_points.shape[0] < 80:
|
||||
return None
|
||||
extent = {
|
||||
name: [float(np.percentile(near_points[:, idx], 5)), float(np.percentile(near_points[:, idx], 95))]
|
||||
for idx, name in enumerate("xyz")
|
||||
}
|
||||
return {
|
||||
"semantic_label": "floor",
|
||||
"axis": "xyz"[up_axis],
|
||||
"coordinate": round(coordinate, 6),
|
||||
"area": near_area,
|
||||
"area_ratio": round(float(area_ratio), 6),
|
||||
"face_count": int(near_points.shape[0]),
|
||||
"extent": extent,
|
||||
"recovery_method": "low_horizontal_support_quantile",
|
||||
"recovery_confidence": round(min(1.0, area_ratio / 0.05), 6),
|
||||
"collision_proxy_hint": collision_proxy_hint(
|
||||
label="floor",
|
||||
axis=up_axis,
|
||||
coordinate=coordinate,
|
||||
extent=extent,
|
||||
bounds=bounds,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def clustered_axis_planes(
|
||||
*,
|
||||
axis: int,
|
||||
normals: np.ndarray,
|
||||
centroids: np.ndarray,
|
||||
areas: np.ndarray,
|
||||
threshold: float,
|
||||
cluster_tolerance: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
aligned = np.abs(normals[:, axis]) >= threshold
|
||||
if not np.any(aligned):
|
||||
return []
|
||||
aligned_centroids = centroids[aligned]
|
||||
aligned_areas = areas[aligned]
|
||||
order = np.argsort(aligned_centroids[:, axis])
|
||||
sorted_centroids = aligned_centroids[order]
|
||||
sorted_areas = aligned_areas[order]
|
||||
clusters: list[dict[str, Any]] = []
|
||||
start = 0
|
||||
for index in range(1, sorted_centroids.shape[0] + 1):
|
||||
boundary = index == sorted_centroids.shape[0]
|
||||
if not boundary:
|
||||
boundary = float(sorted_centroids[index, axis] - sorted_centroids[index - 1, axis]) > cluster_tolerance
|
||||
if boundary:
|
||||
points = sorted_centroids[start:index]
|
||||
weights = sorted_areas[start:index]
|
||||
area = float(np.sum(weights))
|
||||
if points.size > 0 and area > 0:
|
||||
clusters.append(
|
||||
{
|
||||
"coordinate": float(np.average(points[:, axis], weights=weights)),
|
||||
"area": area,
|
||||
"area_ratio": area / float(np.sum(areas)) if np.sum(areas) > 0 else 0.0,
|
||||
"face_count": int(points.shape[0]),
|
||||
"extent": {
|
||||
name: [float(np.percentile(points[:, idx], 5)), float(np.percentile(points[:, idx], 95))]
|
||||
for idx, name in enumerate("xyz")
|
||||
},
|
||||
}
|
||||
)
|
||||
start = index
|
||||
clusters.sort(key=lambda item: item["area"], reverse=True)
|
||||
return clusters[:8]
|
||||
|
||||
|
||||
def semantic_label_for_plane(*, axis: int, up_axis: int, coordinate: float, bounds: dict[str, list[float]]) -> str:
|
||||
low = float(bounds["p05"][axis])
|
||||
high = float(bounds["p95"][axis])
|
||||
span = max(high - low, 1e-6)
|
||||
if axis == up_axis:
|
||||
if coordinate <= low + span * 0.12:
|
||||
return "floor"
|
||||
if coordinate >= high - span * 0.12:
|
||||
return "ceiling"
|
||||
return "support_surface"
|
||||
if coordinate <= low + span * 0.18:
|
||||
return f"wall_{'xyz'[axis]}_low"
|
||||
if coordinate >= high - span * 0.18:
|
||||
return f"wall_{'xyz'[axis]}_high"
|
||||
return "vertical_partition"
|
||||
|
||||
|
||||
def collision_proxy_hint(
|
||||
*,
|
||||
label: str,
|
||||
axis: int,
|
||||
coordinate: float,
|
||||
extent: dict[str, list[float]],
|
||||
bounds: dict[str, list[float]],
|
||||
) -> dict[str, Any]:
|
||||
center = [(float(extent[name][0]) + float(extent[name][1])) * 0.5 for name in "xyz"]
|
||||
size = [max(float(extent[name][1]) - float(extent[name][0]), 0.02) for name in "xyz"]
|
||||
center[axis] = coordinate
|
||||
size[axis] = max(float(bounds["p95"][axis] - bounds["p05"][axis]) * 0.01, 0.02)
|
||||
return {
|
||||
"type": "box",
|
||||
"label": label,
|
||||
"center": [round(value, 6) for value in center],
|
||||
"size": [round(value, 6) for value in size],
|
||||
}
|
||||
|
||||
|
||||
def semantic_priority(label: str) -> int:
|
||||
if label == "floor":
|
||||
return 0
|
||||
if label.startswith("wall_"):
|
||||
return 1
|
||||
if label == "support_surface":
|
||||
return 2
|
||||
if label == "ceiling":
|
||||
return 3
|
||||
return 4
|
||||
|
||||
|
||||
def semantic_structure_score(planes: list[dict[str, Any]]) -> float:
|
||||
has_floor = any(plane["semantic_label"] == "floor" for plane in planes)
|
||||
wall_count = sum(1 for plane in planes if plane["semantic_label"].startswith("wall_"))
|
||||
support_count = sum(1 for plane in planes if plane["semantic_label"] == "support_surface")
|
||||
return min(1.0, (0.35 if has_floor else 0.0) + min(wall_count, 4) * 0.12 + min(support_count, 3) * 0.08)
|
||||
|
||||
|
||||
def plane_summary(values: np.ndarray, areas: np.ndarray, *, quantile: float) -> dict[str, Any] | None:
|
||||
if values.size == 0:
|
||||
return None
|
||||
value = float(np.percentile(values, quantile))
|
||||
span = max(float(np.percentile(values, 95) - np.percentile(values, 5)), 1e-6)
|
||||
near = np.abs(values - value) <= max(span * 0.04, 0.02)
|
||||
near_area = float(np.sum(areas[near])) if np.any(near) else 0.0
|
||||
return {
|
||||
"coordinate": value,
|
||||
"quantile": float(quantile),
|
||||
"near_face_count": int(np.count_nonzero(near)),
|
||||
"near_area": near_area,
|
||||
}
|
||||
|
||||
|
||||
def read_obj_triangles(path: Path) -> tuple[np.ndarray, np.ndarray]:
|
||||
vertices: list[tuple[float, float, float]] = []
|
||||
faces: list[tuple[int, int, int]] = []
|
||||
for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
if raw.startswith("v "):
|
||||
parts = raw.split()
|
||||
if len(parts) >= 4:
|
||||
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
||||
elif raw.startswith("f "):
|
||||
refs = [parse_vertex_ref(token) for token in raw.split()[1:]]
|
||||
for idx in range(1, len(refs) - 1):
|
||||
faces.append((refs[0], refs[idx], refs[idx + 1]))
|
||||
return np.asarray(vertices, dtype=np.float64), np.asarray(faces, dtype=np.int64)
|
||||
|
||||
|
||||
def parse_vertex_ref(token: str) -> int:
|
||||
return int(token.split("/")[0]) - 1
|
||||
|
||||
|
||||
def triangle_normals(points: np.ndarray) -> np.ndarray:
|
||||
normals = np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0])
|
||||
norms = np.linalg.norm(normals, axis=1, keepdims=True)
|
||||
return normals / np.maximum(norms, 1e-9)
|
||||
|
||||
|
||||
def triangle_areas(points: np.ndarray) -> np.ndarray:
|
||||
return np.linalg.norm(np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0]), axis=1) * 0.5
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
910
vendor/gs_playground/scripts/real2sim_one_image_motrix_scene.py
vendored
Normal file
910
vendor/gs_playground/scripts/real2sim_one_image_motrix_scene.py
vendored
Normal file
@ -0,0 +1,910 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-image Real2Sim scene reconstruction workflow for MotrixSim.
|
||||
|
||||
This script is intentionally orchestration-only: it does not store API keys and
|
||||
does not replace the project CLI. It chains the existing image-edit multiview
|
||||
adapter, scene generation CLI, package verification, preview projection, and
|
||||
archive creation into one reproducible command.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
PIL_BILINEAR = getattr(getattr(Image, "Resampling", Image), "BILINEAR")
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if REPO_ROOT.as_posix() not in sys.path:
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.camera import load_anysplat_source_camera
|
||||
|
||||
|
||||
DEFAULT_PROMPT = (
|
||||
"Generate a reconstruction-grade, geometry-consistent 8-view image-to-image contact sheet for Real2Sim and MotrixSim import. "
|
||||
"Use the input photo as exact source truth. Preserve all visible object identities, relative spacing, material colors, reflections, "
|
||||
"shadows, the supporting surface, static background, camera scale, and lighting. Output exactly one 4 columns x 2 rows contact sheet, "
|
||||
"8 equal portrait panels, thin white gutters, no labels, no captions, no extra text, no watermarks. Views in order: original front, "
|
||||
"front-left 45 degrees, left 90 degrees, rear-left 135 degrees, rear/back 180 degrees, rear-right 225 degrees, right 270 degrees, "
|
||||
"top oblique 35 degrees. Do not add people, hands, robots, extra objects, UI, a different room, or a product render background. "
|
||||
"Prefer sharp contours and reconstruction-useful texture over artistic beautification."
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate and verify a MotrixSim-importable scene package from one image")
|
||||
parser.add_argument("--image", type=Path, required=True, help="Source RGB image")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output Real2Sim scene package directory")
|
||||
parser.add_argument("--work-dir", type=Path, required=True, help="Intermediate workflow directory")
|
||||
parser.add_argument("--prompt", default=DEFAULT_PROMPT, help="Image-to-image multiview prompt")
|
||||
parser.add_argument("--scene-prompt", default="", help="Scene reconstruction prompt; defaults to --prompt")
|
||||
parser.add_argument("--config", type=Path, default=REPO_ROOT / "configs/real2sim.external.example.json")
|
||||
parser.add_argument("--backend", default="builtin", choices=["builtin", "external", "hybrid", "auto", "anysplat-direct"])
|
||||
parser.add_argument("--api-key-env", default="REAL2SIM_IMAGE_EDIT_API_KEY")
|
||||
parser.add_argument("--api-url", default=os.environ.get("REAL2SIM_IMAGE_EDIT_API_URL", "https://chatapi.applyma.com/v1/images/edits"))
|
||||
parser.add_argument("--model", default=os.environ.get("REAL2SIM_IMAGE_EDIT_MODEL", "gpt-image-2"))
|
||||
parser.add_argument("--size", default=os.environ.get("REAL2SIM_IMAGE_EDIT_SIZE", "1536x1024"))
|
||||
parser.add_argument("--quality", default=os.environ.get("REAL2SIM_IMAGE_EDIT_QUALITY", "auto"))
|
||||
parser.add_argument(
|
||||
"--candidates",
|
||||
type=int,
|
||||
default=int(os.environ.get("REAL2SIM_IMAGE_EDIT_CANDIDATES", "1")),
|
||||
help="Generate and score multiple image-edit multiview candidates, then reconstruct the best one",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-quality-score",
|
||||
type=float,
|
||||
default=float(os.environ.get("REAL2SIM_MIN_MULTIVIEW_QUALITY", "0.42")),
|
||||
help="Fail before reconstruction when synthetic multiview quality is below this score",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-low-quality",
|
||||
action="store_true",
|
||||
help="Continue even when the best synthetic multiview candidate is below --min-quality-score",
|
||||
)
|
||||
parser.add_argument("--frames-dir", type=Path, default=None, help="Reuse existing multiview frames instead of calling the image API")
|
||||
parser.add_argument("--no-image-edit", action="store_true", help="Require --frames-dir and skip the image-edit API call")
|
||||
parser.add_argument("--no-prune", action="store_true", help="Pass --no-prune to scene generation")
|
||||
parser.add_argument("--anysplat-python", default=os.environ.get("REAL2SIM_ANY_SPLAT_PYTHON", ""), help="Python executable for AnySplat direct mode")
|
||||
parser.add_argument("--anysplat-root", type=Path, default=os.environ.get("REAL2SIM_ANYSPLAT_ROOT"), help="AnySplat repository root for direct mode")
|
||||
parser.add_argument("--anysplat-max-views", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MAX_VIEWS", "8")))
|
||||
parser.add_argument("--anysplat-mesh-grid", type=int, default=int(os.environ.get("REAL2SIM_ANYSPLAT_MESH_GRID", "48")))
|
||||
parser.add_argument("--source-gs-preview", type=Path, default=None, help="Optional source-view 3DGS preview PNG to include in the Motrix package")
|
||||
parser.add_argument(
|
||||
"--min-source-preview-psnr",
|
||||
type=float,
|
||||
default=float(os.environ.get("REAL2SIM_MIN_SOURCE_PREVIEW_PSNR", "18.0")),
|
||||
help="Fail AnySplat direct mode when source-view render PSNR is below this threshold",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-source-preview-mae",
|
||||
type=float,
|
||||
default=float(os.environ.get("REAL2SIM_MAX_SOURCE_PREVIEW_MAE", "0.10")),
|
||||
help="Fail AnySplat direct mode when source-view render MAE is above this threshold",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-scene-vertices",
|
||||
type=int,
|
||||
default=int(os.environ.get("REAL2SIM_MIN_SCENE_VERTICES", "100000")),
|
||||
help="Fail final package verification when scene.ply has fewer vertices",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-final-package-verify",
|
||||
action="store_true",
|
||||
help="Skip final Motrix package archive verification",
|
||||
)
|
||||
parser.add_argument("--archive", type=Path, default=None, help="Optional .tar.gz path; defaults to <output>.tar.gz")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.image.is_file():
|
||||
raise SystemExit(f"source image not found: {args.image}")
|
||||
args.work_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
frames_dir = args.frames_dir
|
||||
multiview_dir = args.work_dir / "synthetic_multiview"
|
||||
quality_report = args.work_dir / "synthetic_multiview_quality.json"
|
||||
if args.no_image_edit:
|
||||
if frames_dir is None:
|
||||
raise SystemExit("--no-image-edit requires --frames-dir")
|
||||
elif frames_dir is None:
|
||||
frames_dir = run_best_image_edit_candidate(args, multiview_dir, quality_report)
|
||||
|
||||
if frames_dir is None or not frames_dir.is_dir():
|
||||
raise SystemExit(f"frames directory not found: {frames_dir}")
|
||||
if not quality_report.exists():
|
||||
write_multiview_quality_report(args.image, frames_dir, quality_report)
|
||||
|
||||
anysplat_metadata = None
|
||||
if args.backend == "anysplat-direct":
|
||||
anysplat_metadata = run_anysplat_direct_generation(args, frames_dir)
|
||||
else:
|
||||
run_scene_generation(args, frames_dir)
|
||||
report = verify_motrix_package(args.output)
|
||||
preview = write_projection_preview(args.output / "3dgs" / "scene.ply", args.output / "pointcloud_projection_preview.png")
|
||||
if args.backend == "anysplat-direct" and (args.output / "motrix_import.json").is_file():
|
||||
import_files = {
|
||||
"manifest": args.output / "motrix_import.json",
|
||||
"readme": args.output / "MOTRIX_IMPORT.md",
|
||||
}
|
||||
copy_quality_report(args.output, quality_report)
|
||||
if anysplat_metadata is not None:
|
||||
attach_anysplat_metadata_to_package(args.output, anysplat_metadata)
|
||||
else:
|
||||
import_files = write_motrix_import_files(args.output, report, preview, quality_report)
|
||||
archive = create_archive(args.output, args.archive or args.output.with_suffix(".tar.gz"))
|
||||
final_verification = None
|
||||
if not args.skip_final_package_verify and args.backend == "anysplat-direct":
|
||||
final_verification = run_final_package_verification(args, archive)
|
||||
archive = create_archive(args.output, archive)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"package": report,
|
||||
"preview": preview.as_posix(),
|
||||
"motrix_import": import_files["manifest"].as_posix(),
|
||||
"motrix_readme": import_files["readme"].as_posix(),
|
||||
"archive": archive.as_posix(),
|
||||
"anysplat_metadata": (args.output / "artifacts" / "anysplat_metadata.json").as_posix()
|
||||
if (args.output / "artifacts" / "anysplat_metadata.json").is_file()
|
||||
else None,
|
||||
"final_package_verification": final_verification.as_posix() if final_verification is not None else None,
|
||||
},
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def run_final_package_verification(args: argparse.Namespace, archive: Path) -> Path:
|
||||
report_path = args.output / "motrix_package_verification.json"
|
||||
command = [
|
||||
sys.executable,
|
||||
"scripts/real2sim_verify_motrix_package.py",
|
||||
"--package",
|
||||
archive.as_posix(),
|
||||
"--min-scene-vertices",
|
||||
str(args.min_scene_vertices),
|
||||
"--min-source-preview-psnr",
|
||||
str(args.min_source_preview_psnr),
|
||||
"--max-source-preview-mae",
|
||||
str(args.max_source_preview_mae),
|
||||
]
|
||||
completed = subprocess.run(command, cwd=REPO_ROOT, text=True, capture_output=True)
|
||||
if completed.returncode != 0:
|
||||
if completed.stdout:
|
||||
print(completed.stdout, file=sys.stderr)
|
||||
if completed.stderr:
|
||||
print(completed.stderr, file=sys.stderr)
|
||||
raise SystemExit(completed.returncode)
|
||||
report_path.write_text(completed.stdout, encoding="utf-8")
|
||||
return report_path
|
||||
|
||||
|
||||
def run_best_image_edit_candidate(args: argparse.Namespace, multiview_dir: Path, quality_report: Path) -> Path:
|
||||
if args.candidates <= 0:
|
||||
raise SystemExit("--candidates must be positive")
|
||||
candidate_reports: list[dict[str, Any]] = []
|
||||
best_report: dict[str, Any] | None = None
|
||||
best_frames: Path | None = None
|
||||
for index in range(args.candidates):
|
||||
suffix = f"candidate_{index + 1:02d}" if args.candidates > 1 else "candidate_01"
|
||||
candidate_dir = multiview_dir / suffix
|
||||
frames_dir = candidate_dir / "frames"
|
||||
run_image_edit(args, candidate_dir, frames_dir)
|
||||
report = score_multiview_frames(args.image, frames_dir)
|
||||
report["candidate"] = index + 1
|
||||
report["frames_dir"] = frames_dir.as_posix()
|
||||
report["contact_sheet"] = (candidate_dir / "contact_sheet.png").as_posix()
|
||||
candidate_reports.append(report)
|
||||
if best_report is None or float(report["score"]) > float(best_report["score"]):
|
||||
best_report = report
|
||||
best_frames = frames_dir
|
||||
|
||||
if best_report is None or best_frames is None:
|
||||
raise SystemExit("image-edit API produced no synthetic multiview candidates")
|
||||
quality = {
|
||||
"selected_candidate": best_report["candidate"],
|
||||
"selected_frames_dir": best_frames.as_posix(),
|
||||
"min_quality_score": args.min_quality_score,
|
||||
"passed": float(best_report["score"]) >= args.min_quality_score,
|
||||
"candidates": candidate_reports,
|
||||
}
|
||||
quality_report.parent.mkdir(parents=True, exist_ok=True)
|
||||
quality_report.write_text(json.dumps(quality, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if not quality["passed"] and not args.allow_low_quality:
|
||||
raise SystemExit(
|
||||
"synthetic multiview quality gate failed: "
|
||||
f"best score {float(best_report['score']):.3f} < {args.min_quality_score:.3f}; "
|
||||
"inspect synthetic_multiview_quality.json or rerun with --allow-low-quality"
|
||||
)
|
||||
return best_frames
|
||||
|
||||
|
||||
def run_image_edit(args: argparse.Namespace, multiview_dir: Path, frames_dir: Path) -> None:
|
||||
api_key = os.environ.get(args.api_key_env)
|
||||
if not api_key:
|
||||
raise SystemExit(f"missing image-edit API key: set {args.api_key_env}")
|
||||
command = [
|
||||
sys.executable,
|
||||
"scripts/real2sim_external/synthetic_multiview_expand.py",
|
||||
"--image",
|
||||
args.image.as_posix(),
|
||||
"--output-dir",
|
||||
multiview_dir.as_posix(),
|
||||
"--frames-dir",
|
||||
frames_dir.as_posix(),
|
||||
"--mode",
|
||||
"scene",
|
||||
"--api-url",
|
||||
args.api_url,
|
||||
"--model",
|
||||
args.model,
|
||||
"--size",
|
||||
args.size,
|
||||
"--quality",
|
||||
args.quality,
|
||||
"--prompt",
|
||||
args.prompt,
|
||||
]
|
||||
env = os.environ.copy()
|
||||
env["REAL2SIM_IMAGE_EDIT_API_KEY"] = api_key
|
||||
run(command, env=env)
|
||||
|
||||
|
||||
def write_multiview_quality_report(source_image: Path, frames_dir: Path, output: Path) -> None:
|
||||
report = score_multiview_frames(source_image, frames_dir)
|
||||
quality = {
|
||||
"selected_candidate": None,
|
||||
"selected_frames_dir": frames_dir.as_posix(),
|
||||
"min_quality_score": None,
|
||||
"passed": True,
|
||||
"candidates": [report],
|
||||
}
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(quality, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def score_multiview_frames(source_image: Path, frames_dir: Path) -> dict[str, Any]:
|
||||
frame_paths = sorted(
|
||||
path
|
||||
for path in frames_dir.iterdir()
|
||||
if path.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp", ".bmp"}
|
||||
)
|
||||
frames = [Image.open(path).convert("RGB") for path in frame_paths]
|
||||
source = Image.open(source_image).convert("RGB")
|
||||
|
||||
frame_count_score = min(len(frames), 8) / 8.0
|
||||
resolution_score = 0.0
|
||||
exposure_score = 0.0
|
||||
sharpness_score = 0.0
|
||||
if frames:
|
||||
resolution_score = float(np.mean([min((img.width * img.height) / (360 * 360), 1.0) for img in frames]))
|
||||
exposure_score = float(np.mean([exposure_quality(img) for img in frames]))
|
||||
sharpness_score = float(np.mean([sharpness_quality(img) for img in frames]))
|
||||
source_similarity = first_view_similarity(source, frames[0]) if frames else 0.0
|
||||
diversity_score = view_diversity(frames)
|
||||
|
||||
score = (
|
||||
0.20 * frame_count_score
|
||||
+ 0.12 * resolution_score
|
||||
+ 0.18 * source_similarity
|
||||
+ 0.22 * diversity_score
|
||||
+ 0.18 * sharpness_score
|
||||
+ 0.10 * exposure_score
|
||||
)
|
||||
return {
|
||||
"score": round(float(score), 4),
|
||||
"frame_count": len(frames),
|
||||
"frame_count_score": round(float(frame_count_score), 4),
|
||||
"resolution_score": round(float(resolution_score), 4),
|
||||
"source_similarity": round(float(source_similarity), 4),
|
||||
"view_diversity": round(float(diversity_score), 4),
|
||||
"sharpness_score": round(float(sharpness_score), 4),
|
||||
"exposure_score": round(float(exposure_score), 4),
|
||||
"frames": [path.as_posix() for path in frame_paths],
|
||||
"warnings": multiview_quality_warnings(
|
||||
frame_count=len(frames),
|
||||
source_similarity=source_similarity,
|
||||
diversity_score=diversity_score,
|
||||
sharpness_score=sharpness_score,
|
||||
exposure_score=exposure_score,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def image_array(image: Image.Image, size: tuple[int, int] = (160, 120)) -> np.ndarray:
|
||||
return np.asarray(image.resize(size, PIL_BILINEAR), dtype=np.float32) / 255.0
|
||||
|
||||
|
||||
def first_view_similarity(source: Image.Image, first_view: Image.Image) -> float:
|
||||
src = image_array(source)
|
||||
view = image_array(first_view)
|
||||
mean_abs = float(np.mean(np.abs(src - view)))
|
||||
color_delta = float(np.mean(np.abs(src.mean(axis=(0, 1)) - view.mean(axis=(0, 1)))))
|
||||
return max(0.0, min(1.0, 1.0 - (1.7 * mean_abs + 0.8 * color_delta)))
|
||||
|
||||
|
||||
def view_diversity(frames: list[Image.Image]) -> float:
|
||||
if len(frames) < 2:
|
||||
return 0.0
|
||||
arrays = [image_array(frame, size=(96, 72)) for frame in frames]
|
||||
diffs = [float(np.mean(np.abs(arrays[index] - arrays[index - 1]))) for index in range(1, len(arrays))]
|
||||
mean_diff = float(np.mean(diffs))
|
||||
if mean_diff < 0.035:
|
||||
return mean_diff / 0.035 * 0.45
|
||||
if mean_diff > 0.42:
|
||||
return max(0.0, 1.0 - (mean_diff - 0.42) / 0.30)
|
||||
return min(1.0, 0.45 + (mean_diff - 0.035) / (0.42 - 0.035) * 0.55)
|
||||
|
||||
|
||||
def sharpness_quality(image: Image.Image) -> float:
|
||||
gray = np.asarray(image.convert("L").resize((192, 144), PIL_BILINEAR), dtype=np.float32) / 255.0
|
||||
gx = np.diff(gray, axis=1)
|
||||
gy = np.diff(gray, axis=0)
|
||||
energy = float(np.mean(np.abs(gx)) + np.mean(np.abs(gy)))
|
||||
return max(0.0, min(1.0, energy / 0.11))
|
||||
|
||||
|
||||
def exposure_quality(image: Image.Image) -> float:
|
||||
gray = np.asarray(image.convert("L").resize((96, 72), PIL_BILINEAR), dtype=np.float32) / 255.0
|
||||
clipped = float(np.mean((gray < 0.025) | (gray > 0.975)))
|
||||
contrast = float(np.std(gray))
|
||||
return max(0.0, min(1.0, (1.0 - clipped * 2.0) * min(contrast / 0.22, 1.0)))
|
||||
|
||||
|
||||
def multiview_quality_warnings(
|
||||
*,
|
||||
frame_count: int,
|
||||
source_similarity: float,
|
||||
diversity_score: float,
|
||||
sharpness_score: float,
|
||||
exposure_score: float,
|
||||
) -> list[str]:
|
||||
warnings: list[str] = []
|
||||
if frame_count < 8:
|
||||
warnings.append("expected 8 synthetic views")
|
||||
if source_similarity < 0.35:
|
||||
warnings.append("first synthetic view may not preserve the source image")
|
||||
if diversity_score < 0.35:
|
||||
warnings.append("synthetic views may be too similar for scene reconstruction")
|
||||
if sharpness_score < 0.35:
|
||||
warnings.append("synthetic views look blurry or low-detail")
|
||||
if exposure_score < 0.35:
|
||||
warnings.append("synthetic views have weak contrast or clipped exposure")
|
||||
return warnings
|
||||
|
||||
|
||||
def run_scene_generation(args: argparse.Namespace, frames_dir: Path) -> None:
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"real2sim",
|
||||
"generate-scene",
|
||||
"--input",
|
||||
frames_dir.as_posix(),
|
||||
"--output",
|
||||
args.output.as_posix(),
|
||||
"--work-dir",
|
||||
(args.work_dir / "scene_work").as_posix(),
|
||||
"--backend",
|
||||
args.backend,
|
||||
"--config",
|
||||
args.config.as_posix(),
|
||||
"--prompt",
|
||||
args.scene_prompt or args.prompt,
|
||||
]
|
||||
if args.no_prune:
|
||||
command.append("--no-prune")
|
||||
run(command)
|
||||
|
||||
|
||||
def run_anysplat_direct_generation(args: argparse.Namespace, frames_dir: Path) -> Path | None:
|
||||
scene_dir = args.work_dir / "anysplat_scene"
|
||||
scene_dir.mkdir(parents=True, exist_ok=True)
|
||||
scene_ply = scene_dir / "scene.ply"
|
||||
scene_mesh = scene_dir / "mesh.obj"
|
||||
cameras_json = scene_dir / "cameras.json"
|
||||
depth_map = scene_dir / "depth.pgm"
|
||||
source_preview = args.source_gs_preview or (scene_dir / "source_gs_preview.png")
|
||||
anysplat_python = args.anysplat_python or sys.executable
|
||||
|
||||
command = [
|
||||
anysplat_python,
|
||||
"scripts/real2sim_external/anysplat_scene.py",
|
||||
"--image",
|
||||
args.image.as_posix(),
|
||||
"--frames-dir",
|
||||
frames_dir.as_posix(),
|
||||
"--output-dir",
|
||||
scene_dir.as_posix(),
|
||||
"--ply",
|
||||
scene_ply.as_posix(),
|
||||
"--mesh",
|
||||
scene_mesh.as_posix(),
|
||||
"--cameras",
|
||||
cameras_json.as_posix(),
|
||||
"--depth",
|
||||
depth_map.as_posix(),
|
||||
"--source-preview",
|
||||
source_preview.as_posix(),
|
||||
"--max-views",
|
||||
str(args.anysplat_max_views),
|
||||
"--mesh-grid",
|
||||
str(max(8, int(getattr(args, "anysplat_mesh_grid", 48)))),
|
||||
]
|
||||
if args.anysplat_root is not None:
|
||||
command.extend(["--anysplat-root", args.anysplat_root.as_posix()])
|
||||
run(command)
|
||||
metadata = scene_dir / "anysplat_metadata.json"
|
||||
if metadata.exists():
|
||||
enforce_anysplat_preview_quality(
|
||||
metadata,
|
||||
min_psnr=args.min_source_preview_psnr,
|
||||
max_mae=args.max_source_preview_mae,
|
||||
allow_low_quality=args.allow_low_quality,
|
||||
)
|
||||
|
||||
package_command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"real2sim",
|
||||
"package-existing",
|
||||
"--source",
|
||||
args.image.as_posix(),
|
||||
"--scene-ply",
|
||||
scene_ply.as_posix(),
|
||||
"--scene-mesh",
|
||||
scene_mesh.as_posix(),
|
||||
"--cameras-json",
|
||||
cameras_json.as_posix(),
|
||||
"--depth-map",
|
||||
depth_map.as_posix(),
|
||||
"--background-image",
|
||||
args.image.as_posix(),
|
||||
"--output",
|
||||
args.output.as_posix(),
|
||||
"--prompt",
|
||||
args.scene_prompt or args.prompt,
|
||||
]
|
||||
package_command.extend(["--preview-image", source_preview.as_posix()])
|
||||
run(package_command)
|
||||
return metadata if metadata.exists() else None
|
||||
|
||||
|
||||
def copy_quality_report(package_dir: Path, quality_report: Path) -> None:
|
||||
if quality_report.exists():
|
||||
quality_dest = package_dir / "synthetic_multiview_quality.json"
|
||||
quality_dest.write_text(quality_report.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
|
||||
|
||||
def enforce_anysplat_preview_quality(
|
||||
metadata_path: Path,
|
||||
*,
|
||||
min_psnr: float,
|
||||
max_mae: float,
|
||||
allow_low_quality: bool,
|
||||
) -> dict[str, Any]:
|
||||
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||
metrics = metadata.get("source_preview_metrics")
|
||||
if not isinstance(metrics, dict):
|
||||
raise SystemExit(f"AnySplat metadata missing source_preview_metrics: {metadata_path}")
|
||||
psnr = float(metrics.get("psnr", 0.0))
|
||||
mae = float(metrics.get("mae", 1.0))
|
||||
passed = psnr >= min_psnr and mae <= max_mae
|
||||
gate = {
|
||||
"min_source_preview_psnr": min_psnr,
|
||||
"max_source_preview_mae": max_mae,
|
||||
"passed": passed,
|
||||
}
|
||||
metadata["source_preview_quality_gate"] = gate
|
||||
metadata_path.write_text(json.dumps(metadata, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if not passed and not allow_low_quality:
|
||||
raise SystemExit(
|
||||
"AnySplat source-view quality gate failed: "
|
||||
f"psnr={psnr:.3f} < {min_psnr:.3f} or mae={mae:.4f} > {max_mae:.4f}; "
|
||||
"rerun with --allow-low-quality to keep the package"
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
def attach_anysplat_metadata_to_package(package_dir: Path, metadata_path: Path) -> Path:
|
||||
artifact_dir = package_dir / "artifacts"
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = artifact_dir / "anysplat_metadata.json"
|
||||
dest.write_text(metadata_path.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
|
||||
import_path = package_dir / "motrix_import.json"
|
||||
if import_path.is_file():
|
||||
import_manifest = json.loads(import_path.read_text(encoding="utf-8"))
|
||||
import_manifest["reconstruction_metadata"] = dest.relative_to(package_dir).as_posix()
|
||||
metadata = json.loads(dest.read_text(encoding="utf-8"))
|
||||
metrics = metadata.get("source_preview_metrics")
|
||||
gate = metadata.get("source_preview_quality_gate")
|
||||
if isinstance(metrics, dict):
|
||||
import_manifest.setdefault("verification", {})["source_preview_metrics"] = metrics
|
||||
if isinstance(gate, dict):
|
||||
import_manifest.setdefault("verification", {})["source_preview_quality_gate"] = gate
|
||||
import_path.write_text(json.dumps(import_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
return dest
|
||||
|
||||
|
||||
def verify_motrix_package(package_dir: Path) -> dict[str, Any]:
|
||||
scene_xml = package_dir / "mjcf" / "scene.xml"
|
||||
scene_ply = package_dir / "3dgs" / "scene.ply"
|
||||
config_json = package_dir / "configs" / "go2_real2sim.json"
|
||||
manifest_json = package_dir / "package_manifest.json"
|
||||
cameras_json = package_dir / "artifacts" / "cameras_json.json"
|
||||
for path in (scene_xml, scene_ply, config_json, manifest_json, cameras_json):
|
||||
if not path.is_file() or path.stat().st_size <= 0:
|
||||
raise SystemExit(f"package verification failed; missing or empty file: {path}")
|
||||
|
||||
ET.parse(scene_xml)
|
||||
config = json.loads(config_json.read_text(encoding="utf-8"))
|
||||
manifest = json.loads(manifest_json.read_text(encoding="utf-8"))
|
||||
cameras = json.loads(cameras_json.read_text(encoding="utf-8"))
|
||||
load_anysplat_source_camera(cameras_json)
|
||||
|
||||
scene_gaussians = config.get("scene_gaussians")
|
||||
if not isinstance(scene_gaussians, dict) or "scene" not in scene_gaussians:
|
||||
raise SystemExit("package verification failed; config missing scene_gaussians.scene")
|
||||
gaussian_path = resolve_package_config_path(config_json, scene_gaussians["scene"])
|
||||
if not gaussian_path.is_file():
|
||||
raise SystemExit(f"package verification failed; scene Gaussian PLY does not exist: {gaussian_path}")
|
||||
|
||||
manifest_scene = manifest.get("scene", {})
|
||||
manifest_ply = package_dir / str(manifest_scene.get("gaussian_ply", ""))
|
||||
if not manifest_ply.is_file():
|
||||
raise SystemExit(f"package verification failed; manifest scene gaussian_ply does not exist: {manifest_ply}")
|
||||
|
||||
return {
|
||||
"root": package_dir.as_posix(),
|
||||
"scene_xml": scene_xml.as_posix(),
|
||||
"config_json": config_json.as_posix(),
|
||||
"scene_ply": scene_ply.as_posix(),
|
||||
"scene_ply_vertices": count_ply_vertices(scene_ply),
|
||||
"capture_kind": manifest.get("metadata", {}).get("capture_kind"),
|
||||
"scene_fusion": manifest.get("metadata", {}).get("builtin_scene_fusion"),
|
||||
"scene_view_count": manifest.get("metadata", {}).get("builtin_scene_view_count"),
|
||||
"camera_view_count": len(cameras.get("views", [])) if isinstance(cameras.get("views"), list) else 0,
|
||||
}
|
||||
|
||||
|
||||
def resolve_package_config_path(config_json: Path, value: Any) -> Path:
|
||||
path = Path(str(value))
|
||||
if path.is_absolute():
|
||||
return path
|
||||
return (config_json.parent / path).resolve()
|
||||
|
||||
|
||||
def count_ply_vertices(path: Path) -> int:
|
||||
with path.open("r", encoding="ascii", errors="ignore") as handle:
|
||||
for line in handle:
|
||||
if line.startswith("element vertex "):
|
||||
return int(line.split()[-1])
|
||||
if line.strip() == "end_header":
|
||||
break
|
||||
raise SystemExit(f"PLY vertex count not found: {path}")
|
||||
|
||||
|
||||
def write_projection_preview(ply: Path, output: Path) -> Path:
|
||||
points, colors = load_ply_points_for_preview(ply, max_points=90000)
|
||||
width, height = 900, 520
|
||||
canvas = Image.new("RGB", (width * 2 + 20, height), (245, 245, 245))
|
||||
draw_projection(canvas, points, colors, (0, 2), 0, width, height, "front projection: x/z")
|
||||
draw_projection(canvas, points, colors, (0, 1), width + 20, width, height, "top projection: x/y")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
canvas.save(output)
|
||||
return output
|
||||
|
||||
|
||||
def write_motrix_import_files(
|
||||
package_dir: Path,
|
||||
report: dict[str, Any],
|
||||
preview: Path,
|
||||
quality_report: Path | None = None,
|
||||
) -> dict[str, Path]:
|
||||
"""Write explicit MotrixSim import entrypoints into the generated package."""
|
||||
manifest_path = package_dir / "motrix_import.json"
|
||||
readme_path = package_dir / "MOTRIX_IMPORT.md"
|
||||
package_name = package_dir.name
|
||||
recommended_camera = "source_3dgs_preview"
|
||||
viewer_args = [
|
||||
"--package",
|
||||
f"/path/to/{package_name}",
|
||||
"--camera",
|
||||
"closeup",
|
||||
"--gs-width",
|
||||
"900",
|
||||
"--gs-height",
|
||||
"700",
|
||||
"--static-gs-preview",
|
||||
"--with-gs",
|
||||
"--show-source-gs",
|
||||
"--source-gs-only",
|
||||
]
|
||||
import_manifest = {
|
||||
"format": "gs-playground-real2sim-motrix-import-v1",
|
||||
"import_type": "mjcf_scene_with_3dgs",
|
||||
"scene_xml": "mjcf/scene.xml",
|
||||
"config": "configs/go2_real2sim.json",
|
||||
"scene_gaussians": {"scene": "3dgs/scene.ply"},
|
||||
"camera_artifacts": {"cameras_json": "artifacts/cameras_json.json"},
|
||||
"preview": preview.relative_to(package_dir).as_posix(),
|
||||
"recommended_camera": recommended_camera,
|
||||
"viewer_args": viewer_args,
|
||||
"wuying_archive_launch": (
|
||||
"scripts/real2sim_wuying_launch_package.sh "
|
||||
f"--archive /path/to/{package_name}.tar.gz "
|
||||
f"--dest /tmp/{package_name}_import --with-gs"
|
||||
),
|
||||
"verification": {
|
||||
"scene_ply_vertices": report.get("scene_ply_vertices"),
|
||||
"scene_fusion": report.get("scene_fusion"),
|
||||
"scene_view_count": report.get("scene_view_count"),
|
||||
"camera_view_count": report.get("camera_view_count"),
|
||||
"recommended_preview": recommended_camera,
|
||||
},
|
||||
}
|
||||
if quality_report is not None and quality_report.exists():
|
||||
quality_dest = package_dir / "synthetic_multiview_quality.json"
|
||||
quality_dest.write_text(quality_report.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
import_manifest["synthetic_multiview_quality"] = quality_dest.relative_to(package_dir).as_posix()
|
||||
manifest_path.write_text(json.dumps(import_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
readme_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"# MotrixSim Import",
|
||||
"",
|
||||
"This package is a scene-level Real2Sim reconstruction from one source image plus synthetic multiview completion.",
|
||||
"",
|
||||
"Required entrypoints:",
|
||||
"",
|
||||
"- MJCF scene: `mjcf/scene.xml`",
|
||||
"- Scene Gaussian asset: `3dgs/scene.ply`",
|
||||
"- Viewer config: `configs/go2_real2sim.json`",
|
||||
"- Source cameras: `artifacts/cameras_json.json`",
|
||||
"- Projection preview: `pointcloud_projection_preview.png`",
|
||||
"",
|
||||
"Wuying launch command:",
|
||||
"",
|
||||
"```bash",
|
||||
import_manifest["wuying_archive_launch"],
|
||||
"```",
|
||||
"",
|
||||
"Direct viewer arguments:",
|
||||
"",
|
||||
"```bash",
|
||||
" ".join(import_manifest["viewer_args"]),
|
||||
"```",
|
||||
"",
|
||||
"Verification:",
|
||||
"",
|
||||
f"- scene vertices: {report.get('scene_ply_vertices')}",
|
||||
f"- scene fusion: {report.get('scene_fusion')}",
|
||||
f"- scene views: {report.get('scene_view_count')}",
|
||||
f"- camera views: {report.get('camera_view_count')}",
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return {"manifest": manifest_path, "readme": readme_path}
|
||||
|
||||
|
||||
def load_ply_points_for_preview(path: Path, *, max_points: int) -> tuple[np.ndarray, np.ndarray]:
|
||||
ply_format = read_ply_format(path)
|
||||
if ply_format != "ascii":
|
||||
return load_binary_ply_points_for_preview(path, max_points=max_points)
|
||||
|
||||
with path.open("r", encoding="ascii", errors="ignore") as handle:
|
||||
vertex_count = 0
|
||||
for line in handle:
|
||||
if line.startswith("element vertex "):
|
||||
vertex_count = int(line.split()[-1])
|
||||
if line.strip() == "end_header":
|
||||
break
|
||||
stride = max(1, vertex_count // max_points)
|
||||
points = []
|
||||
colors = []
|
||||
for index, line in enumerate(handle):
|
||||
if index >= vertex_count:
|
||||
break
|
||||
if index % stride:
|
||||
continue
|
||||
values = line.split()
|
||||
if len(values) < 9:
|
||||
continue
|
||||
points.append(tuple(map(float, values[:3])))
|
||||
rgb = np.asarray(list(map(float, values[6:9])), dtype=np.float32)
|
||||
rgb = 1.0 / (1.0 + np.exp(-rgb))
|
||||
colors.append((rgb * 255.0).clip(0.0, 255.0).astype(np.uint8))
|
||||
if not points:
|
||||
raise SystemExit(f"no previewable points found in {path}")
|
||||
return np.asarray(points, dtype=np.float32), np.asarray(colors, dtype=np.uint8)
|
||||
|
||||
|
||||
def read_ply_format(path: Path) -> str:
|
||||
with path.open("rb") as handle:
|
||||
for raw in handle:
|
||||
line = raw.decode("ascii", errors="ignore").strip()
|
||||
if line.startswith("format "):
|
||||
parts = line.split()
|
||||
return parts[1] if len(parts) > 1 else "ascii"
|
||||
if line == "end_header":
|
||||
break
|
||||
return "ascii"
|
||||
|
||||
|
||||
def load_binary_ply_points_for_preview(path: Path, *, max_points: int) -> tuple[np.ndarray, np.ndarray]:
|
||||
try:
|
||||
from plyfile import PlyData
|
||||
except Exception as exc:
|
||||
return load_binary_ply_points_without_plyfile(path, max_points=max_points, import_error=exc)
|
||||
|
||||
ply = PlyData.read(path)
|
||||
vertex = ply["vertex"].data
|
||||
names = set(vertex.dtype.names or ())
|
||||
required = {"x", "y", "z"}
|
||||
if not required.issubset(names):
|
||||
raise SystemExit(f"binary PLY missing required xyz fields: {path}")
|
||||
|
||||
vertex_count = len(vertex)
|
||||
stride = max(1, vertex_count // max_points)
|
||||
sampled = vertex[::stride]
|
||||
points = np.column_stack([sampled["x"], sampled["y"], sampled["z"]]).astype(np.float32, copy=False)
|
||||
if {"red", "green", "blue"}.issubset(names):
|
||||
colors = np.column_stack([sampled["red"], sampled["green"], sampled["blue"]]).astype(np.uint8, copy=False)
|
||||
elif {"f_dc_0", "f_dc_1", "f_dc_2"}.issubset(names):
|
||||
rgb = np.column_stack([sampled["f_dc_0"], sampled["f_dc_1"], sampled["f_dc_2"]]).astype(np.float32, copy=False)
|
||||
colors = (1.0 / (1.0 + np.exp(-rgb)) * 255.0).clip(0.0, 255.0).astype(np.uint8)
|
||||
else:
|
||||
colors = np.full((len(points), 3), 220, dtype=np.uint8)
|
||||
if len(points) == 0:
|
||||
raise SystemExit(f"no previewable points found in {path}")
|
||||
return points, colors
|
||||
|
||||
|
||||
def load_binary_ply_points_without_plyfile(
|
||||
path: Path,
|
||||
*,
|
||||
max_points: int,
|
||||
import_error: Exception | None = None,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
data = path.read_bytes()
|
||||
header_end = data.find(b"end_header\n")
|
||||
newline_len = 1
|
||||
if header_end < 0:
|
||||
header_end = data.find(b"end_header\r\n")
|
||||
newline_len = 2
|
||||
if header_end < 0:
|
||||
raise SystemExit(f"binary PLY missing end_header: {path}") from import_error
|
||||
|
||||
header_text = data[: header_end + len("end_header")].decode("ascii", errors="ignore")
|
||||
header_lines = [line.strip() for line in header_text.splitlines()]
|
||||
ply_format = "binary_little_endian"
|
||||
vertex_count = 0
|
||||
vertex_properties: list[tuple[str, str]] = []
|
||||
in_vertex = False
|
||||
for line in header_lines:
|
||||
if line.startswith("format "):
|
||||
parts = line.split()
|
||||
if len(parts) > 1:
|
||||
ply_format = parts[1]
|
||||
elif line.startswith("element "):
|
||||
parts = line.split()
|
||||
in_vertex = len(parts) >= 3 and parts[1] == "vertex"
|
||||
if in_vertex:
|
||||
vertex_count = int(parts[2])
|
||||
elif in_vertex and line.startswith("property "):
|
||||
parts = line.split()
|
||||
if len(parts) == 3:
|
||||
vertex_properties.append((parts[1], parts[2]))
|
||||
elif len(parts) > 3 and parts[1] == "list":
|
||||
raise SystemExit(f"binary PLY vertex list properties are unsupported for preview: {path}") from import_error
|
||||
|
||||
if vertex_count <= 0 or not vertex_properties:
|
||||
raise SystemExit(f"binary PLY has no previewable vertex table: {path}") from import_error
|
||||
endian = "<" if ply_format == "binary_little_endian" else ">"
|
||||
dtype = np.dtype([(name, numpy_dtype_for_ply_type(kind, endian)) for kind, name in vertex_properties])
|
||||
offset = header_end + len("end_header") + newline_len
|
||||
vertex = np.frombuffer(data, dtype=dtype, count=vertex_count, offset=offset)
|
||||
names = set(vertex.dtype.names or ())
|
||||
if not {"x", "y", "z"}.issubset(names):
|
||||
raise SystemExit(f"binary PLY missing required xyz fields: {path}") from import_error
|
||||
|
||||
stride = max(1, vertex_count // max_points)
|
||||
sampled = vertex[::stride]
|
||||
points = np.column_stack([sampled["x"], sampled["y"], sampled["z"]]).astype(np.float32, copy=False)
|
||||
if {"red", "green", "blue"}.issubset(names):
|
||||
colors = np.column_stack([sampled["red"], sampled["green"], sampled["blue"]]).astype(np.uint8, copy=False)
|
||||
elif {"f_dc_0", "f_dc_1", "f_dc_2"}.issubset(names):
|
||||
rgb = np.column_stack([sampled["f_dc_0"], sampled["f_dc_1"], sampled["f_dc_2"]]).astype(np.float32, copy=False)
|
||||
colors = (1.0 / (1.0 + np.exp(-rgb)) * 255.0).clip(0.0, 255.0).astype(np.uint8)
|
||||
else:
|
||||
colors = np.full((len(points), 3), 220, dtype=np.uint8)
|
||||
return points, colors
|
||||
|
||||
|
||||
def numpy_dtype_for_ply_type(kind: str, endian: str) -> str:
|
||||
mapping = {
|
||||
"char": "i1",
|
||||
"int8": "i1",
|
||||
"uchar": "u1",
|
||||
"uint8": "u1",
|
||||
"short": "i2",
|
||||
"int16": "i2",
|
||||
"ushort": "u2",
|
||||
"uint16": "u2",
|
||||
"int": "i4",
|
||||
"int32": "i4",
|
||||
"uint": "u4",
|
||||
"uint32": "u4",
|
||||
"float": "f4",
|
||||
"float32": "f4",
|
||||
"double": "f8",
|
||||
"float64": "f8",
|
||||
}
|
||||
if kind not in mapping:
|
||||
raise SystemExit(f"unsupported binary PLY property type: {kind}")
|
||||
code = mapping[kind]
|
||||
return code if code.endswith("1") else endian + code
|
||||
|
||||
|
||||
def draw_projection(
|
||||
canvas: Image.Image,
|
||||
points: np.ndarray,
|
||||
colors: np.ndarray,
|
||||
axes: tuple[int, int],
|
||||
offset: int,
|
||||
width: int,
|
||||
height: int,
|
||||
title: str,
|
||||
) -> None:
|
||||
sub = Image.new("RGB", (width, height), (250, 250, 250))
|
||||
draw = ImageDraw.Draw(sub)
|
||||
xy = points[:, list(axes)]
|
||||
low = xy.min(axis=0)
|
||||
high = xy.max(axis=0)
|
||||
span = np.maximum(high - low, 1e-6)
|
||||
uv = (xy - low) / span
|
||||
px = (uv[:, 0] * (width - 60) + 30).astype(int)
|
||||
py = ((1.0 - uv[:, 1]) * (height - 60) + 30).astype(int)
|
||||
order = np.argsort(points[:, 1] if axes != (0, 1) else points[:, 2])
|
||||
for index in order:
|
||||
x = int(px[index])
|
||||
y = int(py[index])
|
||||
if 0 <= x < width and 0 <= y < height:
|
||||
sub.putpixel((x, y), tuple(int(v) for v in colors[index]))
|
||||
draw.rectangle((0, 0, width - 1, height - 1), outline=(190, 190, 190))
|
||||
draw.text((14, 12), title, fill=(30, 30, 30))
|
||||
canvas.paste(sub, (offset, 0))
|
||||
|
||||
|
||||
def create_archive(package_dir: Path, archive: Path) -> Path:
|
||||
archive.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(archive, "w:gz") as tar:
|
||||
tar.add(package_dir, arcname=package_dir.name)
|
||||
return archive
|
||||
|
||||
|
||||
def run(command: list[str], *, env: dict[str, str] | None = None) -> None:
|
||||
result = subprocess.run(command, cwd=REPO_ROOT, env=env, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(result.returncode)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
381
vendor/gs_playground/scripts/real2sim_ply_mesh_to_obj.py
vendored
Normal file
381
vendor/gs_playground/scripts/real2sim_ply_mesh_to_obj.py
vendored
Normal file
@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert a colored PLY mesh to a Motrix-friendly OBJ.
|
||||
|
||||
The converter keeps vertex colors for tools that understand them and also emits
|
||||
an OBJ material palette so viewers that ignore OBJ vertex colors still show the
|
||||
scene with explicit diffuse materials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import struct
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
|
||||
SCALAR_TYPES: dict[str, tuple[str, int]] = {
|
||||
"char": ("b", 1),
|
||||
"int8": ("b", 1),
|
||||
"uchar": ("B", 1),
|
||||
"uint8": ("B", 1),
|
||||
"short": ("h", 2),
|
||||
"int16": ("h", 2),
|
||||
"ushort": ("H", 2),
|
||||
"uint16": ("H", 2),
|
||||
"int": ("i", 4),
|
||||
"int32": ("i", 4),
|
||||
"uint": ("I", 4),
|
||||
"uint32": ("I", 4),
|
||||
"float": ("f", 4),
|
||||
"float32": ("f", 4),
|
||||
"double": ("d", 8),
|
||||
"float64": ("d", 8),
|
||||
}
|
||||
|
||||
COLOR_NAMES = (
|
||||
("red", "green", "blue"),
|
||||
("diffuse_red", "diffuse_green", "diffuse_blue"),
|
||||
("r", "g", "b"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlyHeader:
|
||||
ply_format: str
|
||||
vertex_count: int
|
||||
face_count: int
|
||||
vertex_properties: tuple[tuple[str, str], ...]
|
||||
face_count_type: str
|
||||
face_index_type: str
|
||||
data_offset: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjVertex:
|
||||
x: float
|
||||
y: float
|
||||
z: float
|
||||
r: float
|
||||
g: float
|
||||
b: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjTriangle:
|
||||
a: int
|
||||
b: int
|
||||
c: int
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Convert a colored PLY mesh to a Motrix-compatible OBJ")
|
||||
parser.add_argument("--input-ply", type=Path, required=True)
|
||||
parser.add_argument("--output-obj", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, default=None)
|
||||
parser.add_argument("--no-material-palette", action="store_true", help="Write vertex-color-only OBJ without a sidecar MTL")
|
||||
parser.add_argument("--material-palette-levels", type=int, default=18, help="RGB quantization levels for generated diffuse materials")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = convert_ply_mesh_to_obj(
|
||||
args.input_ply,
|
||||
args.output_obj,
|
||||
material_palette=not args.no_material_palette,
|
||||
material_palette_levels=max(int(args.material_palette_levels), 2),
|
||||
)
|
||||
if args.report is not None:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
else:
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def convert_ply_mesh_to_obj(
|
||||
input_ply: Path,
|
||||
output_obj: Path,
|
||||
*,
|
||||
material_palette: bool = True,
|
||||
material_palette_levels: int = 18,
|
||||
) -> dict[str, Any]:
|
||||
header = read_ply_header(input_ply)
|
||||
output_obj.parent.mkdir(parents=True, exist_ok=True)
|
||||
with input_ply.open("rb") as src:
|
||||
src.seek(header.data_offset)
|
||||
if header.ply_format == "ascii":
|
||||
vertices, triangles = read_ascii_mesh(src, header)
|
||||
elif header.ply_format == "binary_little_endian":
|
||||
vertices, triangles = read_binary_mesh(src, header, endian="<")
|
||||
elif header.ply_format == "binary_big_endian":
|
||||
vertices, triangles = read_binary_mesh(src, header, endian=">")
|
||||
else:
|
||||
raise SystemExit(f"unsupported PLY format: {header.ply_format}")
|
||||
material_count = write_obj_mesh(
|
||||
input_ply=input_ply,
|
||||
output_obj=output_obj,
|
||||
vertices=vertices,
|
||||
triangles=triangles,
|
||||
material_palette=material_palette,
|
||||
material_palette_levels=max(int(material_palette_levels), 2),
|
||||
)
|
||||
return {
|
||||
"format": "gs-playground-real2sim-ply-mesh-to-obj-report-v1",
|
||||
"input_ply": input_ply.resolve().as_posix(),
|
||||
"output_obj": output_obj.resolve().as_posix(),
|
||||
"vertex_count": int(header.vertex_count),
|
||||
"face_count": int(len(triangles)),
|
||||
"texcoord_count": int(len(vertices)),
|
||||
"material_palette": bool(material_palette),
|
||||
"material_palette_levels": int(max(int(material_palette_levels), 2)),
|
||||
"material_count": int(material_count),
|
||||
"material_library": output_obj.with_suffix(".mtl").resolve().as_posix() if material_palette else None,
|
||||
}
|
||||
|
||||
|
||||
def read_ply_header(path: Path) -> PlyHeader:
|
||||
ply_format = "ascii"
|
||||
vertex_count = 0
|
||||
face_count = 0
|
||||
vertex_properties: list[tuple[str, str]] = []
|
||||
face_count_type = "uchar"
|
||||
face_index_type = "int"
|
||||
active_element: str | None = None
|
||||
with path.open("rb") as handle:
|
||||
first = handle.readline().decode("ascii", errors="ignore").strip()
|
||||
if first != "ply":
|
||||
raise SystemExit(f"not a PLY file: {path}")
|
||||
while True:
|
||||
raw = handle.readline()
|
||||
if not raw:
|
||||
raise SystemExit(f"PLY header missing end_header: {path}")
|
||||
line = raw.decode("ascii", errors="ignore").strip()
|
||||
if line == "end_header":
|
||||
data_offset = handle.tell()
|
||||
break
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
if parts[0] == "format" and len(parts) >= 2:
|
||||
ply_format = parts[1]
|
||||
elif parts[0] == "element" and len(parts) >= 3:
|
||||
active_element = parts[1]
|
||||
if active_element == "vertex":
|
||||
vertex_count = int(parts[2])
|
||||
elif active_element == "face":
|
||||
face_count = int(parts[2])
|
||||
elif parts[0] == "property" and active_element == "vertex" and len(parts) == 3:
|
||||
vertex_properties.append((parts[2], parts[1]))
|
||||
elif parts[0] == "property" and active_element == "face" and len(parts) >= 5 and parts[1] == "list":
|
||||
face_count_type = parts[2]
|
||||
face_index_type = parts[3]
|
||||
if vertex_count <= 0:
|
||||
raise SystemExit(f"PLY has no vertices: {path}")
|
||||
return PlyHeader(
|
||||
ply_format=ply_format,
|
||||
vertex_count=vertex_count,
|
||||
face_count=face_count,
|
||||
vertex_properties=tuple(vertex_properties),
|
||||
face_count_type=face_count_type,
|
||||
face_index_type=face_index_type,
|
||||
data_offset=data_offset,
|
||||
)
|
||||
|
||||
|
||||
def read_ascii_mesh(src: BinaryIO, header: PlyHeader) -> tuple[list[ObjVertex], list[ObjTriangle]]:
|
||||
property_names = [name for name, _kind in header.vertex_properties]
|
||||
index = {name: idx for idx, name in enumerate(property_names)}
|
||||
color_indices = find_color_indices(index)
|
||||
vertices: list[ObjVertex] = []
|
||||
for _ in range(header.vertex_count):
|
||||
parts = src.readline().decode("utf-8", errors="ignore").split()
|
||||
if len(parts) < len(property_names):
|
||||
raise SystemExit("truncated ASCII PLY vertex table")
|
||||
vertices.append(make_obj_vertex(parts, index, color_indices))
|
||||
triangles: list[ObjTriangle] = []
|
||||
for _ in range(header.face_count):
|
||||
parts = src.readline().decode("utf-8", errors="ignore").split()
|
||||
if not parts:
|
||||
continue
|
||||
count = int(parts[0])
|
||||
indices = [int(value) + 1 for value in parts[1 : 1 + count]]
|
||||
triangles.extend(triangulate(indices))
|
||||
return vertices, triangles
|
||||
|
||||
|
||||
def read_binary_mesh(src: BinaryIO, header: PlyHeader, *, endian: str) -> tuple[list[ObjVertex], list[ObjTriangle]]:
|
||||
property_names = [name for name, _kind in header.vertex_properties]
|
||||
index = {name: idx for idx, name in enumerate(property_names)}
|
||||
color_indices = find_color_indices(index)
|
||||
vertex_struct = struct.Struct(endian + "".join(scalar_format(kind) for _name, kind in header.vertex_properties))
|
||||
vertices: list[ObjVertex] = []
|
||||
for _ in range(header.vertex_count):
|
||||
data = src.read(vertex_struct.size)
|
||||
if len(data) != vertex_struct.size:
|
||||
raise SystemExit("truncated binary PLY vertex table")
|
||||
values = vertex_struct.unpack(data)
|
||||
vertices.append(make_obj_vertex(values, index, color_indices))
|
||||
count_struct = struct.Struct(endian + scalar_format(header.face_count_type))
|
||||
index_struct = struct.Struct(endian + scalar_format(header.face_index_type))
|
||||
triangles: list[ObjTriangle] = []
|
||||
for _ in range(header.face_count):
|
||||
raw_count = src.read(count_struct.size)
|
||||
if len(raw_count) != count_struct.size:
|
||||
raise SystemExit("truncated binary PLY face table")
|
||||
count = int(count_struct.unpack(raw_count)[0])
|
||||
raw_indices = src.read(index_struct.size * count)
|
||||
if len(raw_indices) != index_struct.size * count:
|
||||
raise SystemExit("truncated binary PLY face indices")
|
||||
indices = [
|
||||
int(index_struct.unpack_from(raw_indices, offset)[0]) + 1
|
||||
for offset in range(0, len(raw_indices), index_struct.size)
|
||||
]
|
||||
triangles.extend(triangulate(indices))
|
||||
return vertices, triangles
|
||||
|
||||
|
||||
def scalar_format(kind: str) -> str:
|
||||
if kind not in SCALAR_TYPES:
|
||||
raise SystemExit(f"unsupported PLY scalar type: {kind}")
|
||||
return SCALAR_TYPES[kind][0]
|
||||
|
||||
|
||||
def find_color_indices(index: dict[str, int]) -> tuple[int, int, int] | None:
|
||||
for names in COLOR_NAMES:
|
||||
if all(name in index for name in names):
|
||||
return tuple(index[name] for name in names)
|
||||
return None
|
||||
|
||||
|
||||
def make_obj_vertex(values: Any, index: dict[str, int], color_indices: tuple[int, int, int] | None) -> ObjVertex:
|
||||
x = float(values[index["x"]])
|
||||
y = float(values[index["y"]])
|
||||
z = float(values[index["z"]])
|
||||
if color_indices is None:
|
||||
r, g, b = 0.72, 0.72, 0.72
|
||||
else:
|
||||
channels = [float(values[idx]) for idx in color_indices]
|
||||
if max(channels) > 1.0:
|
||||
channels = [value / 255.0 for value in channels]
|
||||
r, g, b = [min(max(value, 0.0), 1.0) for value in channels]
|
||||
return ObjVertex(x=x, y=y, z=z, r=r, g=g, b=b)
|
||||
|
||||
|
||||
def triangulate(indices: list[int]) -> list[ObjTriangle]:
|
||||
if len(indices) < 3:
|
||||
return []
|
||||
triangles: list[ObjTriangle] = []
|
||||
for idx in range(1, len(indices) - 1):
|
||||
triangles.append(ObjTriangle(indices[0], indices[idx], indices[idx + 1]))
|
||||
return triangles
|
||||
|
||||
|
||||
def write_obj_mesh(
|
||||
*,
|
||||
input_ply: Path,
|
||||
output_obj: Path,
|
||||
vertices: list[ObjVertex],
|
||||
triangles: list[ObjTriangle],
|
||||
material_palette: bool,
|
||||
material_palette_levels: int,
|
||||
) -> int:
|
||||
material_path = output_obj.with_suffix(".mtl")
|
||||
uv_axes = choose_uv_axes(vertices)
|
||||
uv_bounds = axis_bounds(vertices, uv_axes)
|
||||
materials: dict[str, tuple[float, float, float]] = {}
|
||||
with output_obj.open("w", encoding="utf-8") as dst:
|
||||
dst.write(f"# Converted from {input_ply.name} for Real2Sim Motrix scene mesh\n")
|
||||
if material_palette:
|
||||
dst.write(f"mtllib {material_path.name}\n")
|
||||
dst.write("o real2sim_colmap_dense_scene\n")
|
||||
for vertex in vertices:
|
||||
dst.write(
|
||||
f"v {vertex.x:.9g} {vertex.y:.9g} {vertex.z:.9g} "
|
||||
f"{vertex.r:.6f} {vertex.g:.6f} {vertex.b:.6f}\n"
|
||||
)
|
||||
for vertex in vertices:
|
||||
u, v = vertex_uv(vertex, uv_axes, uv_bounds)
|
||||
dst.write(f"vt {u:.7g} {v:.7g}\n")
|
||||
current_material = None
|
||||
for triangle in triangles:
|
||||
if material_palette:
|
||||
material_name, color = triangle_material(vertices, triangle, output_obj.stem, material_palette_levels)
|
||||
materials.setdefault(material_name, color)
|
||||
if material_name != current_material:
|
||||
dst.write(f"usemtl {material_name}\n")
|
||||
current_material = material_name
|
||||
dst.write(f"f {triangle.a}/{triangle.a} {triangle.b}/{triangle.b} {triangle.c}/{triangle.c}\n")
|
||||
if material_palette:
|
||||
write_material_library(material_path, materials)
|
||||
elif material_path.exists():
|
||||
material_path.unlink()
|
||||
return len(materials)
|
||||
|
||||
|
||||
def choose_uv_axes(vertices: list[ObjVertex]) -> tuple[str, str]:
|
||||
if not vertices:
|
||||
return ("x", "y")
|
||||
extents = {}
|
||||
for axis in ("x", "y", "z"):
|
||||
values = [getattr(vertex, axis) for vertex in vertices]
|
||||
extents[axis] = max(values) - min(values)
|
||||
ranked = sorted(extents, key=lambda axis: extents[axis], reverse=True)
|
||||
return ranked[0], ranked[1]
|
||||
|
||||
|
||||
def axis_bounds(vertices: list[ObjVertex], axes: tuple[str, str]) -> tuple[tuple[float, float], tuple[float, float]]:
|
||||
bounds: list[tuple[float, float]] = []
|
||||
for axis in axes:
|
||||
values = [getattr(vertex, axis) for vertex in vertices]
|
||||
low = min(values) if values else 0.0
|
||||
high = max(values) if values else 1.0
|
||||
if math.isclose(high, low):
|
||||
high = low + 1.0
|
||||
bounds.append((low, high))
|
||||
return bounds[0], bounds[1]
|
||||
|
||||
|
||||
def vertex_uv(vertex: ObjVertex, axes: tuple[str, str], bounds: tuple[tuple[float, float], tuple[float, float]]) -> tuple[float, float]:
|
||||
first, second = axes
|
||||
(u_low, u_high), (v_low, v_high) = bounds
|
||||
u = (getattr(vertex, first) - u_low) / max(u_high - u_low, 1e-9)
|
||||
v = (getattr(vertex, second) - v_low) / max(v_high - v_low, 1e-9)
|
||||
return min(max(u, 0.0), 1.0), min(max(v, 0.0), 1.0)
|
||||
|
||||
|
||||
def triangle_material(
|
||||
vertices: list[ObjVertex],
|
||||
triangle: ObjTriangle,
|
||||
stem: str,
|
||||
levels: int,
|
||||
) -> tuple[str, tuple[float, float, float]]:
|
||||
tri_vertices = [vertices[index - 1] for index in (triangle.a, triangle.b, triangle.c)]
|
||||
color = tuple(sum(getattr(vertex, channel) for vertex in tri_vertices) / 3.0 for channel in ("r", "g", "b"))
|
||||
quantized = tuple(round(channel * (levels - 1)) / (levels - 1) for channel in color)
|
||||
suffix = "_".join(f"{int(round(channel * 255)):02x}" for channel in quantized)
|
||||
return f"{safe_token(stem)}_mat_{suffix}", quantized
|
||||
|
||||
|
||||
def safe_token(value: str) -> str:
|
||||
token = re.sub(r"[^A-Za-z0-9_]+", "_", value.strip())
|
||||
return token.strip("_") or "scene"
|
||||
|
||||
|
||||
def write_material_library(path: Path, materials: dict[str, tuple[float, float, float]]) -> None:
|
||||
with path.open("w", encoding="utf-8") as dst:
|
||||
dst.write("# Generated Real2Sim dense scene material palette\n")
|
||||
for name, (r, g, b) in sorted(materials.items()):
|
||||
dst.write(f"newmtl {name}\n")
|
||||
dst.write(f"Ka {r:.6f} {g:.6f} {b:.6f}\n")
|
||||
dst.write(f"Kd {r:.6f} {g:.6f} {b:.6f}\n")
|
||||
dst.write("Ks 0.030000 0.030000 0.030000\n")
|
||||
dst.write("Ns 8.000000\n")
|
||||
dst.write("d 1.000000\n")
|
||||
dst.write("illum 2\n\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1052
vendor/gs_playground/scripts/real2sim_pose_init_gsplat_train.py
vendored
Executable file
1052
vendor/gs_playground/scripts/real2sim_pose_init_gsplat_train.py
vendored
Executable file
File diff suppressed because it is too large
Load Diff
218
vendor/gs_playground/scripts/real2sim_sam3d_movable_batch.py
vendored
Normal file
218
vendor/gs_playground/scripts/real2sim_sam3d_movable_batch.py
vendored
Normal file
@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run SAM-3D over movable extraction tasks and emit production object assets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from real2sim.contracts import write_json
|
||||
from real2sim.image_io import read_gray, read_rgb, write_gray, write_rgb
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Convert movable extraction tasks into production SAM-3D object assets")
|
||||
parser.add_argument("--package", type=Path, required=True, help="Real2Sim package root")
|
||||
parser.add_argument("--tasks", type=Path, default=None, help="movable_extraction_tasks.json; defaults under package")
|
||||
parser.add_argument("--output-objects-json", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--command", default=os.environ.get("REAL2SIM_SAM3D_COMMAND", ""))
|
||||
parser.add_argument("--max-objects", type=int, default=3)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Write planned SAM-3D jobs without executing them")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = run_sam3d_movable_batch(
|
||||
package_root=args.package,
|
||||
tasks_path=args.tasks or args.package / "artifacts" / "assets" / "movable_extraction_tasks.json",
|
||||
output_objects_json=args.output_objects_json,
|
||||
output_dir=args.output_dir,
|
||||
command_template=args.command,
|
||||
max_objects=max(1, int(args.max_objects)),
|
||||
dry_run=bool(args.dry_run),
|
||||
)
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def run_sam3d_movable_batch(
|
||||
*,
|
||||
package_root: Path,
|
||||
tasks_path: Path,
|
||||
output_objects_json: Path,
|
||||
output_dir: Path,
|
||||
command_template: str,
|
||||
max_objects: int,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
tasks_data = json.loads(tasks_path.read_text(encoding="utf-8"))
|
||||
tasks = tasks_data.get("tasks", []) if isinstance(tasks_data, dict) else []
|
||||
if not dry_run and not command_template.strip():
|
||||
raise SystemExit("SAM-3D command is required via --command or REAL2SIM_SAM3D_COMMAND")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
objects: list[dict[str, Any]] = []
|
||||
jobs: list[dict[str, Any]] = []
|
||||
for index, task in enumerate(tasks[:max_objects]):
|
||||
if not isinstance(task, dict):
|
||||
continue
|
||||
job = prepare_sam3d_task(package_root=package_root, output_dir=output_dir, task=task, index=index)
|
||||
jobs.append(job)
|
||||
if dry_run:
|
||||
continue
|
||||
command = render_command(command_template, job["context"])
|
||||
completed = subprocess.run(command, cwd=REPO_ROOT, check=False)
|
||||
if completed.returncode != 0:
|
||||
raise SystemExit(f"SAM-3D command failed for {job['id']} with exit code {completed.returncode}")
|
||||
objects.append(production_object_from_job(job, package_root=package_root))
|
||||
payload = {
|
||||
"format": "gs-playground-real2sim-production-objects-v1",
|
||||
"objects": objects,
|
||||
"quality": {
|
||||
"object_count": len(objects),
|
||||
"source_task_count": len(tasks),
|
||||
"planned_job_count": len(jobs),
|
||||
"production": bool(objects) and len(objects) == len(jobs),
|
||||
"dry_run": bool(dry_run),
|
||||
},
|
||||
"sam3d_jobs": [
|
||||
{
|
||||
"id": job["id"],
|
||||
"label": job["label"],
|
||||
"instance_json": job["instance_json"],
|
||||
"image": job["image"],
|
||||
"mask": job["mask"],
|
||||
"output_dir": job["output_dir"],
|
||||
}
|
||||
for job in jobs
|
||||
],
|
||||
}
|
||||
write_json(output_objects_json, payload)
|
||||
return {"objects_json": output_objects_json.as_posix(), **payload["quality"]}
|
||||
|
||||
|
||||
def prepare_sam3d_task(*, package_root: Path, output_dir: Path, task: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
task_id = str(task.get("id") or f"task_{index:03d}")
|
||||
label = best_prompt_label(task)
|
||||
name = f"{index:02d}_{safe_token(label)}"
|
||||
object_dir = output_dir / name
|
||||
object_dir.mkdir(parents=True, exist_ok=True)
|
||||
crop_path = package_root / str(task["crop"])
|
||||
mask_path = package_root / str(task["mask"])
|
||||
rgb = read_rgb(crop_path)
|
||||
mask = read_gray(mask_path)
|
||||
local_mask = local_crop_mask(mask, task.get("bbox_xyxy"), rgb.shape[:2])
|
||||
image_out = object_dir / "sam3d_image.ppm"
|
||||
mask_out = object_dir / "sam3d_mask.pgm"
|
||||
write_rgb(image_out, rgb)
|
||||
write_gray(mask_out, local_mask.astype(np.float32))
|
||||
instance_json = object_dir / "instance.json"
|
||||
write_json(
|
||||
instance_json,
|
||||
{
|
||||
"id": task_id,
|
||||
"label": label,
|
||||
"bbox_xyxy": task.get("bbox_xyxy"),
|
||||
"prompt_candidates": task.get("prompt_candidates"),
|
||||
"source_task": task,
|
||||
},
|
||||
)
|
||||
gaussian = object_dir / "gaussian.ply"
|
||||
mesh = object_dir / "mesh.obj"
|
||||
pose = object_dir / "pose.json"
|
||||
cameras = package_root / "artifacts" / "cameras_json.json"
|
||||
context = {
|
||||
"image": image_out,
|
||||
"mask": mask_out,
|
||||
"instance_json": instance_json,
|
||||
"object_dir": object_dir,
|
||||
"object_ply": gaussian,
|
||||
"object_mesh": mesh,
|
||||
"object_pose": pose,
|
||||
"cameras_json": cameras if cameras.is_file() else "",
|
||||
}
|
||||
return {
|
||||
"id": task_id,
|
||||
"name": name,
|
||||
"label": label,
|
||||
"image": image_out.as_posix(),
|
||||
"mask": mask_out.as_posix(),
|
||||
"instance_json": instance_json.as_posix(),
|
||||
"output_dir": object_dir.as_posix(),
|
||||
"gaussian": gaussian.as_posix(),
|
||||
"mesh": mesh.as_posix(),
|
||||
"pose": pose.as_posix(),
|
||||
"context": context,
|
||||
}
|
||||
|
||||
|
||||
def render_command(template: str, context: dict[str, Any]) -> list[str]:
|
||||
rendered = template.format_map({key: str(value) for key, value in context.items()})
|
||||
return shlex.split(rendered)
|
||||
|
||||
|
||||
def production_object_from_job(job: dict[str, Any], *, package_root: Path) -> dict[str, Any]:
|
||||
gaussian = require_file(Path(job["gaussian"]), "SAM-3D object gaussian")
|
||||
mesh = require_file(Path(job["mesh"]), "SAM-3D object mesh")
|
||||
pose_path = require_file(Path(job["pose"]), "SAM-3D object pose")
|
||||
pose = json.loads(pose_path.read_text(encoding="utf-8"))
|
||||
metadata = Path(job["output_dir"]) / "sam3d_metadata.json"
|
||||
return {
|
||||
"name": job["name"],
|
||||
"label": job["label"],
|
||||
"gaussian_ply": gaussian.as_posix(),
|
||||
"mesh_obj": mesh.as_posix(),
|
||||
"pose": pose,
|
||||
"dynamic": True,
|
||||
"source_instance_id": f"sam3d_{job['id']}",
|
||||
"provisional": False,
|
||||
"reconstruction_quality": {
|
||||
"reconstruction_source": "sam3d_objects",
|
||||
"commercial_asset_grade": True,
|
||||
"has_gaussian_asset": True,
|
||||
"has_mesh_asset": True,
|
||||
"has_pose": True,
|
||||
"sam3d_metadata": metadata.as_posix() if metadata.is_file() else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def require_file(path: Path, label: str) -> Path:
|
||||
if not path.is_file() or path.stat().st_size <= 0:
|
||||
raise SystemExit(f"missing {label}: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def local_crop_mask(global_mask: np.ndarray, bbox: Any, crop_shape: tuple[int, int]) -> np.ndarray:
|
||||
if not isinstance(bbox, list) or len(bbox) != 4:
|
||||
return np.ones(crop_shape, dtype=bool)
|
||||
x0, y0, x1, y1 = [int(value) for value in bbox]
|
||||
cropped = global_mask[max(0, y0) : max(0, y1), max(0, x0) : max(0, x1)] > 0.5
|
||||
if cropped.shape != crop_shape:
|
||||
return np.ones(crop_shape, dtype=bool)
|
||||
return cropped
|
||||
|
||||
|
||||
def best_prompt_label(task: dict[str, Any]) -> str:
|
||||
prompts = task.get("prompt_candidates")
|
||||
if isinstance(prompts, list) and prompts:
|
||||
return str(prompts[0])
|
||||
return "movable_object"
|
||||
|
||||
|
||||
def safe_token(value: str) -> str:
|
||||
return "".join(ch if ch.isalnum() else "_" for ch in value.lower()).strip("_") or "object"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
372
vendor/gs_playground/scripts/real2sim_scene_viewer.py
vendored
Normal file
372
vendor/gs_playground/scripts/real2sim_scene_viewer.py
vendored
Normal file
@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Desktop viewer for a generated Real2Sim MotrixSim scene package.
|
||||
|
||||
The viewer can inspect only the MJCF scene, or load a Real2Sim package config
|
||||
and render the corresponding 3D Gaussian assets without attaching a robot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("SDL_AUDIODRIVER", "dummy")
|
||||
os.environ.setdefault("ALSOFT_DRIVERS", "null")
|
||||
os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "8.9")
|
||||
if "XDG_RUNTIME_DIR" not in os.environ:
|
||||
runtime_dir = Path(f"/run/user/{os.getuid()}")
|
||||
if runtime_dir.is_dir():
|
||||
os.environ["XDG_RUNTIME_DIR"] = runtime_dir.as_posix()
|
||||
|
||||
sys.path[:] = [
|
||||
entry
|
||||
for entry in sys.path
|
||||
if not entry
|
||||
or (
|
||||
Path(entry).name != "speedy-splat"
|
||||
and "real2sim_paper_runtime/speedy-splat" not in Path(entry).as_posix()
|
||||
)
|
||||
]
|
||||
|
||||
import numpy as np
|
||||
from motrixsim import SceneData, msd, run
|
||||
from motrixsim.render import Layout, RenderApp, RenderSettings
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if REPO_ROOT.as_posix() not in sys.path:
|
||||
sys.path.insert(0, REPO_ROOT.as_posix())
|
||||
|
||||
from real2sim.camera import load_anysplat_source_camera
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Open a Real2Sim scene package without attaching a robot")
|
||||
parser.add_argument("--config", type=Path, default=None, help="Path to a generated configs/*_real2sim.json")
|
||||
parser.add_argument("--scene", type=Path, default=None, help="Path to the generated mjcf/scene.xml")
|
||||
parser.add_argument(
|
||||
"--camera",
|
||||
default="overview",
|
||||
help="Camera name from scene.xml to use as the main view; use 'system' for the interactive free camera",
|
||||
)
|
||||
parser.add_argument("--no-gs", action="store_true", help="Disable 3DGS preview widgets and load only the Motrix/MJCF scene")
|
||||
parser.add_argument("--gs-width", type=int, default=640, help="3DGS camera widget width")
|
||||
parser.add_argument("--gs-height", type=int, default=480, help="3DGS camera widget height")
|
||||
parser.add_argument("--gs-widget-left", type=int, default=10, help="Left offset for 3DGS preview widgets")
|
||||
parser.add_argument("--gs-widget-top", type=int, default=10, help="Top offset for 3DGS preview widgets")
|
||||
parser.add_argument("--gs-render-fps", type=float, default=12.0, help="Maximum 3DGS widget render refresh rate; use 0 for every frame")
|
||||
parser.add_argument("--static-gs-preview", action="store_true", help="Render 3DGS preview widgets once, then leave interaction to MotrixRender")
|
||||
parser.add_argument("--viewer-fps", type=float, default=60.0, help="MotrixRender window refresh rate")
|
||||
parser.add_argument("--frame-sleep-ms", type=float, default=0.0, help="Extra sleep after each render frame to keep desktop interaction responsive")
|
||||
parser.add_argument("--show-source-gs", action="store_true", help="Render the AnySplat source camera from camera_artifacts.cameras_json")
|
||||
parser.add_argument("--source-gs-only", action="store_true", help="Use the source-camera 3DGS render as the only 3DGS preview widget")
|
||||
parser.add_argument("--source-camera-json", type=Path, default=None, help="Override source-view cameras.json artifact")
|
||||
parser.add_argument("--save-source-gs", type=Path, default=None, help="Optional PNG path for the rendered source-camera 3DGS view")
|
||||
parser.add_argument("--save-main-gs", type=Path, default=None, help="Optional PNG path for the rendered main-camera 3DGS view")
|
||||
parser.add_argument("--show-system-gs", action="store_true", help="Also render a 3DGS view from the interactive camera")
|
||||
parser.add_argument("--exit-after-static-preview", action="store_true", help="Exit after static 3DGS preview images have been rendered and saved")
|
||||
parser.add_argument("--no-sync", action="store_true", help="Disable real-time sync")
|
||||
args = parser.parse_args()
|
||||
|
||||
config = load_config(args.config)
|
||||
scene_path = resolve_scene_path(args.scene, config, args.config)
|
||||
scene_gaussians = {} if args.no_gs else collect_scene_gaussians(config, args.config)
|
||||
camera_artifacts = collect_camera_artifacts(config, args.config)
|
||||
source_camera_json = args.source_camera_json or camera_artifacts.get("cameras_json")
|
||||
gaussian_links = {
|
||||
str(gs_name): str(link_name)
|
||||
for gs_name, link_name in config.get("gaussian_links", {}).items()
|
||||
}
|
||||
|
||||
world = msd.from_file(scene_path.as_posix())
|
||||
model = world.build()
|
||||
data = SceneData(model)
|
||||
|
||||
render_target_fps = max(int(round(float(args.viewer_fps))), 1)
|
||||
with RenderApp(fps=render_target_fps) as render:
|
||||
print(f"Launching Real2Sim scene viewer: {scene_path}", flush=True)
|
||||
render.launch(model, render_settings=RenderSettings.performance())
|
||||
for group_id in range(6):
|
||||
try:
|
||||
render.opt.set_group_vis(group_id, True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
use_system_camera = args.camera.lower() in {"system", "free", "interactive"}
|
||||
camera = None if use_system_camera else find_camera(model, args.camera)
|
||||
camera_id = None if use_system_camera else find_camera_id(model, args.camera)
|
||||
if use_system_camera:
|
||||
render.set_main_camera(None)
|
||||
seed_camera = find_camera(model, "overview") or (model.cameras[0] if len(model.cameras) else None)
|
||||
if seed_camera is not None:
|
||||
try:
|
||||
render.system_camera.pose = seed_camera.get_pose(data).astype(np.float32)
|
||||
except Exception:
|
||||
pass
|
||||
print("Using system camera", flush=True)
|
||||
elif camera is not None:
|
||||
render.set_main_camera(camera)
|
||||
print(f"Using camera: {args.camera}", flush=True)
|
||||
|
||||
gs_renderer = None
|
||||
gs_image = None
|
||||
gs_source_image = None
|
||||
gs_system_image = None
|
||||
gs_debug_printed = False
|
||||
source_rendered = False
|
||||
source_render_failed = False
|
||||
source_camera = None
|
||||
if scene_gaussians:
|
||||
print(f"Loading 3DGS assets: {', '.join(sorted(scene_gaussians))}", flush=True)
|
||||
from gaussian_renderer import GSRendererMotrixSim
|
||||
|
||||
gs_renderer = GSRendererMotrixSim(scene_gaussians, model)
|
||||
apply_gaussian_links(gs_renderer, model, gaussian_links)
|
||||
widget_top = int(args.gs_widget_top)
|
||||
widget_left = int(args.gs_widget_left)
|
||||
if args.source_gs_only:
|
||||
args.show_source_gs = True
|
||||
if args.show_source_gs or args.save_source_gs is not None:
|
||||
if source_camera_json is None:
|
||||
print("Source 3DGS view requested but no camera_artifacts.cameras_json was found.", flush=True)
|
||||
elif source_camera_json.is_file():
|
||||
source_camera = load_anysplat_source_camera(source_camera_json)
|
||||
gs_source_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8))
|
||||
if args.show_source_gs:
|
||||
render.widgets.create_image_widget(
|
||||
gs_source_image,
|
||||
layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height),
|
||||
)
|
||||
widget_top += args.gs_height + 10
|
||||
print(f"Source 3DGS camera ready: {source_camera_json}", flush=True)
|
||||
else:
|
||||
print(f"Source camera JSON is missing: {source_camera_json}", flush=True)
|
||||
if not args.source_gs_only:
|
||||
gs_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8))
|
||||
render.widgets.create_image_widget(
|
||||
gs_image,
|
||||
layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height),
|
||||
)
|
||||
widget_top += args.gs_height + 10
|
||||
if args.show_system_gs:
|
||||
gs_system_image = render.create_image(np.zeros((args.gs_height, args.gs_width, 3), dtype=np.uint8))
|
||||
render.widgets.create_image_widget(
|
||||
gs_system_image,
|
||||
layout=Layout(left=widget_left, top=widget_top, width=args.gs_width, height=args.gs_height),
|
||||
)
|
||||
print("3DGS renderer ready.", flush=True)
|
||||
|
||||
print(f"Scene viewer ready. Bodies: {len(model.body_names)} Links: {len(model.link_names)}", flush=True)
|
||||
|
||||
def phys_step() -> None:
|
||||
return None
|
||||
|
||||
last_gs_render_at = 0.0
|
||||
gs_preview_rendered = False
|
||||
gs_render_interval = 0.0 if args.gs_render_fps <= 0 else 1.0 / max(float(args.gs_render_fps), 1e-6)
|
||||
|
||||
def render_step() -> bool:
|
||||
nonlocal gs_debug_printed, source_rendered, source_render_failed, last_gs_render_at, gs_preview_rendered
|
||||
if render.input.is_key_just_pressed("esc") or render.input.is_key_just_pressed("escape"):
|
||||
return False
|
||||
if gs_renderer is not None:
|
||||
now = time.monotonic()
|
||||
should_render_gs = (
|
||||
not (args.static_gs_preview and gs_preview_rendered)
|
||||
and (gs_render_interval <= 0.0 or (now - last_gs_render_at) >= gs_render_interval)
|
||||
)
|
||||
if source_camera is not None and gs_source_image is not None and not source_rendered and not source_render_failed:
|
||||
try:
|
||||
gs_renderer.update_gaussians(data)
|
||||
source_rgb, _source_depth = gs_renderer.render_batch(
|
||||
source_camera.position.reshape(1, 3),
|
||||
source_camera.xmat.reshape(1, 9),
|
||||
args.gs_height,
|
||||
args.gs_width,
|
||||
np.array([source_camera.fovy_degrees], dtype=np.float32),
|
||||
y_up=source_camera.y_up,
|
||||
)
|
||||
gs_source_image.pixels = rgb_to_uint8(source_rgb[0])
|
||||
if args.save_source_gs is not None:
|
||||
args.save_source_gs.parent.mkdir(parents=True, exist_ok=True)
|
||||
gs_source_image.save_to_disk(args.save_source_gs.as_posix())
|
||||
print(f"Saved source 3DGS render: {args.save_source_gs}", flush=True)
|
||||
source_rendered = True
|
||||
print("3DGS rendered source camera.", flush=True)
|
||||
if args.source_gs_only and args.static_gs_preview and not gs_preview_rendered:
|
||||
gs_preview_rendered = True
|
||||
print("3DGS static preview rendered.", flush=True)
|
||||
except Exception as exc:
|
||||
source_render_failed = True
|
||||
print(f"Source 3DGS render failed: {exc}", flush=True)
|
||||
if should_render_gs and not args.source_gs_only:
|
||||
last_gs_render_at = now
|
||||
gs_renderer.update_gaussians(data)
|
||||
render_ids = []
|
||||
if camera_id is not None:
|
||||
render_ids.append(camera_id)
|
||||
if args.show_system_gs:
|
||||
render_ids.append(-1)
|
||||
if render_ids:
|
||||
results = gs_renderer.render(
|
||||
model,
|
||||
data,
|
||||
render_ids,
|
||||
args.gs_width,
|
||||
args.gs_height,
|
||||
system_camera=render.system_camera,
|
||||
)
|
||||
if not gs_debug_printed:
|
||||
print(f"3DGS rendered camera ids: {sorted(results.keys())}", flush=True)
|
||||
gs_debug_printed = True
|
||||
if camera_id in results and gs_image is not None:
|
||||
gs_image.pixels = rgb_to_uint8(results[camera_id][0])
|
||||
if args.save_main_gs is not None:
|
||||
args.save_main_gs.parent.mkdir(parents=True, exist_ok=True)
|
||||
gs_image.save_to_disk(args.save_main_gs.as_posix())
|
||||
print(f"Saved main 3DGS render: {args.save_main_gs}", flush=True)
|
||||
if -1 in results and gs_system_image is not None:
|
||||
gs_system_image.pixels = rgb_to_uint8(results[-1][0])
|
||||
if args.static_gs_preview:
|
||||
gs_preview_rendered = True
|
||||
print("3DGS static preview rendered.", flush=True)
|
||||
should_exit_after_preview = args.exit_after_static_preview and args.static_gs_preview and gs_preview_rendered
|
||||
render.sync(data)
|
||||
if args.frame_sleep_ms > 0:
|
||||
time.sleep(float(args.frame_sleep_ms) / 1000.0)
|
||||
if should_exit_after_preview:
|
||||
# Motrix's render loop can ignore a false-y callback return on
|
||||
# the Wuying headless display. This flag is only for automated
|
||||
# static screenshot capture, so exit explicitly after flushing.
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
os._exit(0)
|
||||
return True
|
||||
|
||||
render_fps = 1000000.0 if args.no_sync else max(float(args.viewer_fps), 1.0)
|
||||
run.render_loop(model.options.timestep, render_fps, phys_step, render_step)
|
||||
return 0
|
||||
|
||||
|
||||
def find_camera(model, name: str):
|
||||
try:
|
||||
return model.cameras[name]
|
||||
except Exception:
|
||||
pass
|
||||
for camera in model.cameras:
|
||||
if getattr(camera, "name", None) == name:
|
||||
return camera
|
||||
return model.cameras[0] if len(model.cameras) else None
|
||||
|
||||
|
||||
def find_camera_id(model, name: str) -> int | None:
|
||||
for idx, camera in enumerate(model.cameras):
|
||||
if getattr(camera, "name", None) == name:
|
||||
return idx
|
||||
return 0 if len(model.cameras) else None
|
||||
|
||||
|
||||
def load_config(config_path: Path | None) -> dict:
|
||||
if config_path is None:
|
||||
return {}
|
||||
with config_path.open("r", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError(f"Real2Sim viewer config must be a JSON object: {config_path}")
|
||||
return config
|
||||
|
||||
|
||||
def resolve_config_path(path: str | Path, config_path: Path | None) -> Path:
|
||||
resolved = Path(path)
|
||||
if resolved.is_absolute():
|
||||
return resolved
|
||||
if config_path is not None:
|
||||
return (config_path.resolve().parent / resolved).resolve()
|
||||
return resolved.resolve()
|
||||
|
||||
|
||||
def resolve_scene_path(scene_arg: Path | None, config: dict, config_path: Path | None) -> Path:
|
||||
scene = scene_arg if scene_arg is not None else config.get("scene")
|
||||
if not scene:
|
||||
raise ValueError("Provide --scene or a --config containing a scene path")
|
||||
return resolve_config_path(scene, config_path)
|
||||
|
||||
|
||||
def collect_scene_gaussians(config: dict, config_path: Path | None) -> dict[str, str]:
|
||||
scene_gaussians = config.get("scene_gaussians", {})
|
||||
if not isinstance(scene_gaussians, dict):
|
||||
raise ValueError("scene_gaussians must be an object mapping Gaussian names to PLY paths")
|
||||
|
||||
gaussians: dict[str, str] = {}
|
||||
missing: list[Path] = []
|
||||
for name, path in scene_gaussians.items():
|
||||
ply = resolve_config_path(path, config_path)
|
||||
if ply.exists():
|
||||
gaussians[str(name)] = ply.as_posix()
|
||||
else:
|
||||
missing.append(ply)
|
||||
if missing:
|
||||
print("Skipping missing 3DGS assets: " + ", ".join(path.as_posix() for path in missing), flush=True)
|
||||
return gaussians
|
||||
|
||||
|
||||
def collect_camera_artifacts(config: dict, config_path: Path | None) -> dict[str, Path]:
|
||||
raw_artifacts = config.get("camera_artifacts", {})
|
||||
if not isinstance(raw_artifacts, dict):
|
||||
raise ValueError("camera_artifacts must be an object mapping artifact names to paths")
|
||||
|
||||
artifacts: dict[str, Path] = {}
|
||||
missing: list[Path] = []
|
||||
for name, path in raw_artifacts.items():
|
||||
artifact = resolve_config_path(path, config_path)
|
||||
if artifact.exists():
|
||||
artifacts[str(name)] = artifact
|
||||
else:
|
||||
missing.append(artifact)
|
||||
if missing:
|
||||
print("Skipping missing camera artifacts: " + ", ".join(path.as_posix() for path in missing), flush=True)
|
||||
return artifacts
|
||||
|
||||
|
||||
def apply_gaussian_links(gs_renderer, model, gaussian_links: dict[str, str]) -> None:
|
||||
if not gaussian_links:
|
||||
return
|
||||
|
||||
link_ids = {name: idx for idx, name in enumerate(model.link_names)}
|
||||
objects_info = []
|
||||
body_ids = []
|
||||
skipped = []
|
||||
for gs_name, link_name in gaussian_links.items():
|
||||
if gs_name not in gs_renderer.gaussian_start_indices:
|
||||
skipped.append(f"{gs_name}:missing-gaussian")
|
||||
continue
|
||||
if link_name not in link_ids:
|
||||
skipped.append(f"{gs_name}:{link_name}:missing-link")
|
||||
continue
|
||||
objects_info.append((gs_name, gs_renderer.gaussian_start_indices[gs_name], gs_renderer.gaussian_end_indices[gs_name]))
|
||||
body_ids.append(link_ids[link_name])
|
||||
|
||||
if skipped:
|
||||
print("Skipping Gaussian links: " + ", ".join(skipped), flush=True)
|
||||
if not objects_info:
|
||||
return
|
||||
|
||||
gs_renderer.gs_idx_start = np.array([start for _, start, _ in objects_info])
|
||||
gs_renderer.gs_idx_end = np.array([end for _, _, end in objects_info])
|
||||
gs_renderer.gs_body_ids = np.array(body_ids)
|
||||
gs_renderer.set_objects_mapping(objects_info)
|
||||
print(f"Bound 3DGS assets to MJCF links: {', '.join(name for name, _, _ in objects_info)}", flush=True)
|
||||
|
||||
|
||||
def rgb_to_uint8(rgb_tensor) -> np.ndarray:
|
||||
rgb = rgb_tensor.detach().cpu().numpy()
|
||||
if rgb.dtype == np.uint8:
|
||||
return rgb
|
||||
return np.clip(rgb * 255.0, 0, 255).astype(np.uint8)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
253
vendor/gs_playground/scripts/real2sim_sweep_motrix_scene_completion.py
vendored
Normal file
253
vendor/gs_playground/scripts/real2sim_sweep_motrix_scene_completion.py
vendored
Normal file
@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sweep MotrixSim scene-completion parameters for a Real2Sim package.
|
||||
|
||||
The sweep keeps the reconstructed 3DGS and textured scene mesh fixed, varies
|
||||
the photo-completion backdrop margin, packages each candidate, runs the MuJoCo
|
||||
offscreen verifier, and ranks candidates by the verifier's render-quality
|
||||
score. It is intended for same-capture candidate comparison, not as an absolute
|
||||
commercial-quality metric.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Sweep Real2Sim Motrix scene-completion backdrop margins")
|
||||
parser.add_argument("--source", type=Path, required=True, help="Original capture directory or source path")
|
||||
parser.add_argument("--scene-ply", type=Path, required=True)
|
||||
parser.add_argument("--scene-mesh", type=Path, required=True)
|
||||
parser.add_argument("--cameras-json", type=Path, required=True)
|
||||
parser.add_argument("--preview-image", type=Path, required=True)
|
||||
parser.add_argument("--mesh-cleanup-report", type=Path, required=True)
|
||||
parser.add_argument("--texture-bake-report", type=Path, required=True)
|
||||
parser.add_argument("--completion-report", type=Path, required=True)
|
||||
parser.add_argument("--completion-texture", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
parser.add_argument("--name-prefix", default="real2sim_scene_completion_sweep")
|
||||
parser.add_argument("--margins", default="0,0.15,0.25,0.35,0.55", help="Comma-separated backdrop margins to test")
|
||||
parser.add_argument("--scene-faces", type=int, default=30_000, help="Default scene mesh LOD face budget")
|
||||
parser.add_argument("--scene-face-options", default="", help="Comma-separated scene mesh LOD face budgets to sweep")
|
||||
parser.add_argument("--min-scene-vertices", type=int, default=100_000)
|
||||
parser.add_argument("--robot", default="go2", choices=["go1", "go2", "g1"])
|
||||
parser.add_argument("--prompt", default="photogrammetry textured scene with swept photo completion backdrop")
|
||||
parser.add_argument("--python", default=sys.executable)
|
||||
parser.add_argument("--archive", action="store_true", help="Write a .tar.gz for each candidate package")
|
||||
parser.add_argument("--skip-existing", action="store_true", help="Reuse package and verifier output directories when present")
|
||||
args = parser.parse_args()
|
||||
|
||||
margins = parse_margins(args.margins)
|
||||
face_options = parse_int_options(args.scene_face_options) if args.scene_face_options else [int(args.scene_faces)]
|
||||
args.output_root.mkdir(parents=True, exist_ok=True)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
for margin in margins:
|
||||
for scene_faces in face_options:
|
||||
candidates.append(run_candidate(args, margin, scene_faces))
|
||||
|
||||
ranked = rank_candidates(candidates)
|
||||
report = {
|
||||
"format": "gs-playground-real2sim-scene-completion-sweep-v1",
|
||||
"target_quality": "polycam_realitycapture_like_space",
|
||||
"margins": margins,
|
||||
"scene_face_options": face_options,
|
||||
"best": ranked[0] if ranked else None,
|
||||
"candidates": ranked,
|
||||
"notes": [
|
||||
"Scores are comparable only within the same capture, cameras, render size, and verifier version.",
|
||||
"A high score does not prove Polycam/RealityCapture-level fidelity; inspect contact sheets before accepting.",
|
||||
],
|
||||
}
|
||||
report_path = args.output_root / "sweep_report.json"
|
||||
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def parse_margins(raw: str) -> list[float]:
|
||||
values: list[float] = []
|
||||
for item in raw.split(","):
|
||||
item = item.strip()
|
||||
if not item:
|
||||
continue
|
||||
value = float(item)
|
||||
if value < 0:
|
||||
raise SystemExit(f"margin must be non-negative: {value}")
|
||||
values.append(value)
|
||||
if not values:
|
||||
raise SystemExit("at least one margin is required")
|
||||
return values
|
||||
|
||||
|
||||
def parse_int_options(raw: str) -> list[int]:
|
||||
values: list[int] = []
|
||||
for item in raw.split(","):
|
||||
item = item.strip()
|
||||
if not item:
|
||||
continue
|
||||
value = int(item)
|
||||
if value <= 0:
|
||||
raise SystemExit(f"scene face option must be positive: {value}")
|
||||
values.append(value)
|
||||
if not values:
|
||||
raise SystemExit("at least one scene face option is required")
|
||||
return values
|
||||
|
||||
|
||||
def run_candidate(args: argparse.Namespace, margin: float, scene_faces: int) -> dict[str, Any]:
|
||||
token = candidate_token(margin, scene_faces)
|
||||
package_dir = args.output_root / f"{args.name_prefix}_{token}_package"
|
||||
verify_dir = args.output_root / f"{args.name_prefix}_{token}_verify"
|
||||
report_path = verify_dir / "report.json"
|
||||
|
||||
if not (args.skip_existing and report_path.is_file()):
|
||||
if package_dir.exists():
|
||||
shutil.rmtree(package_dir)
|
||||
if verify_dir.exists():
|
||||
shutil.rmtree(verify_dir)
|
||||
verify_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"REAL2SIM_SCENE_COMPLETION_REPORT": args.completion_report.as_posix(),
|
||||
"REAL2SIM_SCENE_COMPLETION_TEXTURE": args.completion_texture.as_posix(),
|
||||
"REAL2SIM_SCENE_COMPLETION_BACKDROP_MARGIN": str(margin),
|
||||
"REAL2SIM_MOTRIX_SCENE_VISUAL": "photogrammetry",
|
||||
"REAL2SIM_MOTRIX_PREVIEW_MESH_FACES": str(max(int(scene_faces), 1)),
|
||||
}
|
||||
)
|
||||
package_cmd = [
|
||||
args.python,
|
||||
"-m",
|
||||
"real2sim",
|
||||
"package-existing",
|
||||
"--source",
|
||||
args.source.as_posix(),
|
||||
"--scene-ply",
|
||||
args.scene_ply.as_posix(),
|
||||
"--scene-mesh",
|
||||
args.scene_mesh.as_posix(),
|
||||
"--cameras-json",
|
||||
args.cameras_json.as_posix(),
|
||||
"--preview-image",
|
||||
args.preview_image.as_posix(),
|
||||
"--mesh-cleanup-report",
|
||||
args.mesh_cleanup_report.as_posix(),
|
||||
"--texture-bake-report",
|
||||
args.texture_bake_report.as_posix(),
|
||||
"--output",
|
||||
package_dir.as_posix(),
|
||||
"--prompt",
|
||||
args.prompt,
|
||||
"--robot",
|
||||
args.robot,
|
||||
]
|
||||
run(package_cmd, env=env, cwd=REPO_ROOT)
|
||||
|
||||
verify_cmd = [
|
||||
args.python,
|
||||
(REPO_ROOT / "scripts" / "real2sim_verify_motrix_package.py").as_posix(),
|
||||
"--package",
|
||||
package_dir.as_posix(),
|
||||
"--min-scene-vertices",
|
||||
str(int(args.min_scene_vertices)),
|
||||
"--mujoco-offscreen-render-dir",
|
||||
verify_dir.as_posix(),
|
||||
"--visual-report-dir",
|
||||
verify_dir.as_posix(),
|
||||
]
|
||||
with report_path.open("w", encoding="utf-8") as handle:
|
||||
run(verify_cmd, env={"MUJOCO_GL": "egl", **env}, cwd=REPO_ROOT, stdout=handle)
|
||||
|
||||
if args.archive:
|
||||
archive_path = package_dir.with_suffix(".tar.gz")
|
||||
run(["tar", "-C", package_dir.parent.as_posix(), "-czf", archive_path.as_posix(), package_dir.name], cwd=REPO_ROOT)
|
||||
|
||||
verification = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
return candidate_summary(margin, scene_faces, package_dir, verify_dir, verification)
|
||||
|
||||
|
||||
def run(
|
||||
cmd: list[str],
|
||||
*,
|
||||
cwd: Path,
|
||||
env: dict[str, str] | None = None,
|
||||
stdout: Any | None = None,
|
||||
) -> None:
|
||||
process_env = os.environ.copy()
|
||||
if env:
|
||||
process_env.update(env)
|
||||
subprocess.run(cmd, cwd=cwd, env=process_env, stdout=stdout, check=True)
|
||||
|
||||
|
||||
def candidate_summary(
|
||||
margin: float,
|
||||
scene_faces: int,
|
||||
package_dir: Path,
|
||||
verify_dir: Path,
|
||||
verification: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
renders = verification.get("mujoco_offscreen_render", {}).get("renders", [])
|
||||
scores = [
|
||||
float(render.get("quality", {}).get("score", 0.0))
|
||||
for render in renders
|
||||
if isinstance(render, dict)
|
||||
]
|
||||
overview = next((render for render in renders if render.get("camera") == "overview"), {}) if isinstance(renders, list) else {}
|
||||
closeup = next((render for render in renders if render.get("camera") == "closeup"), {}) if isinstance(renders, list) else {}
|
||||
avg_score = sum(scores) / len(scores) if scores else 0.0
|
||||
return {
|
||||
"margin": float(margin),
|
||||
"scene_faces": int(scene_faces),
|
||||
"score": float(avg_score),
|
||||
"package": package_dir.as_posix(),
|
||||
"verify_dir": verify_dir.as_posix(),
|
||||
"contact_sheet": (verify_dir / "visual_acceptance_contact_sheet.png").as_posix(),
|
||||
"overview": render_quality_summary(overview),
|
||||
"closeup": render_quality_summary(closeup),
|
||||
}
|
||||
|
||||
|
||||
def render_quality_summary(render: dict[str, Any]) -> dict[str, Any]:
|
||||
quality = render.get("quality") if isinstance(render, dict) else {}
|
||||
return {
|
||||
"path": render.get("path") if isinstance(render, dict) else None,
|
||||
"score": float(quality.get("score", 0.0)) if isinstance(quality, dict) else 0.0,
|
||||
"black_ratio": float(quality.get("black_ratio", 0.0)) if isinstance(quality, dict) else 0.0,
|
||||
"high_saturation_ratio": float(quality.get("high_saturation_ratio", 0.0)) if isinstance(quality, dict) else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def rank_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return sorted(
|
||||
candidates,
|
||||
key=lambda item: (
|
||||
float(item.get("score", 0.0)),
|
||||
int(item.get("scene_faces", 0)),
|
||||
-float(item.get("margin", 0.0)),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def margin_token(margin: float) -> str:
|
||||
return f"margin_{margin:.3f}".replace(".", "p").replace("-", "m")
|
||||
|
||||
|
||||
def candidate_token(margin: float, scene_faces: int) -> str:
|
||||
return f"{margin_token(margin)}_faces_{int(scene_faces)}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1827
vendor/gs_playground/scripts/real2sim_verify_motrix_package.py
vendored
Normal file
1827
vendor/gs_playground/scripts/real2sim_verify_motrix_package.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1902
vendor/gs_playground/scripts/real2sim_video_motrix_scene.py
vendored
Normal file
1902
vendor/gs_playground/scripts/real2sim_video_motrix_scene.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
108
vendor/gs_playground/scripts/real2sim_wuying_launch_package.sh
vendored
Executable file
108
vendor/gs_playground/scripts/real2sim_wuying_launch_package.sh
vendored
Executable file
@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'USAGE'
|
||||
Usage:
|
||||
scripts/real2sim_wuying_launch_package.sh --archive /path/to/real2sim_scene.tar.gz [--dest /tmp/real2sim_import] [--with-gs]
|
||||
|
||||
Extract a generated one-image Real2Sim Motrix package on the Wuying desktop and
|
||||
launch the visible MotrixSim viewer.
|
||||
USAGE
|
||||
}
|
||||
|
||||
archive=""
|
||||
dest="/tmp/real2sim_import"
|
||||
with_gs=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--archive)
|
||||
archive="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--dest)
|
||||
dest="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--with-gs)
|
||||
with_gs=1
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${archive}" || ! -f "${archive}" ]]; then
|
||||
echo "--archive must point to an existing .tar.gz package" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
mkdir -p "${dest}"
|
||||
tar -xzf "${archive}" -C "${dest}"
|
||||
package_dir="$(find "${dest}" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1)"
|
||||
if [[ -z "${package_dir}" || ! -f "${package_dir}/configs/go2_real2sim.json" ]]; then
|
||||
echo "Extracted package missing configs/go2_real2sim.json under ${dest}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
recommended_camera=""
|
||||
if [[ -f "${package_dir}/motrix_import.json" ]]; then
|
||||
recommended_camera="$(python3 - "${package_dir}/motrix_import.json" <<'PY' || true
|
||||
import json
|
||||
import sys
|
||||
try:
|
||||
data = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
except Exception:
|
||||
data = {}
|
||||
print(data.get("recommended_camera", ""))
|
||||
PY
|
||||
)"
|
||||
fi
|
||||
|
||||
viewer_camera="overview"
|
||||
if [[ "${recommended_camera}" == "source_3dgs_preview" ]]; then
|
||||
viewer_camera="closeup"
|
||||
fi
|
||||
|
||||
args=(--config "${package_dir}/configs/go2_real2sim.json" --camera "${viewer_camera}" --gs-width 900 --gs-height 700 --static-gs-preview)
|
||||
if [[ "${with_gs}" -eq 1 ]]; then
|
||||
args+=(--show-source-gs)
|
||||
if [[ "${recommended_camera}" == "source_3dgs_preview" ]]; then
|
||||
args+=(--source-gs-only)
|
||||
fi
|
||||
args=(--with-gs "${args[@]}")
|
||||
fi
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
runtime_repo="${REAL2SIM_WUYING_RUNTIME_REPO:-$(cd "${script_dir}/.." && pwd)}"
|
||||
if [[ ! -x "${runtime_repo}/scripts/real2sim_wuying_viewer.sh" || ! -x "${runtime_repo}/.venv/bin/python" ]]; then
|
||||
for candidate in \
|
||||
/opt/gs_playground_real2sim_48q \
|
||||
/opt/gs_playground-real2sim_48q \
|
||||
/opt/gs_playground \
|
||||
/workspace/gs_playground \
|
||||
/root/gs_playground; do
|
||||
if [[ -x "${candidate}/scripts/real2sim_wuying_viewer.sh" && -x "${candidate}/.venv/bin/python" ]]; then
|
||||
runtime_repo="${candidate}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [[ ! -x "${runtime_repo}/scripts/real2sim_wuying_viewer.sh" || ! -x "${runtime_repo}/.venv/bin/python" ]]; then
|
||||
echo "Could not find a Real2Sim runtime repo with scripts/real2sim_wuying_viewer.sh and .venv/bin/python." >&2
|
||||
echo "Set REAL2SIM_WUYING_RUNTIME_REPO=/path/to/gs_playground on the Wuying desktop." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Launching MotrixSim Real2Sim package: ${package_dir}"
|
||||
echo "Runtime repo: ${runtime_repo}"
|
||||
cd "${runtime_repo}"
|
||||
exec scripts/real2sim_wuying_viewer.sh "${args[@]}"
|
||||
53
vendor/gs_playground/scripts/real2sim_wuying_paper_env.sh
vendored
Normal file
53
vendor/gs_playground/scripts/real2sim_wuying_paper_env.sh
vendored
Normal file
@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runtime wiring for the 48 GiB Wuying Real2Sim paper-alignment machine.
|
||||
#
|
||||
# Usage:
|
||||
# source scripts/real2sim_wuying_paper_env.sh
|
||||
# python -m real2sim check --paper-strict --config configs/real2sim.external.example.json --backend external
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
export REAL2SIM_PROJECT_ROOT="${REAL2SIM_PROJECT_ROOT:-/opt/gs_playground_real2sim_48q}"
|
||||
export REAL2SIM_RUNTIME_ROOT="${REAL2SIM_RUNTIME_ROOT:-/opt/real2sim_paper_runtime}"
|
||||
|
||||
export REAL2SIM_ANYSPLAT_ROOT="${REAL2SIM_ANYSPLAT_ROOT:-${REAL2SIM_RUNTIME_ROOT}/AnySplat}"
|
||||
export REAL2SIM_SAM2_ROOT="${REAL2SIM_SAM2_ROOT:-${REAL2SIM_RUNTIME_ROOT}/sam2}"
|
||||
export REAL2SIM_SAM3D_ROOT="${REAL2SIM_SAM3D_ROOT:-${REAL2SIM_RUNTIME_ROOT}/sam-3d-objects}"
|
||||
export REAL2SIM_SPEEDY_SPLAT_ROOT="${REAL2SIM_SPEEDY_SPLAT_ROOT:-${REAL2SIM_RUNTIME_ROOT}/speedy-splat}"
|
||||
|
||||
export REAL2SIM_ANY_SPLAT_PYTHON="${REAL2SIM_ANY_SPLAT_PYTHON:-${REAL2SIM_RUNTIME_ROOT}/envs/anysplat/bin/python}"
|
||||
export REAL2SIM_SAM3D_PYTHON="${REAL2SIM_SAM3D_PYTHON:-${REAL2SIM_RUNTIME_ROOT}/envs/sam3d-objects/bin/python}"
|
||||
|
||||
export CUDA_HOME="${CUDA_HOME:-/usr/local/cuda-12.1}"
|
||||
export CONDA_PREFIX="${CONDA_PREFIX:-${CUDA_HOME}}"
|
||||
export LIDRA_SKIP_INIT="${LIDRA_SKIP_INIT:-1}"
|
||||
export PATH="${CUDA_HOME}/bin:${PATH}"
|
||||
export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}"
|
||||
|
||||
export REAL2SIM_SAM2_CHECKPOINT="${REAL2SIM_SAM2_CHECKPOINT:-${REAL2SIM_SAM2_ROOT}/sam2.1_hiera_large.pt}"
|
||||
export REAL2SIM_SAM2_MODEL_CFG="${REAL2SIM_SAM2_MODEL_CFG:-configs/sam2.1/sam2.1_hiera_l.yaml}"
|
||||
export REAL2SIM_SAM3D_CONFIG="${REAL2SIM_SAM3D_CONFIG:-${REAL2SIM_SAM3D_ROOT}/checkpoints/hf/pipeline.yaml}"
|
||||
|
||||
_real2sim_pythonpath_parts=(
|
||||
"${REAL2SIM_SAM2_ROOT}"
|
||||
"${REAL2SIM_SAM3D_ROOT}/notebook"
|
||||
"${REAL2SIM_SAM3D_ROOT}"
|
||||
"${REAL2SIM_ANYSPLAT_ROOT}"
|
||||
"${REAL2SIM_SPEEDY_SPLAT_ROOT}"
|
||||
)
|
||||
if [[ -n "${PYTHONPATH:-}" ]]; then
|
||||
_real2sim_pythonpath_parts+=("${PYTHONPATH}")
|
||||
fi
|
||||
export PYTHONPATH="$(IFS=:; echo "${_real2sim_pythonpath_parts[*]}")"
|
||||
unset _real2sim_pythonpath_parts
|
||||
|
||||
export REAL2SIM_SYNTHETIC_MULTIVIEW_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/synthetic_multiview_expand.py --image {image} --output-dir {synthetic_multiview_dir} --frames-dir {synthetic_frames_dir} --prompt {synthetic_multiview_prompt} --mode {synthetic_multiview_mode} --size 1536x1024"
|
||||
export REAL2SIM_GROUNDING_DINO_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/groundingdino_hf_detect.py --image {image} --prompt {prompt} --output {detections_json} --max-detections 12"
|
||||
export REAL2SIM_SAM2_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/sam2_segment.py --image {image} --detections {detections_json} --masks-dir {masks_dir} --output {instances_json}"
|
||||
export REAL2SIM_INSTANCE_REFINEMENT_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/instance_refine.py --instances {instances_json} --output {instances_json} --image {image} --merge-saturated-accessories --accessory-search-margin-ratio 0.25 --accessory-contact-passes 4 --accessory-saturation-threshold 0.45 --accessory-max-area-growth 0.15 --accessory-min-component-area-ratio 0.001 --accessory-max-components 8"
|
||||
export REAL2SIM_LAMA_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/lama_inpaint.py --image {image} --instances {instances_json} --output {background_image} --prompt {prompt} --work-dir {work_dir} --redetect-command-env REAL2SIM_GROUNDING_DINO_COMMAND --segment-command-env REAL2SIM_SAM2_COMMAND --require-redetect-loop"
|
||||
export REAL2SIM_DEPTH_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/depth_anything_depth.py --image {background_image} --output {depth_map} --model depth-anything/Depth-Anything-V2-Small-hf"
|
||||
export REAL2SIM_ANYSPLAT_COMMAND="${REAL2SIM_ANY_SPLAT_PYTHON} scripts/real2sim_external/anysplat_scene.py --image {background_image} --frames-dir {frames_dir} --output-dir {scene_reconstruction_dir} --ply {scene_ply} --mesh {scene_mesh} --cameras {cameras_json} --depth {depth_map} --max-views 8"
|
||||
export REAL2SIM_SAM3D_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/sam3d_objects_reconstruct.py --image {image} --mask {mask} --instance {instance_json} --output-dir {object_dir} --ply {object_ply} --mesh {object_mesh} --pose {object_pose} --cameras {cameras_json} --match-camera-preprocess --object-centric-preprocess --object-preprocess-size 896 --object-crop-margin 0.12 --canonicalize-gaussian-to-mesh --remove-video-overlay --texture-baking --texture-render-backend gsplat --texture-baking-engine pytorch3d"
|
||||
export REAL2SIM_POSE_ALIGNMENT_COMMAND="${REAL2SIM_SAM3D_PYTHON} scripts/real2sim_external/depth_mask_pose_align.py --assets {aligned_assets_json} --instances {instances_json} --depth {depth_map} --cameras {cameras_json} --work-dir {work_dir} --output {aligned_assets_json}"
|
||||
export REAL2SIM_SPEEDY_SPLAT_COMMAND="${REAL2SIM_ANY_SPLAT_PYTHON} scripts/real2sim_external/speedy_splat_prune.py --input {input} --output {output} --image {source_image} --cameras {cameras_json} --keep-ratio {keep_ratio}"
|
||||
200
vendor/gs_playground/scripts/real2sim_wuying_viewer.sh
vendored
Executable file
200
vendor/gs_playground/scripts/real2sim_wuying_viewer.sh
vendored
Executable file
@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env bash
|
||||
# Launch the generated Real2Sim viewer on the visible Wuying desktop.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${repo_root}"
|
||||
|
||||
foreground=0
|
||||
with_gs=0
|
||||
force_render_app=0
|
||||
package_dir="${REAL2SIM_WUYING_PACKAGE:-}"
|
||||
config_path="${REAL2SIM_WUYING_CONFIG:-}"
|
||||
scene_path="${REAL2SIM_WUYING_SCENE:-}"
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "${1}" in
|
||||
--foreground)
|
||||
foreground=1
|
||||
shift
|
||||
;;
|
||||
--with-gs)
|
||||
with_gs=1
|
||||
force_render_app=1
|
||||
shift
|
||||
;;
|
||||
--render-app)
|
||||
force_render_app=1
|
||||
shift
|
||||
;;
|
||||
--native)
|
||||
force_render_app=0
|
||||
shift
|
||||
;;
|
||||
--package)
|
||||
package_dir="${2:?--package requires a package directory}"
|
||||
shift 2
|
||||
;;
|
||||
--config)
|
||||
config_path="${2:?--config requires a config JSON path}"
|
||||
force_render_app=1
|
||||
shift 2
|
||||
;;
|
||||
--scene)
|
||||
scene_path="${2:?--scene requires an MJCF scene path}"
|
||||
shift 2
|
||||
;;
|
||||
--stop)
|
||||
pkill -f "real2sim_scene_viewer[.]py" || true
|
||||
pkill -f "motrixsim[.]interactive_viewer" || true
|
||||
echo "Stopped Real2Sim viewer processes."
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
unset PYTHONPATH
|
||||
export SDL_AUDIODRIVER="${SDL_AUDIODRIVER:-dummy}"
|
||||
export ALSOFT_DRIVERS="${ALSOFT_DRIVERS:-null}"
|
||||
export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-8.9}"
|
||||
if [[ -z "${XDG_RUNTIME_DIR:-}" && -d "/run/user/$(id -u)" ]]; then
|
||||
export XDG_RUNTIME_DIR="/run/user/$(id -u)"
|
||||
fi
|
||||
|
||||
choose_display() {
|
||||
local first_ok=""
|
||||
local candidate
|
||||
local seen=" "
|
||||
for candidate in "${DISPLAY:-}" :1 :92; do
|
||||
[[ -n "${candidate}" ]] || continue
|
||||
[[ "${seen}" == *" ${candidate} "* ]] && continue
|
||||
seen+="${candidate} "
|
||||
DISPLAY="${candidate}" timeout 2s xdpyinfo >/dev/null 2>&1 || continue
|
||||
if [[ -z "${first_ok}" ]]; then
|
||||
first_ok="${candidate}"
|
||||
fi
|
||||
if DISPLAY="${candidate}" timeout 2s xwininfo -root -tree 2>/dev/null | grep -Eq 'gnome-shell|gnome-terminal|MotrixRender'; then
|
||||
printf '%s\n' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
if [[ -n "${first_ok}" ]]; then
|
||||
printf '%s\n' "${first_ok}"
|
||||
return 0
|
||||
fi
|
||||
printf '%s\n' "${DISPLAY:-:1}"
|
||||
}
|
||||
|
||||
export DISPLAY="$(choose_display)"
|
||||
echo "Launching Real2Sim viewer on DISPLAY=${DISPLAY}"
|
||||
|
||||
resolve_latest_package() {
|
||||
find outputs -path "*/mjcf/scene.xml" -type f -printf '%T@ %h\n' 2>/dev/null \
|
||||
| sort -nr \
|
||||
| head -1 \
|
||||
| awk '{print $2}' \
|
||||
| sed 's#/mjcf$##'
|
||||
}
|
||||
|
||||
if [[ -z "${package_dir}" && ( -z "${scene_path}" || -z "${config_path}" ) ]]; then
|
||||
package_dir="$(resolve_latest_package || true)"
|
||||
fi
|
||||
if [[ -n "${package_dir}" ]]; then
|
||||
if [[ -z "${scene_path}" ]]; then
|
||||
scene_path="${package_dir%/}/mjcf/scene.xml"
|
||||
fi
|
||||
if [[ -z "${config_path}" ]]; then
|
||||
config_path="$(find "${package_dir%/}/configs" -maxdepth 1 -name '*_real2sim.json' -type f 2>/dev/null | sort | head -1 || true)"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$#" -eq 0 ]]; then
|
||||
if [[ "${force_render_app}" -eq 0 ]]; then
|
||||
if [[ -z "${scene_path}" ]]; then
|
||||
echo "No scene path found. Pass --scene or --package." >&2
|
||||
exit 1
|
||||
fi
|
||||
set -- \
|
||||
--file="${scene_path}"
|
||||
elif [[ "${with_gs}" -eq 1 ]]; then
|
||||
if [[ -z "${config_path}" ]]; then
|
||||
echo "No Real2Sim config found. Pass --config or --package." >&2
|
||||
exit 1
|
||||
fi
|
||||
set -- \
|
||||
--config "${config_path}" \
|
||||
--camera overview \
|
||||
--gs-width 320 \
|
||||
--gs-height 210 \
|
||||
--gs-widget-left 930 \
|
||||
--gs-widget-top 10 \
|
||||
--gs-render-fps 8 \
|
||||
--static-gs-preview \
|
||||
--viewer-fps 20 \
|
||||
--frame-sleep-ms 30 \
|
||||
--show-source-gs \
|
||||
--show-system-gs
|
||||
else
|
||||
if [[ -n "${config_path}" ]]; then
|
||||
set -- \
|
||||
--config "${config_path}" \
|
||||
--camera system \
|
||||
--no-gs \
|
||||
--viewer-fps 12 \
|
||||
--frame-sleep-ms 55
|
||||
elif [[ -n "${scene_path}" ]]; then
|
||||
set -- \
|
||||
--scene "${scene_path}" \
|
||||
--camera system \
|
||||
--no-gs \
|
||||
--viewer-fps 12 \
|
||||
--frame-sleep-ms 55
|
||||
else
|
||||
echo "No Real2Sim config or scene found. Pass --config, --scene, or --package." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
force_render_app=1
|
||||
parsed_args=()
|
||||
if [[ -n "${config_path}" ]]; then
|
||||
parsed_args+=(--config "${config_path}")
|
||||
elif [[ -n "${scene_path}" ]]; then
|
||||
parsed_args+=(--scene "${scene_path}")
|
||||
fi
|
||||
set -- "${parsed_args[@]}" "$@"
|
||||
fi
|
||||
|
||||
pkill -f "real2sim_scene_viewer[.]py" || true
|
||||
pkill -f "motrixsim[.]interactive_viewer" || true
|
||||
|
||||
if [[ "${foreground}" -eq 1 ]]; then
|
||||
if [[ "${force_render_app}" -eq 0 ]]; then
|
||||
exec .venv/bin/python -m motrixsim.interactive_viewer "$@"
|
||||
fi
|
||||
exec .venv/bin/python scripts/real2sim_scene_viewer.py "$@"
|
||||
fi
|
||||
|
||||
log_path="${REAL2SIM_WUYING_VIEWER_LOG:-/tmp/real2sim_wuying_viewer.log}"
|
||||
rm -f "${log_path}"
|
||||
if [[ "${force_render_app}" -eq 0 ]]; then
|
||||
cmd=(.venv/bin/python -m motrixsim.interactive_viewer "$@")
|
||||
else
|
||||
cmd=(.venv/bin/python scripts/real2sim_scene_viewer.py "$@")
|
||||
fi
|
||||
printf 'Command:' >"${log_path}"
|
||||
printf ' %q' "${cmd[@]}" >>"${log_path}"
|
||||
printf '\n' >>"${log_path}"
|
||||
setsid "${cmd[@]}" >>"${log_path}" 2>&1 < /dev/null &
|
||||
pid=$!
|
||||
echo "Started Real2Sim viewer PID=${pid}"
|
||||
echo "Log: ${log_path}"
|
||||
sleep 2
|
||||
if ! ps -p "${pid}" >/dev/null 2>&1; then
|
||||
echo "Viewer exited early; last log lines:" >&2
|
||||
tail -80 "${log_path}" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user