gs_playground/scripts/real2sim_external/sam3d_objects_reconstruct.py
2026-06-11 21:14:51 +08:00

898 lines
36 KiB
Python

#!/usr/bin/env python3
"""SAM 3D Objects adapter for one Real2Sim object.
This follows Meta's public quick-start shape:
from inference import Inference, load_image, load_single_mask
output = inference(image, mask)
output["gs"].save_ply("splat.ply")
The script intentionally fails with a clear message if the gated checkpoints
or SAM 3D environment are not available.
"""
from __future__ import annotations
import argparse
import importlib
import inspect
import json
import os
import struct
import sys
from pathlib import Path
import numpy as np
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, REPO_ROOT.as_posix())
from real2sim.contracts import Pose, read_json, write_json
from real2sim.image_io import read_gray, write_gray
class PreparedSam3DInputs:
def __init__(self, image_path: Path, mask_path: Path, mask: np.ndarray, image_size: tuple[int, int], metadata: dict):
self.image_path = image_path
self.mask_path = mask_path
self.mask = mask
self.image_size = image_size
self.metadata = metadata
class PlyVertexTable:
def __init__(
self,
*,
header_lines: list[str],
body_start: int,
fmt: str,
vertex_count: int,
property_names: list[str],
property_types: list[str],
array: np.ndarray,
):
self.header_lines = header_lines
self.body_start = body_start
self.fmt = fmt
self.vertex_count = vertex_count
self.property_names = property_names
self.property_types = property_types
self.array = array
def main() -> int:
parser = argparse.ArgumentParser(description="Run SAM 3D Objects for one detected object")
parser.add_argument("--image", type=Path, required=True)
parser.add_argument("--mask", type=Path, required=True)
parser.add_argument("--instance", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--ply", type=Path, required=True)
parser.add_argument("--mesh", type=Path, required=True)
parser.add_argument("--pose", type=Path, required=True)
parser.add_argument("--cameras", type=Path, default=None, help="AnySplat cameras JSON used to match image preprocessing")
parser.add_argument("--match-camera-preprocess", action="store_true", help="Run SAM-3D on the same aspect-preserving center crop used by AnySplat")
parser.add_argument("--object-centric-preprocess", action="store_true", help="Crop around the instance mask before SAM-3D preprocessing")
parser.add_argument("--object-preprocess-size", type=int, default=0, help="Override SAM-3D object input size with a square image")
parser.add_argument("--object-crop-margin", type=float, default=0.12, help="Relative padding around the instance mask for object-centric preprocessing")
parser.add_argument("--remove-video-overlay", action="store_true", help="Inpaint a central video play overlay before SAM-3D inference")
parser.add_argument("--canonicalize-gaussian-to-mesh", action="store_true", help="Rotate SAM-3D Gaussian PLY into the mesh canonical frame")
parser.add_argument("--texture-baking", action="store_true", help="Use SAM-3D's textured GLB postprocess path for the exported preview mesh")
parser.add_argument("--texture-render-backend", choices=("auto", "inria", "gsplat"), default="gsplat")
parser.add_argument("--texture-baking-engine", choices=("nvdiffrast", "pytorch3d"), default="pytorch3d")
parser.add_argument("--sam3d-root", type=Path, default=os.environ.get("REAL2SIM_SAM3D_ROOT"))
parser.add_argument("--config", type=Path, default=os.environ.get("REAL2SIM_SAM3D_CONFIG"))
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--allow-fallback-mesh", action="store_true", help="Write a collision proxy if SAM 3D returns no exportable mesh")
args = parser.parse_args()
if args.sam3d_root is None:
raise SystemExit("SAM 3D root is required via --sam3d-root or REAL2SIM_SAM3D_ROOT")
if args.config is None:
args.config = args.sam3d_root / "checkpoints" / "hf" / "pipeline.yaml"
if not args.config.exists():
raise SystemExit(f"SAM 3D config/checkpoints are not available: {args.config}")
sys.path.insert(0, (args.sam3d_root / "notebook").as_posix())
try:
from inference import Inference, load_image
except Exception as exc:
raise SystemExit(f"failed to import SAM 3D inference from {args.sam3d_root}: {exc}") from exc
import torch
args.output_dir.mkdir(parents=True, exist_ok=True)
image_path = args.image
mask_np = (read_gray(args.mask) > 0.5).astype(np.uint8)
preprocess_metadata = None
overlay_metadata = None
if args.match_camera_preprocess:
if args.cameras is None:
raise SystemExit("--match-camera-preprocess requires --cameras")
target_shape = None
if args.object_preprocess_size > 0:
target_shape = (int(args.object_preprocess_size), int(args.object_preprocess_size))
prepared = prepare_camera_preprocessed_inputs(
args.image,
args.mask,
args.cameras,
args.output_dir / "camera_preprocess",
object_centric=args.object_centric_preprocess,
target_shape=target_shape,
object_margin_ratio=args.object_crop_margin,
)
image_path = prepared.image_path
mask_np = prepared.mask.astype(np.uint8)
preprocess_metadata = prepared.metadata
if args.remove_video_overlay:
image_path, overlay_metadata = remove_center_video_overlay(image_path, mask_np > 0, args.output_dir / "video_overlay_removal")
image = load_image(image_path.as_posix())
inference = Inference(args.config.as_posix(), compile=False)
with torch.inference_mode():
output = run_sam3d_inference(
inference,
image,
mask_np,
seed=args.seed,
texture_baking=args.texture_baking,
texture_render_backend=args.texture_render_backend,
texture_baking_engine=args.texture_baking_engine,
)
save_gaussian(output, args.ply)
wrote_mesh = export_mesh(output, args.mesh, prefer_textured=args.texture_baking)
if not wrote_mesh and args.allow_fallback_mesh:
write_bbox_mesh_from_mask(args.mesh, mask_np)
wrote_mesh = True
if not wrote_mesh:
raise SystemExit(
"SAM 3D output did not include an exportable mesh. "
"Use --allow-fallback-mesh only for collision-proxy debugging. "
f"Output keys: {sorted(output.keys())}"
)
canonicalization_metadata = None
if args.canonicalize_gaussian_to_mesh:
canonicalization_metadata = canonicalize_gaussian_to_mesh_frame(args.ply, args.mesh)
pose = Pose()
layout = output.get("layout") or output.get("pose") or output
if isinstance(layout, dict):
pose = pose_from_layout(layout)
write_json(args.pose, pose.to_json())
metadata = {
"backend": "sam3d_objects",
"source_instance": read_json(args.instance),
"sam3d_config": args.config.as_posix(),
"texture_baking": bool(args.texture_baking),
"texture_render_backend": args.texture_render_backend,
"texture_baking_engine": args.texture_baking_engine,
}
if preprocess_metadata is not None:
metadata["camera_preprocess"] = preprocess_metadata
if overlay_metadata is not None:
metadata["video_overlay_removal"] = overlay_metadata
if canonicalization_metadata is not None:
metadata["gaussian_canonicalization"] = canonicalization_metadata
write_json(args.output_dir / "sam3d_metadata.json", metadata)
return 0
def prepare_camera_preprocessed_inputs(
image_path: Path,
mask_path: Path,
cameras_path: Path,
output_dir: Path,
*,
object_centric: bool = False,
target_shape: tuple[int, int] | None = None,
object_margin_ratio: float = 0.12,
) -> PreparedSam3DInputs:
cameras = read_json(cameras_path)
target_h, target_w = target_shape or camera_image_shape(cameras)
from PIL import Image
image = Image.open(image_path).convert("RGB")
mask = read_gray(mask_path) > 0.5
if object_centric and mask.any():
processed_image, processed_mask, crop_meta = object_centric_resize_pad_image_and_mask(
image,
mask,
(target_h, target_w),
margin_ratio=object_margin_ratio,
resample=Image.BICUBIC,
)
else:
processed_image, crop_meta = cover_resize_center_crop_image(image, (target_h, target_w), resample=Image.BICUBIC)
processed_mask = cover_resize_center_crop_mask(mask, (target_h, target_w), crop_meta)
crop_meta = {"preprocess_mode": "camera_center_crop", **crop_meta}
output_dir.mkdir(parents=True, exist_ok=True)
processed_image_path = output_dir / "sam3d_camera_image.png"
processed_mask_path = output_dir / "sam3d_camera_mask.pgm"
processed_image.save(processed_image_path)
write_gray(processed_mask_path, processed_mask.astype(np.float32))
metadata = {
"source_image": image_path.as_posix(),
"source_mask": mask_path.as_posix(),
"cameras": cameras_path.as_posix(),
"source_size": [int(image.height), int(image.width)],
"target_size": [int(target_h), int(target_w)],
**crop_meta,
"image": processed_image_path.as_posix(),
"mask": processed_mask_path.as_posix(),
"mask_pixels": int(processed_mask.sum()),
}
return PreparedSam3DInputs(
image_path=processed_image_path,
mask_path=processed_mask_path,
mask=processed_mask,
image_size=(target_h, target_w),
metadata=metadata,
)
def camera_image_shape(cameras: dict) -> tuple[int, int]:
shape = cameras.get("image_shape") if isinstance(cameras, dict) else None
if isinstance(shape, (list, tuple)) and len(shape) >= 2:
return (int(shape[0]), int(shape[1]))
intrinsics = cameras.get("intrinsics", {}) if isinstance(cameras, dict) else {}
height = int(intrinsics.get("height", 448))
width = int(intrinsics.get("width", 448))
return (height, width)
def cover_resize_center_crop_image(image, shape: tuple[int, int], *, resample) -> tuple:
dst_h, dst_w = shape
src_w, src_h = image.size
scale = max(dst_w / float(src_w), dst_h / float(src_h))
resized_w = max(dst_w, int(src_w * scale))
resized_h = max(dst_h, int(src_h * scale))
resized = image.resize((resized_w, resized_h), resample)
left = max(0, (resized_w - dst_w) // 2)
top = max(0, (resized_h - dst_h) // 2)
crop = resized.crop((left, top, left + dst_w, top + dst_h))
return crop, {
"resized_size": [int(resized_h), int(resized_w)],
"crop_xyxy": [int(left), int(top), int(left + dst_w), int(top + dst_h)],
"scale": float(scale),
}
def cover_resize_center_crop_mask(mask: np.ndarray, shape: tuple[int, int], crop_meta: dict) -> np.ndarray:
from PIL import Image
dst_h, dst_w = shape
resized_h, resized_w = crop_meta["resized_size"]
left, top, right, bottom = crop_meta["crop_xyxy"]
mask_img = Image.fromarray(mask.astype(np.uint8) * 255)
resized = mask_img.resize((int(resized_w), int(resized_h)), Image.NEAREST)
cropped = resized.crop((int(left), int(top), int(right), int(bottom)))
arr = np.asarray(cropped, dtype=np.uint8) > 127
if arr.shape == (dst_h, dst_w):
return arr
out = np.zeros((dst_h, dst_w), dtype=bool)
out[: arr.shape[0], : arr.shape[1]] = arr
return out
def object_centric_resize_pad_image_and_mask(image, mask: np.ndarray, shape: tuple[int, int], *, margin_ratio: float, resample) -> tuple:
from PIL import Image
dst_h, dst_w = shape
src_w, src_h = image.size
x0, y0, x1, y1 = mask_bbox_xyxy(mask)
box_w = max(1, x1 - x0)
box_h = max(1, y1 - y0)
margin = max(1, int(round(max(box_w, box_h) * max(float(margin_ratio), 0.0))))
crop_x0 = max(0, x0 - margin)
crop_y0 = max(0, y0 - margin)
crop_x1 = min(src_w, x1 + margin)
crop_y1 = min(src_h, y1 + margin)
if crop_x1 <= crop_x0 or crop_y1 <= crop_y0:
processed_image, crop_meta = cover_resize_center_crop_image(image, shape, resample=resample)
processed_mask = cover_resize_center_crop_mask(mask, shape, crop_meta)
return processed_image, processed_mask, {"preprocess_mode": "camera_center_crop_empty_object", **crop_meta}
crop = image.crop((crop_x0, crop_y0, crop_x1, crop_y1))
crop_mask = mask[crop_y0:crop_y1, crop_x0:crop_x1]
crop_w, crop_h = crop.size
scale = min(dst_w / float(crop_w), dst_h / float(crop_h))
resized_w = max(1, min(dst_w, int(round(crop_w * scale))))
resized_h = max(1, min(dst_h, int(round(crop_h * scale))))
offset_x = (dst_w - resized_w) // 2
offset_y = (dst_h - resized_h) // 2
fill = local_background_color(crop, crop_mask)
canvas = Image.new("RGB", (dst_w, dst_h), fill)
resized = crop.resize((resized_w, resized_h), resample)
canvas.paste(resized, (offset_x, offset_y))
mask_canvas = np.zeros((dst_h, dst_w), dtype=bool)
resized_mask = Image.fromarray(crop_mask.astype(np.uint8) * 255).resize((resized_w, resized_h), Image.NEAREST)
mask_arr = np.asarray(resized_mask, dtype=np.uint8) > 127
mask_canvas[offset_y : offset_y + resized_h, offset_x : offset_x + resized_w] = mask_arr
return canvas, mask_canvas, {
"preprocess_mode": "object_centric",
"object_bbox_xyxy": [int(x0), int(y0), int(x1), int(y1)],
"source_crop_xyxy": [int(crop_x0), int(crop_y0), int(crop_x1), int(crop_y1)],
"resized_size": [int(resized_h), int(resized_w)],
"pad_xy": [int(offset_x), int(offset_y)],
"scale": float(scale),
"object_margin_ratio": float(margin_ratio),
}
def mask_bbox_xyxy(mask: np.ndarray) -> tuple[int, int, int, int]:
ys, xs = np.where(np.asarray(mask, dtype=bool))
if len(xs) == 0:
return (0, 0, int(mask.shape[1]), int(mask.shape[0]))
return (int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1)
def local_background_color(image, mask: np.ndarray) -> tuple[int, int, int]:
arr = np.asarray(image.convert("RGB"), dtype=np.uint8)
bg = arr[~np.asarray(mask, dtype=bool)]
values = bg if bg.size else arr.reshape(-1, 3)
color = np.median(values.reshape(-1, 3), axis=0)
return tuple(int(np.clip(round(float(channel)), 0, 255)) for channel in color)
def remove_center_video_overlay(image_path: Path, object_mask: np.ndarray, output_dir: Path) -> tuple[Path, dict]:
from PIL import Image
image = Image.open(image_path).convert("RGB")
arr = np.asarray(image, dtype=np.uint8)
mask = np.asarray(object_mask, dtype=bool)
if mask.shape != arr.shape[:2]:
mask = resize_bool_mask(mask, arr.shape[:2])
overlay_mask = detect_center_video_overlay(arr, mask)
overlay_pixels = int(overlay_mask.sum())
metadata = {
"source_image": image_path.as_posix(),
"detected": overlay_pixels > 0,
"overlay_pixels": overlay_pixels,
}
if overlay_pixels == 0:
return image_path, metadata
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / "sam3d_overlay_removed.png"
try:
import cv2
inpaint_mask = (overlay_mask.astype(np.uint8) * 255)
cleaned = cv2.inpaint(arr, inpaint_mask, 5, cv2.INPAINT_TELEA)
except Exception:
cleaned = simple_mask_fill(arr, overlay_mask)
metadata["fallback"] = "simple_mask_fill"
Image.fromarray(cleaned.astype(np.uint8)).save(output_path)
metadata["image"] = output_path.as_posix()
return output_path, metadata
def detect_center_video_overlay(image: np.ndarray, object_mask: np.ndarray) -> np.ndarray:
height, width = image.shape[:2]
yy, xx = np.ogrid[:height, :width]
cx = width * 0.5
cy = height * 0.5
radius = max(8.0, min(height, width) * 0.11)
disk = (xx - cx) ** 2 + (yy - cy) ** 2 <= radius ** 2
rgb = image.astype(np.float32) / 255.0
max_channel = rgb.max(axis=2)
min_channel = rgb.min(axis=2)
saturation = (max_channel - min_channel) / np.maximum(max_channel, 1e-6)
bright_neutral = (max_channel > 0.72) & (saturation < 0.28)
raw = disk & object_mask & bright_neutral
if int(raw.sum()) < max(20, int(radius)):
return np.zeros((height, width), dtype=bool)
return dilate_bool_mask(raw, radius=3)
def resize_bool_mask(mask: np.ndarray, shape: tuple[int, int]) -> np.ndarray:
from PIL import Image
height, width = shape
resized = Image.fromarray(mask.astype(np.uint8) * 255).resize((width, height), Image.NEAREST)
return np.asarray(resized, dtype=np.uint8) > 127
def dilate_bool_mask(mask: np.ndarray, *, radius: int) -> np.ndarray:
try:
import cv2
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius * 2 + 1, radius * 2 + 1))
return cv2.dilate(mask.astype(np.uint8), kernel, iterations=1) > 0
except Exception:
padded = np.pad(mask, radius, mode="constant", constant_values=False)
out = np.zeros_like(mask, dtype=bool)
for dy in range(radius * 2 + 1):
for dx in range(radius * 2 + 1):
if (dx - radius) ** 2 + (dy - radius) ** 2 <= radius ** 2:
out |= padded[dy : dy + mask.shape[0], dx : dx + mask.shape[1]]
return out
def simple_mask_fill(image: np.ndarray, mask: np.ndarray) -> np.ndarray:
cleaned = image.copy()
if not mask.any():
return cleaned
yy, xx = np.where(mask)
y0, y1 = max(0, int(yy.min()) - 5), min(image.shape[0], int(yy.max()) + 6)
x0, x1 = max(0, int(xx.min()) - 5), min(image.shape[1], int(xx.max()) + 6)
patch = image[y0:y1, x0:x1]
patch_mask = mask[y0:y1, x0:x1]
fill_values = patch[~patch_mask]
fill = np.median(fill_values, axis=0) if fill_values.size else np.median(image.reshape(-1, 3), axis=0)
cleaned[mask] = fill.astype(np.uint8)
return cleaned
PLY_NUMPY_DTYPES = {
"char": "i1",
"int8": "i1",
"uchar": "u1",
"uint8": "u1",
"short": "i2",
"int16": "i2",
"ushort": "u2",
"uint16": "u2",
"int": "i4",
"int32": "i4",
"uint": "u4",
"uint32": "u4",
"float": "f4",
"float32": "f4",
"double": "f8",
"float64": "f8",
}
def canonicalize_gaussian_to_mesh_frame(gaussian_ply: Path, mesh_obj: Path) -> dict:
table = load_ply_vertex_table(gaussian_ply)
mesh_vertices = load_obj_vertices(mesh_obj)
if table.array.size == 0 or mesh_vertices.size == 0:
return {
"enabled": True,
"status": "skipped_empty_geometry",
"gaussian": gaussian_ply.as_posix(),
"mesh": mesh_obj.as_posix(),
}
xyz_indices = indices_for(table.property_names, ("x", "y", "z"))
points = table.array[:, xyz_indices]
gaussian_extent = extent(points)
mesh_extent = extent(mesh_vertices)
rotation, mapping, signs = canonical_axis_rotation(gaussian_extent, mesh_extent)
changed = not np.allclose(rotation, np.eye(3), atol=1e-6)
if changed:
apply_axis_rotation_to_ply_table(table, rotation)
write_ply_vertex_table(gaussian_ply, table)
return {
"enabled": True,
"status": "ok",
"changed": bool(changed),
"gaussian": gaussian_ply.as_posix(),
"mesh": mesh_obj.as_posix(),
"gaussian_extent_before": vector_json(gaussian_extent),
"mesh_extent": vector_json(mesh_extent),
"axis_mapping": {
"source": "extent_permutation",
"gaussian_axes_to_mesh_axes": [int(value) for value in mapping],
"mesh_axis_signs": [int(value) for value in signs],
},
"rotation_matrix": [[round(float(value), 7) for value in row] for row in rotation.tolist()],
}
def canonical_axis_rotation(gaussian_extent: np.ndarray, mesh_extent: np.ndarray) -> tuple[np.ndarray, list[int], list[int]]:
gaussian_order = list(np.argsort(-np.asarray(gaussian_extent, dtype=np.float32)))
mesh_order = list(np.argsort(-np.asarray(mesh_extent, dtype=np.float32)))
mapping = [0, 1, 2]
for mesh_axis, gaussian_axis in zip(mesh_order, gaussian_order):
mapping[int(mesh_axis)] = int(gaussian_axis)
signs = [1, 1, 1]
rotation = rotation_from_axis_mapping(mapping, signs)
if np.linalg.det(rotation) < 0.0:
signs[int(mesh_order[-1])] = -1
rotation = rotation_from_axis_mapping(mapping, signs)
return rotation.astype(np.float32), mapping, signs
def rotation_from_axis_mapping(mapping: list[int], signs: list[int]) -> np.ndarray:
rotation = np.zeros((3, 3), dtype=np.float32)
for mesh_axis, gaussian_axis in enumerate(mapping):
rotation[mesh_axis, gaussian_axis] = float(signs[mesh_axis])
return rotation
def apply_axis_rotation_to_ply_table(table: PlyVertexTable, rotation: np.ndarray) -> None:
names = table.property_names
xyz_indices = indices_for(names, ("x", "y", "z"))
table.array[:, xyz_indices] = table.array[:, xyz_indices] @ rotation.T
if all(name in names for name in ("nx", "ny", "nz")):
normal_indices = indices_for(names, ("nx", "ny", "nz"))
table.array[:, normal_indices] = table.array[:, normal_indices] @ rotation.T
if all(name in names for name in ("rot_0", "rot_1", "rot_2", "rot_3")):
rot_indices = indices_for(names, ("rot_0", "rot_1", "rot_2", "rot_3"))
transform_quat = quat_wxyz_from_rotation_matrix(rotation)
for idx in range(table.array.shape[0]):
table.array[idx, rot_indices] = quat_multiply(transform_quat, normalize_quat(table.array[idx, rot_indices]))
def load_ply_vertex_table(path: Path) -> PlyVertexTable:
data = path.read_bytes()
header_marker = b"end_header"
marker_index = data.find(header_marker)
if marker_index < 0:
raise SystemExit(f"PLY missing end_header: {path}")
body_start = data.find(b"\n", marker_index)
if body_start < 0:
raise SystemExit(f"PLY header missing final newline: {path}")
body_start += 1
header_lines = data[:body_start].decode("ascii", errors="ignore").splitlines()
fmt = "ascii"
vertex_count = 0
property_names: list[str] = []
property_types: list[str] = []
current_element = ""
for line in header_lines:
parts = line.strip().split()
if len(parts) >= 3 and parts[0] == "format":
fmt = parts[1]
elif len(parts) >= 3 and parts[0] == "element":
current_element = parts[1]
if current_element == "vertex":
vertex_count = int(parts[2])
elif len(parts) >= 3 and parts[0] == "property" and current_element == "vertex":
if parts[1] == "list":
raise SystemExit(f"PLY vertex list properties are not supported for canonicalization: {path}")
property_types.append(parts[1])
property_names.append(parts[-1])
if vertex_count <= 0:
return PlyVertexTable(
header_lines=header_lines,
body_start=body_start,
fmt=fmt,
vertex_count=0,
property_names=property_names,
property_types=property_types,
array=np.zeros((0, len(property_names)), dtype=np.float32),
)
if not all(name in property_names for name in ("x", "y", "z")):
raise SystemExit(f"PLY must expose x/y/z vertex properties: {path}")
body = data[body_start:]
if fmt == "ascii":
array = load_ascii_ply_vertex_array(body, vertex_count, len(property_names))
elif fmt in {"binary_little_endian", "binary_big_endian"}:
array = load_binary_ply_vertex_array(body, vertex_count, property_types, little_endian=fmt == "binary_little_endian")
else:
raise SystemExit(f"unsupported PLY format for canonicalization: {fmt}")
return PlyVertexTable(
header_lines=header_lines,
body_start=body_start,
fmt=fmt,
vertex_count=vertex_count,
property_names=property_names,
property_types=property_types,
array=array.astype(np.float32, copy=False),
)
def load_ascii_ply_vertex_array(body: bytes, vertex_count: int, property_count: int) -> np.ndarray:
rows = []
for line in body.decode("ascii", errors="ignore").splitlines():
if len(rows) >= vertex_count:
break
stripped = line.strip()
if not stripped:
continue
values = [float(value) for value in stripped.split()[:property_count]]
if len(values) == property_count:
rows.append(values)
return np.asarray(rows, dtype=np.float32)
def load_binary_ply_vertex_array(body: bytes, vertex_count: int, property_types: list[str], *, little_endian: bool) -> np.ndarray:
byte_order = "<" if little_endian else ">"
try:
dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(property_types)])
except KeyError as exc:
raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc
structured = np.frombuffer(body, dtype=dtype, count=vertex_count)
columns = [structured[f"f{idx}"].astype(np.float32) for idx in range(len(property_types))]
return np.stack(columns, axis=1) if columns else np.zeros((vertex_count, 0), dtype=np.float32)
def write_ply_vertex_table(path: Path, table: PlyVertexTable) -> None:
header = "\n".join(table.header_lines) + "\n"
if table.fmt == "ascii":
lines = [" ".join(f"{float(value):.9g}" for value in row) for row in table.array]
path.write_text(header + "\n".join(lines) + "\n", encoding="ascii")
return
if table.fmt in {"binary_little_endian", "binary_big_endian"}:
byte_order = "<" if table.fmt == "binary_little_endian" else ">"
try:
dtype = np.dtype([(f"f{idx}", byte_order + PLY_NUMPY_DTYPES[kind]) for idx, kind in enumerate(table.property_types)])
except KeyError as exc:
raise SystemExit(f"unsupported PLY scalar type for canonicalization: {exc}") from exc
structured = np.empty(table.array.shape[0], dtype=dtype)
for idx in range(table.array.shape[1]):
structured[f"f{idx}"] = table.array[:, idx].astype(dtype[idx])
path.write_bytes(header.encode("ascii") + structured.tobytes())
return
raise SystemExit(f"unsupported PLY format for canonicalization: {table.fmt}")
def load_obj_vertices(path: Path) -> np.ndarray:
vertices = []
with path.open("r", encoding="utf-8", errors="ignore") as f:
for line in f:
if not line.startswith("v "):
continue
parts = line.split()
if len(parts) >= 4:
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
return np.asarray(vertices, dtype=np.float32)
def indices_for(names: list[str], required: tuple[str, ...]) -> list[int]:
return [names.index(name) for name in required]
def extent(points: np.ndarray) -> np.ndarray:
if points.size == 0:
return np.zeros(3, dtype=np.float32)
return np.asarray(points.max(axis=0) - points.min(axis=0), dtype=np.float32)
def normalize_quat(raw: np.ndarray) -> np.ndarray:
quat = np.asarray(raw, dtype=np.float32).reshape(4)
norm = float(np.linalg.norm(quat))
if norm <= 1e-8:
return np.asarray([1.0, 0.0, 0.0, 0.0], dtype=np.float32)
return quat / norm
def quat_multiply(a: np.ndarray, b: np.ndarray) -> np.ndarray:
aw, ax, ay, az = normalize_quat(a)
bw, bx, by, bz = normalize_quat(b)
return normalize_quat(
np.asarray(
[
aw * bw - ax * bx - ay * by - az * bz,
aw * bx + ax * bw + ay * bz - az * by,
aw * by - ax * bz + ay * bw + az * bx,
aw * bz + ax * by - ay * bx + az * bw,
],
dtype=np.float32,
)
)
def quat_wxyz_from_rotation_matrix(rotation: np.ndarray) -> np.ndarray:
matrix = np.asarray(rotation, dtype=np.float64).reshape(3, 3)
trace = float(np.trace(matrix))
if trace > 0.0:
s = np.sqrt(trace + 1.0) * 2.0
quat = np.asarray(
[
0.25 * s,
(matrix[2, 1] - matrix[1, 2]) / s,
(matrix[0, 2] - matrix[2, 0]) / s,
(matrix[1, 0] - matrix[0, 1]) / s,
],
dtype=np.float64,
)
else:
axis = int(np.argmax(np.diag(matrix)))
if axis == 0:
s = np.sqrt(1.0 + matrix[0, 0] - matrix[1, 1] - matrix[2, 2]) * 2.0
quat = np.asarray(
[
(matrix[2, 1] - matrix[1, 2]) / s,
0.25 * s,
(matrix[0, 1] + matrix[1, 0]) / s,
(matrix[0, 2] + matrix[2, 0]) / s,
],
dtype=np.float64,
)
elif axis == 1:
s = np.sqrt(1.0 + matrix[1, 1] - matrix[0, 0] - matrix[2, 2]) * 2.0
quat = np.asarray(
[
(matrix[0, 2] - matrix[2, 0]) / s,
(matrix[0, 1] + matrix[1, 0]) / s,
0.25 * s,
(matrix[1, 2] + matrix[2, 1]) / s,
],
dtype=np.float64,
)
else:
s = np.sqrt(1.0 + matrix[2, 2] - matrix[0, 0] - matrix[1, 1]) * 2.0
quat = np.asarray(
[
(matrix[1, 0] - matrix[0, 1]) / s,
(matrix[0, 2] + matrix[2, 0]) / s,
(matrix[1, 2] + matrix[2, 1]) / s,
0.25 * s,
],
dtype=np.float64,
)
return normalize_quat(quat.astype(np.float32))
def vector_json(values: np.ndarray) -> list[float]:
return [round(float(value), 7) for value in np.asarray(values, dtype=np.float32).reshape(-1).tolist()]
def pose_from_layout(layout: dict) -> Pose:
position = layout.get("position", layout.get("translation", [0.0, 0.0, 0.0]))
scale = layout.get("scale", [1.0, 1.0, 1.0])
quat = layout.get("quat_wxyz", layout.get("quaternion_wxyz", layout.get("rotation", [1.0, 0.0, 0.0, 0.0])))
if "quaternion_xyzw" in layout and "quat_wxyz" not in layout and "quaternion_wxyz" not in layout and "rotation" not in layout:
qx, qy, qz, qw = vector_from_value(layout["quaternion_xyzw"], 4, "quaternion_xyzw")
quat = (qw, qx, qy, qz)
return Pose(
position=vector_from_value(position, 3, "position"),
quat_wxyz=vector_from_value(quat, 4, "quat_wxyz"),
scale=vector_from_value(scale, 3, "scale", broadcast_scalar=True),
)
def run_sam3d_inference(
inference,
image: np.ndarray,
mask_np: np.ndarray,
*,
seed: int,
texture_baking: bool,
texture_render_backend: str = "gsplat",
texture_baking_engine: str = "pytorch3d",
) -> dict:
if not texture_baking:
return inference(image, mask_np, seed=seed)
patch_texture_baking_render_backend(texture_render_backend)
pipeline = getattr(inference, "_pipeline", None)
merge_mask_to_rgba = getattr(inference, "merge_mask_to_rgba", None)
if pipeline is None or not hasattr(pipeline, "run") or merge_mask_to_rgba is None:
raise SystemExit("--texture-baking requires SAM-3D notebook Inference._pipeline.run support")
if hasattr(pipeline, "rendering_engine"):
pipeline.rendering_engine = texture_baking_engine
rgba = merge_mask_to_rgba(image, mask_np)
options = {
"stage1_only": False,
"with_mesh_postprocess": False,
"with_texture_baking": True,
"with_layout_postprocess": False,
"use_vertex_color": False,
"stage1_inference_steps": None,
"pointmap": None,
}
return pipeline.run(rgba, None, seed, **supported_kwargs(pipeline.run, options))
def patch_texture_baking_render_backend(backend: str) -> bool:
if backend in {"", "auto", "inria"}:
return False
if backend != "gsplat":
raise SystemExit(f"unsupported SAM-3D texture render backend: {backend}")
render_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.render_utils")
postprocessing_utils = importlib.import_module("sam3d_objects.model.backbone.tdfy_dit.utils.postprocessing_utils")
def render_multiview_with_backend(sample, resolution=512, nviews=30):
radius = 2
fov = 40
cameras = [render_utils.sphere_hammersley_sequence(index, nviews) for index in range(nviews)]
yaws = [camera[0] for camera in cameras]
pitches = [camera[1] for camera in cameras]
extrinsics, intrinsics = render_utils.yaw_pitch_r_fov_to_extrinsics_intrinsics(yaws, pitches, radius, fov)
result = render_utils.render_frames(
sample,
extrinsics,
intrinsics,
{"resolution": resolution, "bg_color": (0, 0, 0), "backend": "gsplat"},
)
return result["color"], extrinsics, intrinsics
postprocessing_utils.render_multiview = render_multiview_with_backend
return True
def supported_kwargs(callable_obj, options: dict[str, object]) -> dict[str, object]:
signature = inspect.signature(callable_obj)
if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values()):
return options
return {key: value for key, value in options.items() if key in signature.parameters}
def save_gaussian(output: dict, path: Path) -> None:
for key in ("gs", "gaussian", "gaussians"):
if key not in output:
continue
value = first_value(output[key])
if hasattr(value, "save_ply"):
path.parent.mkdir(parents=True, exist_ok=True)
value.save_ply(path.as_posix())
return
raise SystemExit(f"SAM 3D output does not expose a Gaussian save_ply method; keys={sorted(output.keys())}")
def export_mesh(output: dict, path: Path, *, prefer_textured: bool = False) -> bool:
keys = ("glb", "mesh", "trimesh", "obj") if prefer_textured else ("mesh", "trimesh", "glb", "obj")
for key in keys:
if key not in output:
continue
value = first_value(output[key])
if hasattr(value, "export"):
path.parent.mkdir(parents=True, exist_ok=True)
value.export(path.as_posix())
return True
if isinstance(value, (str, os.PathLike)):
source = Path(value)
if source.is_file():
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(source.read_bytes())
return True
return False
def first_value(value):
if isinstance(value, (list, tuple)):
return value[0] if value else value
return value
def vector_from_value(value, size: int, field_name: str, *, broadcast_scalar: bool = False) -> tuple[float, ...]:
if hasattr(value, "detach"):
value = value.detach().cpu().numpy()
arr = np.asarray(value, dtype=np.float64).squeeze()
if arr.ndim == 0:
if broadcast_scalar:
return tuple(round(float(arr), 7) for _ in range(size))
if size == 1:
return (round(float(arr), 7),)
flat = arr.reshape(-1)
if flat.size != size:
raise SystemExit(f"{field_name} must have {size} values, got shape {arr.shape}")
return tuple(round(float(x), 7) for x in flat)
def write_bbox_mesh_from_mask(path: Path, mask) -> None:
import numpy as np
ys, xs = np.where(mask)
if ys.size == 0:
extent = np.array([0.1, 0.1, 0.1], dtype=np.float32)
else:
extent = np.array([max((xs.max() - xs.min()) / 512.0, 0.05), 0.08, max((ys.max() - ys.min()) / 512.0, 0.05)], dtype=np.float32)
half = extent / 2.0
vertices = [
(-half[0], -half[1], -half[2]),
(half[0], -half[1], -half[2]),
(half[0], half[1], -half[2]),
(-half[0], half[1], -half[2]),
(-half[0], -half[1], half[2]),
(half[0], -half[1], half[2]),
(half[0], half[1], half[2]),
(-half[0], half[1], half[2]),
]
faces = [(1, 2, 3), (1, 3, 4), (5, 8, 7), (5, 7, 6), (1, 5, 6), (1, 6, 2), (2, 6, 7), (2, 7, 3), (3, 7, 8), (3, 8, 4), (4, 8, 5), (4, 5, 1)]
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="ascii") as f:
f.write("o sam3d_collision_fallback\n")
for v in vertices:
f.write(f"v {v[0]:.7g} {v[1]:.7g} {v[2]:.7g}\n")
for face in faces:
f.write(f"f {face[0]} {face[1]} {face[2]}\n")
if __name__ == "__main__":
raise SystemExit(main())