1199 lines
47 KiB
Python
1199 lines
47 KiB
Python
#!/usr/bin/env python3
|
|
"""Bake a photo-derived texture for an OBJ using COLMAP registered cameras."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import math
|
|
import re
|
|
from collections import OrderedDict
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
from PIL import Image, ImageDraw, ImageFilter
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CameraModel:
|
|
camera_id: int
|
|
model: str
|
|
width: int
|
|
height: int
|
|
params: tuple[float, ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RegisteredImage:
|
|
image_id: int
|
|
qvec: tuple[float, float, float, float]
|
|
tvec: tuple[float, float, float]
|
|
camera_id: int
|
|
image_name: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ObjFace:
|
|
vertices: tuple[int, int, int]
|
|
texcoords: tuple[int, int, int]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ObjMesh:
|
|
vertices: np.ndarray
|
|
colors: np.ndarray
|
|
texcoords: np.ndarray
|
|
faces: tuple[ObjFace, ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProjectionCamera:
|
|
image_name: str
|
|
image_path: Path
|
|
width: int
|
|
height: int
|
|
rotation: np.ndarray
|
|
translation: np.ndarray
|
|
center: np.ndarray
|
|
model: CameraModel
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CameraVisibilityMap:
|
|
width: int
|
|
height: int
|
|
depth: np.ndarray
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Bake an OBJ texture from COLMAP registered views")
|
|
parser.add_argument("--input-obj", type=Path, required=True)
|
|
parser.add_argument("--output-obj", type=Path, required=True)
|
|
parser.add_argument("--output-texture", type=Path, required=True)
|
|
parser.add_argument("--report", type=Path, required=True)
|
|
parser.add_argument("--sfm-report", type=Path, help="SfM report with best_model_text_dir and frames_dir")
|
|
parser.add_argument("--colmap-text", type=Path, help="COLMAP text model directory containing cameras.txt/images.txt")
|
|
parser.add_argument("--images-dir", type=Path, help="Directory containing registered image files")
|
|
parser.add_argument("--atlas-size", type=int, default=4096)
|
|
parser.add_argument("--max-cameras", type=int, default=72)
|
|
parser.add_argument("--max-faces", type=int, default=0, help="Evenly sample faces for baking; 0 means all faces")
|
|
parser.add_argument("--max-output-faces", type=int, default=0, help="Evenly sample output mesh faces before UV unwrapping; 0 means all faces")
|
|
parser.add_argument("--chunk-size", type=int, default=20000)
|
|
parser.add_argument("--texture-fill-mode", choices=["face", "warp"], default="face", help="Use flat face colors or affine-warp source image triangles into the UV atlas")
|
|
parser.add_argument("--color-blend-cameras", type=int, default=1, help="Blend flat fallback colors from this many top-scoring projected cameras")
|
|
parser.add_argument("--visibility-depth-tolerance", type=float, default=0.0, help="Reject projected faces behind the per-camera mesh z-buffer; <=0 disables")
|
|
parser.add_argument("--visibility-grid-size", type=int, default=320, help="Longest side of the approximate per-camera visibility z-buffer")
|
|
parser.add_argument(
|
|
"--max-warped-faces",
|
|
type=int,
|
|
default=0,
|
|
help="When using --texture-fill-mode warp, only affine-warp the largest UV triangles; 0 means try every projected face",
|
|
)
|
|
parser.add_argument("--unwrap-uv", action="store_true", help="Use xatlas to create non-overlapping texture UVs before baking")
|
|
parser.add_argument("--require-unwrap-uv", action="store_true", help="Fail if --unwrap-uv is requested but xatlas is unavailable")
|
|
parser.add_argument("--postprocess-median-radius", type=int, default=0, help="Median-filter baked texels inside the UV mask")
|
|
parser.add_argument("--bleed-iterations", type=int, default=0, help="Expand baked texels outside UV islands to reduce texture seams")
|
|
parser.add_argument("--background", default="120,120,120", help="RGB background for unfilled texture pixels")
|
|
args = parser.parse_args()
|
|
|
|
report = bake_obj_texture(
|
|
input_obj=args.input_obj,
|
|
output_obj=args.output_obj,
|
|
output_texture=args.output_texture,
|
|
report_path=args.report,
|
|
sfm_report=args.sfm_report,
|
|
colmap_text=args.colmap_text,
|
|
images_dir=args.images_dir,
|
|
atlas_size=max(int(args.atlas_size), 64),
|
|
max_cameras=max(int(args.max_cameras), 1),
|
|
max_faces=max(int(args.max_faces), 0),
|
|
max_output_faces=max(int(args.max_output_faces), 0),
|
|
chunk_size=max(int(args.chunk_size), 512),
|
|
texture_fill_mode=args.texture_fill_mode,
|
|
color_blend_cameras=max(int(args.color_blend_cameras), 1),
|
|
visibility_depth_tolerance=max(float(args.visibility_depth_tolerance), 0.0),
|
|
visibility_grid_size=max(int(args.visibility_grid_size), 32),
|
|
max_warped_faces=max(int(args.max_warped_faces), 0),
|
|
unwrap_uv=bool(args.unwrap_uv),
|
|
require_unwrap_uv=bool(args.require_unwrap_uv),
|
|
postprocess_median_radius=max(int(args.postprocess_median_radius), 0),
|
|
bleed_iterations=max(int(args.bleed_iterations), 0),
|
|
background=parse_rgb(args.background),
|
|
)
|
|
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
def bake_obj_texture(
|
|
*,
|
|
input_obj: Path,
|
|
output_obj: Path,
|
|
output_texture: Path,
|
|
report_path: Path,
|
|
sfm_report: Path | None = None,
|
|
colmap_text: Path | None = None,
|
|
images_dir: Path | None = None,
|
|
atlas_size: int = 4096,
|
|
max_cameras: int = 72,
|
|
max_faces: int = 0,
|
|
max_output_faces: int = 0,
|
|
chunk_size: int = 20000,
|
|
texture_fill_mode: str = "face",
|
|
color_blend_cameras: int = 1,
|
|
visibility_depth_tolerance: float = 0.0,
|
|
visibility_grid_size: int = 320,
|
|
max_warped_faces: int = 0,
|
|
unwrap_uv: bool = False,
|
|
require_unwrap_uv: bool = False,
|
|
postprocess_median_radius: int = 0,
|
|
bleed_iterations: int = 0,
|
|
background: tuple[int, int, int] = (120, 120, 120),
|
|
) -> dict[str, Any]:
|
|
resolved_text, resolved_images = resolve_colmap_inputs(
|
|
sfm_report=sfm_report,
|
|
colmap_text=colmap_text,
|
|
images_dir=images_dir,
|
|
)
|
|
mesh = read_obj_mesh(input_obj)
|
|
mesh, mesh_sampling_report = maybe_sample_mesh_faces(mesh, max_output_faces=max_output_faces)
|
|
mesh, unwrap_report = maybe_unwrap_mesh_uvs(
|
|
mesh,
|
|
requested=unwrap_uv,
|
|
require=require_unwrap_uv,
|
|
)
|
|
cameras = load_projection_cameras(resolved_text, resolved_images, max_cameras=max_cameras)
|
|
selected_indices = selected_face_indices(len(mesh.faces), max_faces=max_faces)
|
|
texture_report = bake_texture_image(
|
|
mesh=mesh,
|
|
cameras=cameras,
|
|
face_indices=selected_indices,
|
|
output_texture=output_texture,
|
|
atlas_size=atlas_size,
|
|
chunk_size=chunk_size,
|
|
texture_fill_mode=texture_fill_mode,
|
|
color_blend_cameras=color_blend_cameras,
|
|
visibility_depth_tolerance=visibility_depth_tolerance,
|
|
visibility_grid_size=visibility_grid_size,
|
|
max_warped_faces=max_warped_faces,
|
|
postprocess_median_radius=postprocess_median_radius,
|
|
bleed_iterations=bleed_iterations,
|
|
background=background,
|
|
)
|
|
output_obj.parent.mkdir(parents=True, exist_ok=True)
|
|
output_texture = output_texture.resolve()
|
|
write_textured_obj(
|
|
source_mesh=mesh,
|
|
output_obj=output_obj,
|
|
texture_name=output_texture.name,
|
|
)
|
|
report = {
|
|
"format": "gs-playground-real2sim-colmap-texture-bake-report-v1",
|
|
"input_obj": input_obj.resolve().as_posix(),
|
|
"output_obj": output_obj.resolve().as_posix(),
|
|
"output_texture": output_texture.as_posix(),
|
|
"colmap_text": resolved_text.resolve().as_posix(),
|
|
"images_dir": resolved_images.resolve().as_posix(),
|
|
"vertex_count": int(mesh.vertices.shape[0]),
|
|
"face_count": int(len(mesh.faces)),
|
|
"texcoord_count": int(mesh.texcoords.shape[0]),
|
|
"mesh_sampling": mesh_sampling_report,
|
|
"uv_unwrap": unwrap_report,
|
|
"registered_camera_count": int(len(cameras)),
|
|
"selected_camera_count": int(len(cameras)),
|
|
"selected_face_count": int(len(selected_indices)),
|
|
**texture_report,
|
|
}
|
|
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def resolve_colmap_inputs(
|
|
*,
|
|
sfm_report: Path | None,
|
|
colmap_text: Path | None,
|
|
images_dir: Path | None,
|
|
) -> tuple[Path, Path]:
|
|
if sfm_report is not None:
|
|
data = json.loads(sfm_report.read_text(encoding="utf-8"))
|
|
if colmap_text is None:
|
|
value = data.get("best_model_text_dir")
|
|
if value:
|
|
colmap_text = Path(str(value))
|
|
if images_dir is None:
|
|
value = data.get("frames_dir")
|
|
if value:
|
|
images_dir = Path(str(value))
|
|
if colmap_text is None or not colmap_text.is_dir():
|
|
raise SystemExit(f"COLMAP text model directory not found: {colmap_text}")
|
|
if images_dir is None or not images_dir.is_dir():
|
|
raise SystemExit(f"registered images directory not found: {images_dir}")
|
|
return colmap_text, images_dir
|
|
|
|
|
|
def read_obj_mesh(path: Path) -> ObjMesh:
|
|
vertices: list[tuple[float, float, float]] = []
|
|
colors: list[tuple[float, float, float]] = []
|
|
texcoords: list[tuple[float, float]] = []
|
|
faces: list[ObjFace] = []
|
|
for raw in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
|
if raw.startswith("v "):
|
|
parts = raw.split()
|
|
if len(parts) < 4:
|
|
continue
|
|
vertices.append((float(parts[1]), float(parts[2]), float(parts[3])))
|
|
if len(parts) >= 7:
|
|
colors.append((float(parts[4]), float(parts[5]), float(parts[6])))
|
|
else:
|
|
colors.append((0.5, 0.5, 0.5))
|
|
elif raw.startswith("vt "):
|
|
parts = raw.split()
|
|
if len(parts) >= 3:
|
|
texcoords.append((float(parts[1]), float(parts[2])))
|
|
elif raw.startswith("f "):
|
|
refs = [parse_face_ref(token) for token in raw.split()[1:]]
|
|
if len(refs) < 3:
|
|
continue
|
|
for idx in range(1, len(refs) - 1):
|
|
tri = (refs[0], refs[idx], refs[idx + 1])
|
|
vi = tuple(item[0] for item in tri)
|
|
vti = tuple(item[1] if item[1] is not None else item[0] for item in tri)
|
|
faces.append(ObjFace(vertices=vi, texcoords=vti))
|
|
if not vertices or not faces:
|
|
raise SystemExit(f"OBJ has no usable vertices/faces: {path}")
|
|
vertex_array = np.asarray(vertices, dtype=np.float32)
|
|
color_array = np.asarray(colors, dtype=np.float32)
|
|
if texcoords:
|
|
texcoord_array = np.asarray(texcoords, dtype=np.float32)
|
|
else:
|
|
texcoord_array = planar_texcoords(vertex_array)
|
|
faces = [
|
|
ObjFace(vertices=face.vertices, texcoords=face.vertices)
|
|
for face in faces
|
|
]
|
|
return ObjMesh(
|
|
vertices=vertex_array,
|
|
colors=np.clip(color_array, 0.0, 1.0),
|
|
texcoords=np.clip(texcoord_array, 0.0, 1.0),
|
|
faces=tuple(faces),
|
|
)
|
|
|
|
|
|
def maybe_unwrap_mesh_uvs(
|
|
mesh: ObjMesh,
|
|
*,
|
|
requested: bool,
|
|
require: bool,
|
|
) -> tuple[ObjMesh, dict[str, Any]]:
|
|
report: dict[str, Any] = {
|
|
"requested": bool(requested),
|
|
"required": bool(require),
|
|
"applied": False,
|
|
"method": "input_obj_uv",
|
|
"input_vertex_count": int(mesh.vertices.shape[0]),
|
|
"input_texcoord_count": int(mesh.texcoords.shape[0]),
|
|
"input_face_count": int(len(mesh.faces)),
|
|
}
|
|
if not requested:
|
|
return mesh, report
|
|
try:
|
|
import xatlas # type: ignore[import-not-found]
|
|
except Exception as exc:
|
|
report["error"] = f"xatlas unavailable: {exc}"
|
|
if require:
|
|
raise SystemExit(report["error"])
|
|
return mesh, report
|
|
|
|
faces = np.asarray([face.vertices for face in mesh.faces], dtype=np.uint32)
|
|
try:
|
|
vmapping, indices, texcoords = xatlas.parametrize(
|
|
np.asarray(mesh.vertices, dtype=np.float32),
|
|
faces,
|
|
)
|
|
except Exception as exc:
|
|
report["error"] = f"xatlas unwrap failed: {exc}"
|
|
if require:
|
|
raise SystemExit(report["error"])
|
|
return mesh, report
|
|
|
|
vmapping = np.asarray(vmapping, dtype=np.int64)
|
|
indices = np.asarray(indices, dtype=np.int64)
|
|
texcoords = np.asarray(texcoords, dtype=np.float32)
|
|
if indices.ndim != 2 or indices.shape[1] != 3 or texcoords.ndim != 2 or texcoords.shape[1] != 2:
|
|
report["error"] = "xatlas returned an unexpected mesh layout"
|
|
if require:
|
|
raise SystemExit(report["error"])
|
|
return mesh, report
|
|
|
|
unwrapped = ObjMesh(
|
|
vertices=mesh.vertices[np.clip(vmapping, 0, mesh.vertices.shape[0] - 1)],
|
|
colors=mesh.colors[np.clip(vmapping, 0, mesh.colors.shape[0] - 1)],
|
|
texcoords=np.clip(texcoords, 0.0, 1.0),
|
|
faces=tuple(
|
|
ObjFace(
|
|
vertices=(int(face[0]), int(face[1]), int(face[2])),
|
|
texcoords=(int(face[0]), int(face[1]), int(face[2])),
|
|
)
|
|
for face in indices
|
|
),
|
|
)
|
|
report.update(
|
|
{
|
|
"applied": True,
|
|
"method": "xatlas",
|
|
"output_vertex_count": int(unwrapped.vertices.shape[0]),
|
|
"output_texcoord_count": int(unwrapped.texcoords.shape[0]),
|
|
"output_face_count": int(len(unwrapped.faces)),
|
|
"duplicated_vertex_count": int(max(0, unwrapped.vertices.shape[0] - mesh.vertices.shape[0])),
|
|
}
|
|
)
|
|
return unwrapped, report
|
|
|
|
|
|
def maybe_sample_mesh_faces(mesh: ObjMesh, *, max_output_faces: int) -> tuple[ObjMesh, dict[str, Any]]:
|
|
report: dict[str, Any] = {
|
|
"requested_max_output_faces": int(max_output_faces),
|
|
"applied": False,
|
|
"method": "all_faces",
|
|
"input_face_count": int(len(mesh.faces)),
|
|
"output_face_count": int(len(mesh.faces)),
|
|
}
|
|
if max_output_faces <= 0 or max_output_faces >= len(mesh.faces):
|
|
return mesh, report
|
|
indices = selected_face_indices(len(mesh.faces), max_faces=max_output_faces)
|
|
sampled_faces = tuple(mesh.faces[int(index)] for index in indices)
|
|
sampled = ObjMesh(
|
|
vertices=mesh.vertices,
|
|
colors=mesh.colors,
|
|
texcoords=mesh.texcoords,
|
|
faces=sampled_faces,
|
|
)
|
|
report.update(
|
|
{
|
|
"applied": True,
|
|
"method": "uniform_face_sampling",
|
|
"output_face_count": int(len(sampled.faces)),
|
|
}
|
|
)
|
|
return sampled, report
|
|
|
|
|
|
def parse_face_ref(token: str) -> tuple[int, int | None]:
|
|
parts = token.split("/")
|
|
vertex = parse_obj_index(parts[0])
|
|
texcoord = parse_obj_index(parts[1]) if len(parts) >= 2 and parts[1] else None
|
|
return vertex, texcoord
|
|
|
|
|
|
def parse_obj_index(value: str) -> int:
|
|
index = int(value)
|
|
if index <= 0:
|
|
raise SystemExit("negative OBJ indices are not supported for texture baking")
|
|
return index - 1
|
|
|
|
|
|
def planar_texcoords(vertices: np.ndarray) -> np.ndarray:
|
|
extents = vertices.max(axis=0) - vertices.min(axis=0)
|
|
axes = np.argsort(extents)[-2:]
|
|
selected = vertices[:, axes]
|
|
low = selected.min(axis=0)
|
|
span = np.maximum(selected.max(axis=0) - low, 1e-9)
|
|
return (selected - low) / span
|
|
|
|
|
|
def read_colmap_cameras(path: Path) -> dict[int, CameraModel]:
|
|
cameras: dict[int, CameraModel] = {}
|
|
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
parts = line.split()
|
|
camera_id = int(parts[0])
|
|
cameras[camera_id] = CameraModel(
|
|
camera_id=camera_id,
|
|
model=parts[1].upper(),
|
|
width=int(parts[2]),
|
|
height=int(parts[3]),
|
|
params=tuple(float(value) for value in parts[4:]),
|
|
)
|
|
return cameras
|
|
|
|
|
|
def read_colmap_images(path: Path) -> list[RegisteredImage]:
|
|
images: list[RegisteredImage] = []
|
|
lines = [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines()]
|
|
idx = 0
|
|
while idx < len(lines):
|
|
line = lines[idx]
|
|
idx += 1
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
parts = line.split()
|
|
if len(parts) < 10:
|
|
continue
|
|
images.append(
|
|
RegisteredImage(
|
|
image_id=int(parts[0]),
|
|
qvec=tuple(float(value) for value in parts[1:5]), # type: ignore[arg-type]
|
|
tvec=tuple(float(value) for value in parts[5:8]), # type: ignore[arg-type]
|
|
camera_id=int(parts[8]),
|
|
image_name=" ".join(parts[9:]),
|
|
)
|
|
)
|
|
idx += 1
|
|
return images
|
|
|
|
|
|
def load_projection_cameras(colmap_text: Path, images_dir: Path, *, max_cameras: int) -> list[ProjectionCamera]:
|
|
camera_models = read_colmap_cameras(colmap_text / "cameras.txt")
|
|
registered = read_colmap_images(colmap_text / "images.txt")
|
|
if max_cameras > 0 and len(registered) > max_cameras:
|
|
indices = np.linspace(0, len(registered) - 1, max_cameras).round().astype(int)
|
|
registered = [registered[int(index)] for index in indices]
|
|
cameras: list[ProjectionCamera] = []
|
|
for image in registered:
|
|
model = camera_models.get(image.camera_id)
|
|
image_path = images_dir / image.image_name
|
|
if model is None or not image_path.is_file():
|
|
continue
|
|
rotation = np.asarray(qvec_to_rotmat(image.qvec), dtype=np.float32)
|
|
translation = np.asarray(image.tvec, dtype=np.float32)
|
|
center = -rotation.T @ translation
|
|
cameras.append(
|
|
ProjectionCamera(
|
|
image_name=image.image_name,
|
|
image_path=image_path,
|
|
width=model.width,
|
|
height=model.height,
|
|
rotation=rotation,
|
|
translation=translation,
|
|
center=center,
|
|
model=model,
|
|
)
|
|
)
|
|
if not cameras:
|
|
raise SystemExit("no registered COLMAP image could be loaded for texture baking")
|
|
return cameras
|
|
|
|
|
|
def selected_face_indices(face_count: int, *, max_faces: int) -> np.ndarray:
|
|
if max_faces <= 0 or max_faces >= face_count:
|
|
return np.arange(face_count, dtype=np.int64)
|
|
return np.linspace(0, face_count - 1, max_faces).round().astype(np.int64)
|
|
|
|
|
|
def build_camera_visibility_maps(
|
|
*,
|
|
mesh: ObjMesh,
|
|
cameras: list[ProjectionCamera],
|
|
face_indices: np.ndarray,
|
|
grid_size: int,
|
|
) -> list[CameraVisibilityMap]:
|
|
if len(face_indices) == 0:
|
|
return []
|
|
faces = [mesh.faces[int(index)] for index in face_indices]
|
|
vi = np.asarray([face.vertices for face in faces], dtype=np.int64)
|
|
centroids = mesh.vertices[vi].mean(axis=1)
|
|
maps: list[CameraVisibilityMap] = []
|
|
for camera in cameras:
|
|
scale = max(float(camera.width), float(camera.height), 1.0)
|
|
width = max(int(round(camera.width / scale * grid_size)), 8)
|
|
height = max(int(round(camera.height / scale * grid_size)), 8)
|
|
depth = np.full((height, width), np.inf, dtype=np.float32)
|
|
cam_points = centroids @ camera.rotation.T + camera.translation
|
|
projected = project_camera(camera.model, cam_points)
|
|
if projected is not None:
|
|
u, v = projected
|
|
z = cam_points[:, 2]
|
|
finite = np.isfinite(z) & np.isfinite(u) & np.isfinite(v)
|
|
valid = finite & (z > 1e-6) & (u >= 0) & (u < camera.width) & (v >= 0) & (v < camera.height)
|
|
if np.any(valid):
|
|
gx = np.clip((u[valid] / max(camera.width, 1) * width).astype(np.int64), 0, width - 1)
|
|
gy = np.clip((v[valid] / max(camera.height, 1) * height).astype(np.int64), 0, height - 1)
|
|
flat = gy * width + gx
|
|
np.minimum.at(depth.reshape(-1), flat, z[valid].astype(np.float32))
|
|
maps.append(CameraVisibilityMap(width=width, height=height, depth=depth))
|
|
return maps
|
|
|
|
|
|
def camera_visibility_mask(
|
|
*,
|
|
visibility: CameraVisibilityMap | None,
|
|
camera: ProjectionCamera,
|
|
u: np.ndarray,
|
|
v: np.ndarray,
|
|
z: np.ndarray,
|
|
tolerance: float,
|
|
) -> np.ndarray:
|
|
if visibility is None or tolerance <= 0.0:
|
|
return np.ones(z.shape, dtype=bool)
|
|
gx = np.clip((u / max(camera.width, 1) * visibility.width).astype(np.int64), 0, visibility.width - 1)
|
|
gy = np.clip((v / max(camera.height, 1) * visibility.height).astype(np.int64), 0, visibility.height - 1)
|
|
nearest = visibility.depth[gy, gx].astype(np.float64)
|
|
slack = np.maximum(float(tolerance), np.abs(nearest) * float(tolerance))
|
|
return np.isfinite(nearest) & (z.astype(np.float64) <= nearest + slack)
|
|
|
|
|
|
def warp_allowed_face_indices(
|
|
*,
|
|
mesh: ObjMesh,
|
|
face_indices: np.ndarray,
|
|
atlas_size: int,
|
|
max_warped_faces: int,
|
|
) -> set[int] | None:
|
|
if max_warped_faces <= 0 or max_warped_faces >= len(face_indices):
|
|
return None
|
|
areas: list[tuple[float, int]] = []
|
|
for index in face_indices:
|
|
face = mesh.faces[int(index)]
|
|
texcoords = mesh.texcoords[np.clip(face.texcoords, 0, mesh.texcoords.shape[0] - 1)]
|
|
area = polygon_area(uv_polygon_pixels(texcoords, atlas_size))
|
|
if area >= 0.25:
|
|
areas.append((float(area), int(index)))
|
|
if len(areas) <= max_warped_faces:
|
|
return {index for _area, index in areas}
|
|
areas.sort(reverse=True)
|
|
return {index for _area, index in areas[:max_warped_faces]}
|
|
|
|
|
|
def bake_texture_image(
|
|
*,
|
|
mesh: ObjMesh,
|
|
cameras: list[ProjectionCamera],
|
|
face_indices: np.ndarray,
|
|
output_texture: Path,
|
|
atlas_size: int,
|
|
chunk_size: int,
|
|
texture_fill_mode: str,
|
|
color_blend_cameras: int,
|
|
visibility_depth_tolerance: float,
|
|
visibility_grid_size: int,
|
|
max_warped_faces: int,
|
|
postprocess_median_radius: int,
|
|
bleed_iterations: int,
|
|
background: tuple[int, int, int],
|
|
) -> dict[str, Any]:
|
|
texture = Image.new("RGB", (atlas_size, atlas_size), background)
|
|
mask = Image.new("L", (atlas_size, atlas_size), 0)
|
|
draw = ImageDraw.Draw(texture)
|
|
mask_draw = ImageDraw.Draw(mask)
|
|
image_cache: OrderedDict[str, np.ndarray] = OrderedDict()
|
|
pil_image_cache: OrderedDict[str, Image.Image] = OrderedDict()
|
|
projected_faces = 0
|
|
fallback_faces = 0
|
|
drawn_faces = 0
|
|
warped_faces = 0
|
|
warp_fallback_faces = 0
|
|
warp_skipped_faces = 0
|
|
warp_allowed_faces = warp_allowed_face_indices(
|
|
mesh=mesh,
|
|
face_indices=face_indices,
|
|
atlas_size=atlas_size,
|
|
max_warped_faces=max_warped_faces,
|
|
)
|
|
visibility_maps = build_camera_visibility_maps(
|
|
mesh=mesh,
|
|
cameras=cameras,
|
|
face_indices=face_indices,
|
|
grid_size=visibility_grid_size,
|
|
) if visibility_depth_tolerance > 0.0 else None
|
|
for start in range(0, len(face_indices), chunk_size):
|
|
chunk_indices = face_indices[start : start + chunk_size]
|
|
faces = [mesh.faces[int(index)] for index in chunk_indices]
|
|
vi = np.asarray([face.vertices for face in faces], dtype=np.int64)
|
|
vti = np.asarray([face.texcoords for face in faces], dtype=np.int64)
|
|
points = mesh.vertices[vi]
|
|
colors = mesh.colors[vi].mean(axis=1)
|
|
centroids = points.mean(axis=1)
|
|
normals = face_normals(points)
|
|
best_camera, best_uv, best_score = select_best_camera(
|
|
cameras,
|
|
centroids,
|
|
normals,
|
|
visibility_maps=visibility_maps,
|
|
visibility_depth_tolerance=visibility_depth_tolerance,
|
|
)
|
|
sampled_colors = np.clip((colors * 255.0).round(), 0, 255).astype(np.uint8)
|
|
valid = best_camera >= 0
|
|
if np.any(valid):
|
|
projected_faces += int(np.count_nonzero(valid))
|
|
if color_blend_cameras > 1:
|
|
sampled_colors[valid] = blend_projected_face_colors(
|
|
cameras=cameras,
|
|
centroids=centroids[valid],
|
|
normals=normals[valid],
|
|
image_cache=image_cache,
|
|
max_cameras=color_blend_cameras,
|
|
visibility_maps=visibility_maps,
|
|
visibility_depth_tolerance=visibility_depth_tolerance,
|
|
)
|
|
else:
|
|
for camera_idx in np.unique(best_camera[valid]):
|
|
face_mask = best_camera == camera_idx
|
|
camera = cameras[int(camera_idx)]
|
|
image = cached_image(image_cache, camera.image_path)
|
|
sampled_colors[face_mask] = sample_image(image, best_uv[face_mask])
|
|
fallback_faces += int(np.count_nonzero(~valid))
|
|
uv = mesh.texcoords[np.clip(vti, 0, mesh.texcoords.shape[0] - 1)]
|
|
for local_idx, tri_uv in enumerate(uv):
|
|
polygon = uv_polygon_pixels(tri_uv, atlas_size)
|
|
if polygon_area(polygon) < 0.25:
|
|
continue
|
|
filled_by_warp = False
|
|
can_warp = (
|
|
texture_fill_mode == "warp"
|
|
and best_camera[local_idx] >= 0
|
|
and (warp_allowed_faces is None or int(chunk_indices[local_idx]) in warp_allowed_faces)
|
|
)
|
|
if can_warp:
|
|
camera = cameras[int(best_camera[local_idx])]
|
|
src_tri = project_triangle_pixels(camera, points[local_idx])
|
|
if src_tri is not None:
|
|
image = cached_pil_image(pil_image_cache, camera.image_path)
|
|
filled_by_warp = paste_warped_triangle(
|
|
texture=texture,
|
|
image=image,
|
|
dest_polygon=polygon,
|
|
source_triangle=src_tri,
|
|
)
|
|
if filled_by_warp:
|
|
warped_faces += 1
|
|
else:
|
|
if texture_fill_mode == "warp" and best_camera[local_idx] >= 0:
|
|
if warp_allowed_faces is not None and int(chunk_indices[local_idx]) not in warp_allowed_faces:
|
|
warp_skipped_faces += 1
|
|
else:
|
|
warp_fallback_faces += 1
|
|
color = tuple(int(value) for value in sampled_colors[local_idx])
|
|
draw.polygon(polygon, fill=color)
|
|
mask_draw.polygon(polygon, fill=255)
|
|
drawn_faces += 1
|
|
mask_array = np.asarray(mask, dtype=np.uint8)
|
|
coverage = float(np.count_nonzero(mask_array)) / float(mask_array.size)
|
|
selected_count = max(int(len(face_indices)), 1)
|
|
projected_ratio = float(projected_faces) / float(selected_count)
|
|
drawn_ratio = float(drawn_faces) / float(selected_count)
|
|
postprocess_report = postprocess_texture(
|
|
texture=texture,
|
|
mask=mask,
|
|
median_radius=postprocess_median_radius,
|
|
bleed_iterations=bleed_iterations,
|
|
)
|
|
output_texture.parent.mkdir(parents=True, exist_ok=True)
|
|
texture.save(output_texture)
|
|
return {
|
|
"atlas_size": int(atlas_size),
|
|
"projected_face_count": int(projected_faces),
|
|
"fallback_face_count": int(fallback_faces),
|
|
"drawn_face_count": int(drawn_faces),
|
|
"texture_fill_mode": texture_fill_mode,
|
|
"color_blend_cameras": int(color_blend_cameras),
|
|
"visibility_depth_tolerance": float(visibility_depth_tolerance),
|
|
"visibility_grid_size": int(visibility_grid_size),
|
|
"visibility_camera_count": int(len(visibility_maps)) if visibility_maps is not None else 0,
|
|
"max_warped_faces": int(max_warped_faces),
|
|
"warp_candidate_face_count": int(len(warp_allowed_faces)) if warp_allowed_faces is not None else int(len(face_indices)),
|
|
"warped_face_count": int(warped_faces),
|
|
"warp_fallback_face_count": int(warp_fallback_faces),
|
|
"warp_skipped_face_count": int(warp_skipped_faces),
|
|
"texture_coverage_ratio": round(coverage, 6),
|
|
"atlas_pixel_coverage_ratio": round(coverage, 6),
|
|
"projected_face_ratio": round(projected_ratio, 6),
|
|
"textured_face_ratio": round(drawn_ratio, 6),
|
|
"texture_quality_breakdown": {
|
|
"mesh_face_coverage": round(drawn_ratio, 6),
|
|
"camera_projected_face_coverage": round(projected_ratio, 6),
|
|
"atlas_pixel_coverage": round(coverage, 6),
|
|
"interpretation": "atlas_pixel_coverage measures UV atlas occupancy; mesh_face_coverage measures how much reconstructed mesh received a baked texture.",
|
|
},
|
|
"texture_postprocess": postprocess_report,
|
|
}
|
|
|
|
|
|
def postprocess_texture(
|
|
*,
|
|
texture: Image.Image,
|
|
mask: Image.Image,
|
|
median_radius: int,
|
|
bleed_iterations: int,
|
|
) -> dict[str, Any]:
|
|
report = {
|
|
"median_radius": int(max(median_radius, 0)),
|
|
"median_applied": False,
|
|
"bleed_iterations_requested": int(max(bleed_iterations, 0)),
|
|
"bleed_iterations_applied": 0,
|
|
"bleed_filled_pixels": 0,
|
|
}
|
|
if median_radius > 0:
|
|
size = max(3, int(median_radius) * 2 + 1)
|
|
filtered = texture.filter(ImageFilter.MedianFilter(size=size))
|
|
texture.paste(filtered, mask=mask)
|
|
report["median_applied"] = True
|
|
report["median_filter_size"] = int(size)
|
|
if bleed_iterations > 0:
|
|
filled_texture, filled_mask, filled_pixels, applied = bleed_texture_edges(
|
|
texture,
|
|
mask,
|
|
iterations=int(bleed_iterations),
|
|
)
|
|
texture.paste(filled_texture)
|
|
mask.paste(filled_mask)
|
|
report["bleed_iterations_applied"] = int(applied)
|
|
report["bleed_filled_pixels"] = int(filled_pixels)
|
|
return report
|
|
|
|
|
|
def bleed_texture_edges(
|
|
texture: Image.Image,
|
|
mask: Image.Image,
|
|
*,
|
|
iterations: int,
|
|
) -> tuple[Image.Image, Image.Image, int, int]:
|
|
colors = np.asarray(texture.convert("RGB"), dtype=np.float32).copy()
|
|
valid = np.asarray(mask, dtype=np.uint8) > 0
|
|
total_filled = 0
|
|
applied = 0
|
|
shifts = (
|
|
(-1, -1),
|
|
(-1, 0),
|
|
(-1, 1),
|
|
(0, -1),
|
|
(0, 1),
|
|
(1, -1),
|
|
(1, 0),
|
|
(1, 1),
|
|
)
|
|
for _ in range(max(int(iterations), 0)):
|
|
accum = np.zeros_like(colors, dtype=np.float32)
|
|
counts = np.zeros(valid.shape, dtype=np.float32)
|
|
for dy, dx in shifts:
|
|
shifted_valid = shifted_bool(valid, dy=dy, dx=dx)
|
|
shifted_colors = shifted_rgb(colors, dy=dy, dx=dx)
|
|
accum += shifted_colors * shifted_valid[:, :, None]
|
|
counts += shifted_valid.astype(np.float32)
|
|
fill = (~valid) & (counts > 0)
|
|
filled = int(np.count_nonzero(fill))
|
|
if filled <= 0:
|
|
break
|
|
colors[fill] = accum[fill] / counts[fill, None]
|
|
valid[fill] = True
|
|
total_filled += filled
|
|
applied += 1
|
|
out_texture = Image.fromarray(np.clip(colors, 0, 255).astype(np.uint8), "RGB")
|
|
out_mask = Image.fromarray((valid.astype(np.uint8) * 255), "L")
|
|
return out_texture, out_mask, total_filled, applied
|
|
|
|
|
|
def shifted_bool(values: np.ndarray, *, dy: int, dx: int) -> np.ndarray:
|
|
result = np.zeros_like(values, dtype=bool)
|
|
src_y0 = max(0, -dy)
|
|
src_y1 = values.shape[0] - max(0, dy)
|
|
src_x0 = max(0, -dx)
|
|
src_x1 = values.shape[1] - max(0, dx)
|
|
dst_y0 = max(0, dy)
|
|
dst_y1 = values.shape[0] - max(0, -dy)
|
|
dst_x0 = max(0, dx)
|
|
dst_x1 = values.shape[1] - max(0, -dx)
|
|
result[dst_y0:dst_y1, dst_x0:dst_x1] = values[src_y0:src_y1, src_x0:src_x1]
|
|
return result
|
|
|
|
|
|
def shifted_rgb(values: np.ndarray, *, dy: int, dx: int) -> np.ndarray:
|
|
result = np.zeros_like(values, dtype=np.float32)
|
|
src_y0 = max(0, -dy)
|
|
src_y1 = values.shape[0] - max(0, dy)
|
|
src_x0 = max(0, -dx)
|
|
src_x1 = values.shape[1] - max(0, dx)
|
|
dst_y0 = max(0, dy)
|
|
dst_y1 = values.shape[0] - max(0, -dy)
|
|
dst_x0 = max(0, dx)
|
|
dst_x1 = values.shape[1] - max(0, -dx)
|
|
result[dst_y0:dst_y1, dst_x0:dst_x1] = values[src_y0:src_y1, src_x0:src_x1]
|
|
return result
|
|
|
|
|
|
def face_normals(points: np.ndarray) -> np.ndarray:
|
|
normals = np.cross(points[:, 1] - points[:, 0], points[:, 2] - points[:, 0])
|
|
norms = np.linalg.norm(normals, axis=1, keepdims=True)
|
|
return normals / np.maximum(norms, 1e-9)
|
|
|
|
|
|
def select_best_camera(
|
|
cameras: list[ProjectionCamera],
|
|
centroids: np.ndarray,
|
|
normals: np.ndarray,
|
|
*,
|
|
visibility_maps: list[CameraVisibilityMap] | None = None,
|
|
visibility_depth_tolerance: float = 0.0,
|
|
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
count = centroids.shape[0]
|
|
best_camera = np.full(count, -1, dtype=np.int32)
|
|
best_uv = np.zeros((count, 2), dtype=np.float32)
|
|
best_score = np.full(count, -np.inf, dtype=np.float32)
|
|
for camera_idx, camera in enumerate(cameras):
|
|
cam_points = centroids @ camera.rotation.T + camera.translation
|
|
projected = project_camera(camera.model, cam_points)
|
|
if projected is None:
|
|
continue
|
|
u, v = projected
|
|
z = cam_points[:, 2]
|
|
finite = np.isfinite(z) & np.isfinite(u) & np.isfinite(v)
|
|
valid = finite & (z > 1e-6) & (u >= 1) & (u < camera.width - 1) & (v >= 1) & (v < camera.height - 1)
|
|
if visibility_maps is not None and camera_idx < len(visibility_maps):
|
|
visible = camera_visibility_mask(
|
|
visibility=visibility_maps[camera_idx],
|
|
camera=camera,
|
|
u=u,
|
|
v=v,
|
|
z=z,
|
|
tolerance=visibility_depth_tolerance,
|
|
)
|
|
valid &= visible
|
|
if not np.any(valid):
|
|
continue
|
|
valid_idx = np.nonzero(valid)[0]
|
|
rays = camera.center.astype(np.float64)[None, :] - centroids[valid_idx].astype(np.float64)
|
|
ray_norms = np.linalg.norm(rays, axis=1)
|
|
normal_score = np.abs(
|
|
np.sum(normals[valid_idx].astype(np.float64) * rays, axis=1) / np.maximum(ray_norms, 1e-9)
|
|
)
|
|
center_score = np.full(count, -np.inf, dtype=np.float32)
|
|
du = (u[valid_idx] - (camera.width * 0.5)) / max(camera.width * 0.5, 1.0)
|
|
dv = (v[valid_idx] - (camera.height * 0.5)) / max(camera.height * 0.5, 1.0)
|
|
center_score[valid_idx] = (1.0 - np.sqrt(du * du + dv * dv)).astype(np.float32)
|
|
score = np.full(count, -np.inf, dtype=np.float32)
|
|
score[valid_idx] = (0.70 * normal_score + 0.30 * center_score[valid_idx]).astype(np.float32)
|
|
update = score > best_score
|
|
if not np.any(update):
|
|
continue
|
|
best_score[update] = score[update]
|
|
best_camera[update] = camera_idx
|
|
best_uv[update, 0] = u[update]
|
|
best_uv[update, 1] = v[update]
|
|
return best_camera, best_uv, best_score
|
|
|
|
|
|
def blend_projected_face_colors(
|
|
*,
|
|
cameras: list[ProjectionCamera],
|
|
centroids: np.ndarray,
|
|
normals: np.ndarray,
|
|
image_cache: OrderedDict[str, np.ndarray],
|
|
max_cameras: int,
|
|
visibility_maps: list[CameraVisibilityMap] | None = None,
|
|
visibility_depth_tolerance: float = 0.0,
|
|
) -> np.ndarray:
|
|
count = centroids.shape[0]
|
|
top_k = max(int(max_cameras), 1)
|
|
top_camera = np.full((count, top_k), -1, dtype=np.int32)
|
|
top_uv = np.zeros((count, top_k, 2), dtype=np.float32)
|
|
top_score = np.full((count, top_k), -np.inf, dtype=np.float32)
|
|
for camera_idx, camera in enumerate(cameras):
|
|
cam_points = centroids @ camera.rotation.T + camera.translation
|
|
projected = project_camera(camera.model, cam_points)
|
|
if projected is None:
|
|
continue
|
|
u, v = projected
|
|
z = cam_points[:, 2]
|
|
finite = np.isfinite(z) & np.isfinite(u) & np.isfinite(v)
|
|
valid = finite & (z > 1e-6) & (u >= 1) & (u < camera.width - 1) & (v >= 1) & (v < camera.height - 1)
|
|
if visibility_maps is not None and camera_idx < len(visibility_maps):
|
|
visible = camera_visibility_mask(
|
|
visibility=visibility_maps[camera_idx],
|
|
camera=camera,
|
|
u=u,
|
|
v=v,
|
|
z=z,
|
|
tolerance=visibility_depth_tolerance,
|
|
)
|
|
valid &= visible
|
|
if not np.any(valid):
|
|
continue
|
|
valid_idx = np.nonzero(valid)[0]
|
|
rays = camera.center.astype(np.float64)[None, :] - centroids[valid_idx].astype(np.float64)
|
|
ray_norms = np.linalg.norm(rays, axis=1)
|
|
normal_score = np.abs(
|
|
np.sum(normals[valid_idx].astype(np.float64) * rays, axis=1) / np.maximum(ray_norms, 1e-9)
|
|
)
|
|
du = (u[valid_idx] - (camera.width * 0.5)) / max(camera.width * 0.5, 1.0)
|
|
dv = (v[valid_idx] - (camera.height * 0.5)) / max(camera.height * 0.5, 1.0)
|
|
center_score = (1.0 - np.sqrt(du * du + dv * dv)).astype(np.float32)
|
|
score = (0.70 * normal_score + 0.30 * center_score).astype(np.float32)
|
|
for local, face_index in enumerate(valid_idx):
|
|
weakest = int(np.argmin(top_score[face_index]))
|
|
if score[local] <= top_score[face_index, weakest]:
|
|
continue
|
|
top_score[face_index, weakest] = score[local]
|
|
top_camera[face_index, weakest] = camera_idx
|
|
top_uv[face_index, weakest, 0] = u[face_index]
|
|
top_uv[face_index, weakest, 1] = v[face_index]
|
|
|
|
accum = np.zeros((count, 3), dtype=np.float64)
|
|
weights = np.zeros((count,), dtype=np.float64)
|
|
for slot in range(top_k):
|
|
slot_valid = top_camera[:, slot] >= 0
|
|
if not np.any(slot_valid):
|
|
continue
|
|
for camera_idx in np.unique(top_camera[slot_valid, slot]):
|
|
face_mask = slot_valid & (top_camera[:, slot] == camera_idx)
|
|
camera = cameras[int(camera_idx)]
|
|
image = cached_image(image_cache, camera.image_path)
|
|
colors = sample_image(image, top_uv[face_mask, slot, :]).astype(np.float64)
|
|
score = np.maximum(top_score[face_mask, slot].astype(np.float64), 1e-3)
|
|
accum[face_mask] += colors * score[:, None]
|
|
weights[face_mask] += score
|
|
missing = weights <= 0
|
|
if np.any(missing):
|
|
accum[missing] = 127.0
|
|
weights[missing] = 1.0
|
|
return np.clip(np.rint(accum / weights[:, None]), 0, 255).astype(np.uint8)
|
|
|
|
|
|
def project_camera(model: CameraModel, cam_points: np.ndarray) -> tuple[np.ndarray, np.ndarray] | None:
|
|
z = cam_points[:, 2].astype(np.float64)
|
|
x = np.full(z.shape, np.nan, dtype=np.float64)
|
|
y = np.full(z.shape, np.nan, dtype=np.float64)
|
|
valid_z = np.isfinite(z) & (z > 1e-9)
|
|
x[valid_z] = cam_points[valid_z, 0].astype(np.float64) / z[valid_z]
|
|
y[valid_z] = cam_points[valid_z, 1].astype(np.float64) / z[valid_z]
|
|
params = model.params
|
|
with np.errstate(over="ignore", invalid="ignore", divide="ignore"):
|
|
if model.model == "SIMPLE_PINHOLE":
|
|
f, cx, cy = params[:3]
|
|
return f * x + cx, f * y + cy
|
|
if model.model == "PINHOLE":
|
|
fx, fy, cx, cy = params[:4]
|
|
return fx * x + cx, fy * y + cy
|
|
if model.model in {"SIMPLE_RADIAL", "RADIAL"}:
|
|
f, cx, cy = params[:3]
|
|
k1 = params[3] if len(params) > 3 else 0.0
|
|
k2 = params[4] if len(params) > 4 else 0.0
|
|
r2 = x * x + y * y
|
|
radial = 1.0 + k1 * r2 + k2 * r2 * r2
|
|
return f * x * radial + cx, f * y * radial + cy
|
|
if model.model == "OPENCV":
|
|
fx, fy, cx, cy = params[:4]
|
|
k1 = params[4] if len(params) > 4 else 0.0
|
|
k2 = params[5] if len(params) > 5 else 0.0
|
|
p1 = params[6] if len(params) > 6 else 0.0
|
|
p2 = params[7] if len(params) > 7 else 0.0
|
|
r2 = x * x + y * y
|
|
radial = 1.0 + k1 * r2 + k2 * r2 * r2
|
|
x_dist = x * radial + 2.0 * p1 * x * y + p2 * (r2 + 2.0 * x * x)
|
|
y_dist = y * radial + p1 * (r2 + 2.0 * y * y) + 2.0 * p2 * x * y
|
|
return fx * x_dist + cx, fy * y_dist + cy
|
|
return None
|
|
|
|
|
|
def cached_image(cache: OrderedDict[str, np.ndarray], path: Path, *, max_entries: int = 6) -> np.ndarray:
|
|
key = path.as_posix()
|
|
if key in cache:
|
|
cache.move_to_end(key)
|
|
return cache[key]
|
|
image = np.asarray(Image.open(path).convert("RGB"), dtype=np.uint8)
|
|
cache[key] = image
|
|
cache.move_to_end(key)
|
|
while len(cache) > max_entries:
|
|
cache.popitem(last=False)
|
|
return image
|
|
|
|
|
|
def cached_pil_image(cache: OrderedDict[str, Image.Image], path: Path, *, max_entries: int = 4) -> Image.Image:
|
|
key = path.as_posix()
|
|
if key in cache:
|
|
cache.move_to_end(key)
|
|
return cache[key]
|
|
image = Image.open(path).convert("RGB")
|
|
cache[key] = image
|
|
cache.move_to_end(key)
|
|
while len(cache) > max_entries:
|
|
_old_key, old_image = cache.popitem(last=False)
|
|
old_image.close()
|
|
return image
|
|
|
|
|
|
def sample_image(image: np.ndarray, uv_pixels: np.ndarray) -> np.ndarray:
|
|
h, w = image.shape[:2]
|
|
u = np.clip(np.rint(uv_pixels[:, 0]).astype(np.int64), 0, w - 1)
|
|
v = np.clip(np.rint(uv_pixels[:, 1]).astype(np.int64), 0, h - 1)
|
|
return image[v, u, :]
|
|
|
|
|
|
def project_triangle_pixels(camera: ProjectionCamera, points: np.ndarray) -> list[tuple[float, float]] | None:
|
|
cam_points = points @ camera.rotation.T + camera.translation
|
|
projected = project_camera(camera.model, cam_points)
|
|
if projected is None:
|
|
return None
|
|
u, v = projected
|
|
z = cam_points[:, 2]
|
|
valid = (
|
|
np.isfinite(z)
|
|
& np.isfinite(u)
|
|
& np.isfinite(v)
|
|
& (z > 1e-6)
|
|
& (u >= 0)
|
|
& (u < camera.width)
|
|
& (v >= 0)
|
|
& (v < camera.height)
|
|
)
|
|
if not bool(np.all(valid)):
|
|
return None
|
|
triangle = [(float(u[idx]), float(v[idx])) for idx in range(3)]
|
|
if triangle_area_float(triangle) < 0.5:
|
|
return None
|
|
return triangle
|
|
|
|
|
|
def paste_warped_triangle(
|
|
*,
|
|
texture: Image.Image,
|
|
image: Image.Image,
|
|
dest_polygon: list[tuple[int, int]],
|
|
source_triangle: list[tuple[float, float]],
|
|
) -> bool:
|
|
if polygon_area(dest_polygon) < 0.25 or triangle_area_float(source_triangle) < 0.5:
|
|
return False
|
|
min_x = max(min(x for x, _y in dest_polygon), 0)
|
|
max_x = min(max(x for x, _y in dest_polygon), texture.width - 1)
|
|
min_y = max(min(y for _x, y in dest_polygon), 0)
|
|
max_y = min(max(y for _x, y in dest_polygon), texture.height - 1)
|
|
if max_x <= min_x or max_y <= min_y:
|
|
return False
|
|
local_dest = [(x - min_x, y - min_y) for x, y in dest_polygon]
|
|
coeffs = affine_dest_to_source(local_dest, source_triangle)
|
|
if coeffs is None:
|
|
return False
|
|
width = max_x - min_x + 1
|
|
height = max_y - min_y + 1
|
|
patch = image.transform(
|
|
(width, height),
|
|
image_affine_transform_kind(),
|
|
coeffs,
|
|
resample=image_bilinear_resample_kind(),
|
|
)
|
|
tri_mask = Image.new("L", (width, height), 0)
|
|
ImageDraw.Draw(tri_mask).polygon(local_dest, fill=255)
|
|
texture.paste(patch, (min_x, min_y), tri_mask)
|
|
return True
|
|
|
|
|
|
def affine_dest_to_source(
|
|
dest_triangle: list[tuple[float, float]],
|
|
source_triangle: list[tuple[float, float]],
|
|
) -> tuple[float, float, float, float, float, float] | None:
|
|
matrix = np.asarray(
|
|
[[x, y, 1.0] for x, y in dest_triangle],
|
|
dtype=np.float64,
|
|
)
|
|
src_x = np.asarray([x for x, _y in source_triangle], dtype=np.float64)
|
|
src_y = np.asarray([y for _x, y in source_triangle], dtype=np.float64)
|
|
try:
|
|
ax, bx, cx = np.linalg.solve(matrix, src_x)
|
|
ay, by, cy = np.linalg.solve(matrix, src_y)
|
|
except np.linalg.LinAlgError:
|
|
return None
|
|
coeffs = (float(ax), float(bx), float(cx), float(ay), float(by), float(cy))
|
|
if not all(math.isfinite(value) for value in coeffs):
|
|
return None
|
|
return coeffs
|
|
|
|
|
|
def image_affine_transform_kind() -> Any:
|
|
transform = getattr(Image, "Transform", None)
|
|
if transform is not None and hasattr(transform, "AFFINE"):
|
|
return transform.AFFINE
|
|
return Image.AFFINE
|
|
|
|
|
|
def image_bilinear_resample_kind() -> Any:
|
|
resampling = getattr(Image, "Resampling", None)
|
|
if resampling is not None and hasattr(resampling, "BILINEAR"):
|
|
return resampling.BILINEAR
|
|
return Image.BILINEAR
|
|
|
|
|
|
def uv_polygon_pixels(uv: np.ndarray, atlas_size: int) -> list[tuple[int, int]]:
|
|
u = np.clip(uv[:, 0], 0.0, 1.0)
|
|
v = np.clip(uv[:, 1], 0.0, 1.0)
|
|
x = np.rint(u * (atlas_size - 1)).astype(int)
|
|
y = np.rint((1.0 - v) * (atlas_size - 1)).astype(int)
|
|
return [(int(px), int(py)) for px, py in zip(x, y)]
|
|
|
|
|
|
def polygon_area(points: list[tuple[int, int]]) -> float:
|
|
if len(points) < 3:
|
|
return 0.0
|
|
area = 0.0
|
|
for idx, (x0, y0) in enumerate(points):
|
|
x1, y1 = points[(idx + 1) % len(points)]
|
|
area += x0 * y1 - x1 * y0
|
|
return abs(area) * 0.5
|
|
|
|
|
|
def triangle_area_float(points: list[tuple[float, float]]) -> float:
|
|
if len(points) < 3:
|
|
return 0.0
|
|
(x0, y0), (x1, y1), (x2, y2) = points[:3]
|
|
return abs((x0 * (y1 - y2) + x1 * (y2 - y0) + x2 * (y0 - y1)) * 0.5)
|
|
|
|
|
|
def write_textured_obj(*, source_mesh: ObjMesh, output_obj: Path, texture_name: str) -> None:
|
|
material_name = safe_token(output_obj.stem) + "_baked"
|
|
material_file = output_obj.with_suffix(".mtl")
|
|
with material_file.open("w", encoding="utf-8") as dst:
|
|
dst.write("# Generated Real2Sim COLMAP texture bake material\n")
|
|
dst.write(f"newmtl {material_name}\n")
|
|
dst.write("Ka 1.000000 1.000000 1.000000\n")
|
|
dst.write("Kd 1.000000 1.000000 1.000000\n")
|
|
dst.write("Ks 0.030000 0.030000 0.030000\n")
|
|
dst.write("Ns 8.000000\n")
|
|
dst.write("d 1.000000\n")
|
|
dst.write("illum 2\n")
|
|
dst.write(f"map_Kd {texture_name}\n")
|
|
with output_obj.open("w", encoding="utf-8") as dst:
|
|
dst.write("# Real2Sim COLMAP photo-textured scene mesh\n")
|
|
dst.write(f"mtllib {material_file.name}\n")
|
|
dst.write("o real2sim_colmap_textured_scene\n")
|
|
for vertex, color in zip(source_mesh.vertices, source_mesh.colors):
|
|
dst.write(
|
|
f"v {vertex[0]:.9g} {vertex[1]:.9g} {vertex[2]:.9g} "
|
|
f"{color[0]:.6f} {color[1]:.6f} {color[2]:.6f}\n"
|
|
)
|
|
for texcoord in source_mesh.texcoords:
|
|
dst.write(f"vt {texcoord[0]:.7g} {texcoord[1]:.7g}\n")
|
|
dst.write(f"usemtl {material_name}\n")
|
|
for face in source_mesh.faces:
|
|
dst.write(
|
|
"f "
|
|
f"{face.vertices[0] + 1}/{face.texcoords[0] + 1} "
|
|
f"{face.vertices[1] + 1}/{face.texcoords[1] + 1} "
|
|
f"{face.vertices[2] + 1}/{face.texcoords[2] + 1}\n"
|
|
)
|
|
|
|
|
|
def qvec_to_rotmat(qvec: tuple[float, float, float, float]) -> list[list[float]]:
|
|
w, x, y, z = qvec
|
|
return [
|
|
[1 - 2 * y * y - 2 * z * z, 2 * x * y - 2 * w * z, 2 * z * x + 2 * w * y],
|
|
[2 * x * y + 2 * w * z, 1 - 2 * x * x - 2 * z * z, 2 * y * z - 2 * w * x],
|
|
[2 * z * x - 2 * w * y, 2 * y * z + 2 * w * x, 1 - 2 * x * x - 2 * y * y],
|
|
]
|
|
|
|
|
|
def parse_rgb(value: str) -> tuple[int, int, int]:
|
|
parts = [int(part.strip()) for part in value.split(",")]
|
|
if len(parts) != 3:
|
|
raise SystemExit("--background must be R,G,B")
|
|
return tuple(max(0, min(255, part)) for part in parts) # type: ignore[return-value]
|
|
|
|
|
|
def safe_token(value: str) -> str:
|
|
token = re.sub(r"[^A-Za-z0-9_]+", "_", value.strip())
|
|
return token.strip("_") or "scene"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|