329 lines
13 KiB
Python
329 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Paper-style visual instance refinement for Real2Sim.
|
|
|
|
The GS-Playground paper avoids semantic-similarity de-duplication after
|
|
open-vocabulary detection. This stage uses only visual mask criteria and a
|
|
composite detection/segmentation confidence score.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, REPO_ROOT.as_posix())
|
|
|
|
from real2sim.contracts import read_json, write_json
|
|
from real2sim.image_io import read_gray, read_rgb, write_gray
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Refine SAM/SAM2 instances using visual de-duplication")
|
|
parser.add_argument("--instances", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--image", type=Path, default=None, help="Optional source RGB image for accessory mask recovery")
|
|
parser.add_argument("--mask-iou-threshold", type=float, default=0.75)
|
|
parser.add_argument("--inclusion-threshold", type=float, default=0.92)
|
|
parser.add_argument("--boundary-overlap-threshold", type=float, default=0.85)
|
|
parser.add_argument("--merge-saturated-accessories", action="store_true")
|
|
parser.add_argument("--accessory-search-margin-ratio", type=float, default=0.25)
|
|
parser.add_argument("--accessory-saturation-threshold", type=float, default=0.45)
|
|
parser.add_argument("--accessory-contact-passes", type=int, default=4)
|
|
parser.add_argument("--accessory-max-area-growth", type=float, default=0.15)
|
|
parser.add_argument("--accessory-min-component-area-ratio", type=float, default=0.001)
|
|
parser.add_argument("--accessory-max-components", type=int, default=8)
|
|
args = parser.parse_args()
|
|
|
|
raw = read_json(args.instances)
|
|
entries = raw.get("instances", raw) if isinstance(raw, dict) else raw
|
|
image = read_rgb(args.image) if args.image is not None and args.image.exists() else None
|
|
refined = refine_instance_entries(
|
|
entries,
|
|
args.instances.parent,
|
|
image=image,
|
|
merge_saturated_accessories=args.merge_saturated_accessories,
|
|
mask_iou_threshold=args.mask_iou_threshold,
|
|
inclusion_threshold=args.inclusion_threshold,
|
|
boundary_overlap_threshold=args.boundary_overlap_threshold,
|
|
accessory_search_margin_ratio=args.accessory_search_margin_ratio,
|
|
accessory_saturation_threshold=args.accessory_saturation_threshold,
|
|
accessory_contact_passes=args.accessory_contact_passes,
|
|
accessory_max_area_growth=args.accessory_max_area_growth,
|
|
accessory_min_component_area_ratio=args.accessory_min_component_area_ratio,
|
|
accessory_max_components=args.accessory_max_components,
|
|
)
|
|
metadata = dict(raw.get("metadata", {})) if isinstance(raw, dict) else {}
|
|
metadata["instance_refinement"] = {
|
|
"backend": "visual_mask_dedup",
|
|
"input_instances": len(entries),
|
|
"output_instances": len(refined),
|
|
"mask_iou_threshold": args.mask_iou_threshold,
|
|
"inclusion_threshold": args.inclusion_threshold,
|
|
"boundary_overlap_threshold": args.boundary_overlap_threshold,
|
|
"merge_saturated_accessories": args.merge_saturated_accessories,
|
|
"accessory_search_margin_ratio": args.accessory_search_margin_ratio,
|
|
"accessory_saturation_threshold": args.accessory_saturation_threshold,
|
|
"accessory_contact_passes": args.accessory_contact_passes,
|
|
"accessory_max_area_growth": args.accessory_max_area_growth,
|
|
"accessory_min_component_area_ratio": args.accessory_min_component_area_ratio,
|
|
"accessory_max_components": args.accessory_max_components,
|
|
"accessory_pixels_added": sum(int(entry.get("accessory_pixels_added", 0)) for entry in refined),
|
|
}
|
|
write_json(args.output, {"instances": refined, "metadata": metadata})
|
|
return 0
|
|
|
|
|
|
def refine_instance_entries(
|
|
entries: list[dict[str, Any]],
|
|
base_dir: Path,
|
|
*,
|
|
image: np.ndarray | None = None,
|
|
merge_saturated_accessories: bool = False,
|
|
mask_iou_threshold: float = 0.75,
|
|
inclusion_threshold: float = 0.92,
|
|
boundary_overlap_threshold: float = 0.85,
|
|
accessory_search_margin_ratio: float = 0.25,
|
|
accessory_saturation_threshold: float = 0.45,
|
|
accessory_contact_passes: int = 4,
|
|
accessory_max_area_growth: float = 0.15,
|
|
accessory_min_component_area_ratio: float = 0.001,
|
|
accessory_max_components: int = 8,
|
|
) -> list[dict[str, Any]]:
|
|
prepared = []
|
|
for index, entry in enumerate(entries):
|
|
mask_path = entry.get("mask_path", entry.get("mask"))
|
|
if not mask_path:
|
|
continue
|
|
path = Path(str(mask_path))
|
|
path = path if path.is_absolute() else (base_dir / path).resolve()
|
|
if not path.exists():
|
|
continue
|
|
mask = read_gray(path) > 0.5
|
|
if merge_saturated_accessories and image is not None and image.shape[:2] == mask.shape:
|
|
mask, merge_event = merge_saturated_accessories_into_mask(
|
|
mask,
|
|
image,
|
|
bbox_xyxy=entry.get("bbox_xyxy"),
|
|
search_margin_ratio=accessory_search_margin_ratio,
|
|
saturation_threshold=accessory_saturation_threshold,
|
|
contact_passes=accessory_contact_passes,
|
|
max_area_growth=accessory_max_area_growth,
|
|
min_component_area_ratio=accessory_min_component_area_ratio,
|
|
max_components=accessory_max_components,
|
|
)
|
|
if merge_event["added_pixels"] > 0:
|
|
write_gray(path, mask.astype(np.float32))
|
|
entry = dict(entry)
|
|
entry["accessory_pixels_added"] = merge_event["added_pixels"]
|
|
entry["accessory_components_added"] = merge_event["components_added"]
|
|
score = composite_score(entry)
|
|
item = dict(entry)
|
|
item["composite_score"] = score
|
|
item["_input_index"] = index
|
|
prepared.append((item, mask))
|
|
|
|
prepared.sort(key=lambda pair: (-float(pair[0]["composite_score"]), int(pair[0]["_input_index"])))
|
|
kept: list[tuple[dict[str, Any], np.ndarray]] = []
|
|
for item, mask in prepared:
|
|
duplicate = False
|
|
for kept_item, kept_mask in kept:
|
|
if is_visual_duplicate(
|
|
mask,
|
|
kept_mask,
|
|
mask_iou_threshold=mask_iou_threshold,
|
|
inclusion_threshold=inclusion_threshold,
|
|
boundary_overlap_threshold=boundary_overlap_threshold,
|
|
):
|
|
duplicate = True
|
|
break
|
|
if not duplicate:
|
|
kept.append((item, mask))
|
|
|
|
kept.sort(key=lambda pair: int(pair[0]["_input_index"]))
|
|
output = []
|
|
for idx, (item, _) in enumerate(kept):
|
|
cleaned = dict(item)
|
|
cleaned.pop("_input_index", None)
|
|
cleaned["id"] = str(cleaned.get("id", cleaned.get("instance_id", f"inst_{idx:03d}")))
|
|
output.append(cleaned)
|
|
return output
|
|
|
|
|
|
def composite_score(entry: dict[str, Any]) -> float:
|
|
detection = entry.get("detection_score", entry.get("score", 0.0))
|
|
segmentation = entry.get("segmentation_score", entry.get("mask_score", detection))
|
|
detection = 0.0 if detection is None else float(detection)
|
|
segmentation = 0.0 if segmentation is None else float(segmentation)
|
|
return 0.5 * detection + 0.5 * segmentation
|
|
|
|
|
|
def is_visual_duplicate(
|
|
mask: np.ndarray,
|
|
reference: np.ndarray,
|
|
*,
|
|
mask_iou_threshold: float,
|
|
inclusion_threshold: float,
|
|
boundary_overlap_threshold: float,
|
|
) -> bool:
|
|
if mask.shape != reference.shape:
|
|
return False
|
|
intersection = int((mask & reference).sum())
|
|
area = int(mask.sum())
|
|
reference_area = int(reference.sum())
|
|
if area == 0 or reference_area == 0:
|
|
return False
|
|
union = area + reference_area - intersection
|
|
if union > 0 and intersection / union >= mask_iou_threshold:
|
|
return True
|
|
smaller = mask if area <= reference_area else reference
|
|
larger = reference if area <= reference_area else mask
|
|
inclusion = int((smaller & larger).sum()) / float(int(smaller.sum()))
|
|
boundary_overlap = int((boundary(smaller) & larger).sum()) / max(1, int(boundary(smaller).sum()))
|
|
return inclusion >= inclusion_threshold and boundary_overlap >= boundary_overlap_threshold
|
|
|
|
|
|
def boundary(mask: np.ndarray) -> np.ndarray:
|
|
eroded = mask.copy()
|
|
padded = np.pad(mask, 1, mode="constant", constant_values=False)
|
|
eroded &= padded[:-2, 1:-1]
|
|
eroded &= padded[2:, 1:-1]
|
|
eroded &= padded[1:-1, :-2]
|
|
eroded &= padded[1:-1, 2:]
|
|
return mask & ~eroded
|
|
|
|
|
|
def merge_saturated_accessories_into_mask(
|
|
mask: np.ndarray,
|
|
image: np.ndarray,
|
|
*,
|
|
bbox_xyxy: Any | None = None,
|
|
search_margin_ratio: float = 0.25,
|
|
saturation_threshold: float = 0.45,
|
|
contact_passes: int = 4,
|
|
max_area_growth: float = 0.15,
|
|
min_component_area_ratio: float = 0.001,
|
|
max_components: int = 8,
|
|
) -> tuple[np.ndarray, dict[str, int]]:
|
|
base = np.asarray(mask, dtype=bool)
|
|
if not base.any():
|
|
return base, {"added_pixels": 0, "components_added": 0}
|
|
|
|
search = expanded_search_region(base, bbox_xyxy, margin_ratio=search_margin_ratio)
|
|
rgb = np.asarray(image, dtype=np.float32)
|
|
if rgb.max(initial=0.0) > 1.5:
|
|
rgb = rgb / 255.0
|
|
saturation = rgb_saturation(rgb)
|
|
candidates = (saturation >= saturation_threshold) & search & ~base
|
|
if not candidates.any():
|
|
return base, {"added_pixels": 0, "components_added": 0}
|
|
|
|
contact = dilate(base, max(0, int(contact_passes)))
|
|
max_added = max(1, int(base.sum() * max(0.0, max_area_growth)))
|
|
min_component_area = max(1, int(np.ceil(base.sum() * max(0.0, min_component_area_ratio))))
|
|
merged = base.copy()
|
|
added_pixels = 0
|
|
components_added = 0
|
|
components = connected_components(candidates)
|
|
components.sort(key=lambda comp: int(comp.sum()), reverse=True)
|
|
for component in components:
|
|
if max_components > 0 and components_added >= max_components:
|
|
break
|
|
area = int(component.sum())
|
|
if area < min_component_area or added_pixels + area > max_added:
|
|
continue
|
|
if not (component & contact).any():
|
|
continue
|
|
merged |= component
|
|
contact = dilate(merged, max(0, int(contact_passes)))
|
|
added_pixels += area
|
|
components_added += 1
|
|
return merged, {"added_pixels": added_pixels, "components_added": components_added}
|
|
|
|
|
|
def expanded_search_region(mask: np.ndarray, bbox_xyxy: Any | None, *, margin_ratio: float) -> np.ndarray:
|
|
h, w = mask.shape
|
|
y0, y1, x0, x1 = mask_bounds(mask)
|
|
if bbox_xyxy is not None:
|
|
try:
|
|
bx0, by0, bx1, by1 = [float(v) for v in bbox_xyxy]
|
|
x0 = min(x0, int(np.floor(bx0)))
|
|
y0 = min(y0, int(np.floor(by0)))
|
|
x1 = max(x1, int(np.ceil(bx1)))
|
|
y1 = max(y1, int(np.ceil(by1)))
|
|
except (TypeError, ValueError):
|
|
pass
|
|
box_w = max(1, x1 - x0 + 1)
|
|
box_h = max(1, y1 - y0 + 1)
|
|
margin_x = max(1, int(round(box_w * max(0.0, margin_ratio))))
|
|
margin_y = max(1, int(round(box_h * max(0.0, margin_ratio))))
|
|
x0 = max(0, x0 - margin_x)
|
|
x1 = min(w - 1, x1 + margin_x)
|
|
y0 = max(0, y0 - margin_y)
|
|
y1 = min(h - 1, y1 + margin_y)
|
|
region = np.zeros((h, w), dtype=bool)
|
|
region[y0 : y1 + 1, x0 : x1 + 1] = True
|
|
return region
|
|
|
|
|
|
def mask_bounds(mask: np.ndarray) -> tuple[int, int, int, int]:
|
|
ys, xs = np.where(mask)
|
|
if ys.size == 0:
|
|
return 0, 0, 0, 0
|
|
return int(ys.min()), int(ys.max()), int(xs.min()), int(xs.max())
|
|
|
|
|
|
def rgb_saturation(rgb: np.ndarray) -> np.ndarray:
|
|
clipped = np.clip(rgb[:, :, :3], 0.0, 1.0)
|
|
maxc = clipped.max(axis=2)
|
|
minc = clipped.min(axis=2)
|
|
return (maxc - minc) / np.maximum(maxc, 1e-6)
|
|
|
|
|
|
def connected_components(mask: np.ndarray) -> list[np.ndarray]:
|
|
h, w = mask.shape
|
|
visited = np.zeros_like(mask, dtype=bool)
|
|
components: list[np.ndarray] = []
|
|
ys, xs = np.where(mask)
|
|
for sy, sx in zip(ys.tolist(), xs.tolist()):
|
|
if visited[sy, sx] or not mask[sy, sx]:
|
|
continue
|
|
stack = [(sy, sx)]
|
|
visited[sy, sx] = True
|
|
pixels = []
|
|
while stack:
|
|
y, x = stack.pop()
|
|
pixels.append((y, x))
|
|
for ny in range(y - 1, y + 2):
|
|
for nx in range(x - 1, x + 2):
|
|
if ny == y and nx == x:
|
|
continue
|
|
if 0 <= ny < h and 0 <= nx < w and mask[ny, nx] and not visited[ny, nx]:
|
|
visited[ny, nx] = True
|
|
stack.append((ny, nx))
|
|
component = np.zeros_like(mask, dtype=bool)
|
|
for y, x in pixels:
|
|
component[y, x] = True
|
|
components.append(component)
|
|
return components
|
|
|
|
|
|
def dilate(mask: np.ndarray, passes: int) -> np.ndarray:
|
|
out = np.asarray(mask, dtype=bool).copy()
|
|
for _ in range(max(0, passes)):
|
|
padded = np.pad(out, 1, mode="constant", constant_values=False)
|
|
expanded = out.copy()
|
|
for dy in range(3):
|
|
for dx in range(3):
|
|
expanded |= padded[dy : dy + out.shape[0], dx : dx + out.shape[1]]
|
|
out = expanded
|
|
return out
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|