Enable RT-DETR CoreML export and protect decoder ops from FP16 drift (#24510)
Co-authored-by: UltralyticsAssistant <web@ultralytics.com> Co-authored-by: Onuralp SEZER <onuralp@ultralytics.com> Co-authored-by: Jing Qiu <61612323+Laughing-q@users.noreply.github.com> Co-authored-by: Laughing-q <1185102784@qq.com> Co-authored-by: Ultralytics Assistant <135830346+UltralyticsAssistant@users.noreply.github.com>
This commit is contained in:
parent
9a414a82a2
commit
ef80b5d08b
@ -16,7 +16,17 @@ import torch
|
||||
from tests import SOURCE
|
||||
from ultralytics import YOLO
|
||||
from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS
|
||||
from ultralytics.utils import ARM64, IS_DOCKER, IS_RASPBERRYPI, LINUX, MACOS, MACOS_VERSION, WINDOWS, checks
|
||||
from ultralytics.utils import (
|
||||
ARM64,
|
||||
IS_DOCKER,
|
||||
IS_RASPBERRYPI,
|
||||
LINUX,
|
||||
MACOS,
|
||||
MACOS_VERSION,
|
||||
WEIGHTS_DIR,
|
||||
WINDOWS,
|
||||
checks,
|
||||
)
|
||||
from ultralytics.utils.export.engine import torch2onnx
|
||||
from ultralytics.utils.torch_utils import (
|
||||
TORCH_1_10,
|
||||
@ -273,6 +283,23 @@ def test_export_coreml(isolated_model):
|
||||
assert "You will not be able to run predict()" not in output, "CoreML export has predict() error"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCH_1_11, reason="RTDETR CoreML export requires torch>=1.11")
|
||||
@pytest.mark.skipif(WINDOWS, reason="CoreML not supported on Windows")
|
||||
@pytest.mark.skipif(LINUX and ARM64, reason="CoreML not supported on aarch64 Linux")
|
||||
@pytest.mark.skipif(checks.IS_PYTHON_3_13, reason="CoreML not supported in Python 3.13")
|
||||
def test_export_coreml_rtdetr():
|
||||
"""Test RT-DETR export to CoreML format and check for errors."""
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
file = YOLO(WEIGHTS_DIR / "rtdetr-l.pt").export(format="coreml", imgsz=160)
|
||||
if MACOS:
|
||||
YOLO(file)(SOURCE, imgsz=160)
|
||||
|
||||
output = stdout.getvalue() + stderr.getvalue()
|
||||
assert "Error" not in output, f"RTDETR CoreML export produced errors: {output}"
|
||||
assert "You will not be able to run predict()" not in output, "RTDETR CoreML export has predict() error"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10, reason="TFLite export requires Python>=3.10")
|
||||
@pytest.mark.skipif(not TORCH_1_13, reason="TFLite export requires torch>=1.13")
|
||||
@pytest.mark.skipif(not LINUX, reason="Test disabled as TF suffers from install conflicts on Windows and macOS")
|
||||
|
||||
@ -1604,9 +1604,10 @@ class RTDETRDecoder(nn.Module):
|
||||
cy, w, h, max_class_prob, class_index].
|
||||
"""
|
||||
scores, index = scores.flatten(1).topk(self.num_queries)
|
||||
query_idx = index // self.nc
|
||||
boxes = boxes.gather(dim=1, index=query_idx.unsqueeze(-1).expand(-1, -1, 4))
|
||||
return torch.cat([boxes, scores[..., None], (index % self.nc)[..., None].float()], dim=-1)
|
||||
# CoreML MIL lacks integer floor-div and mod lowering: use torch.div(rounding_mode="floor") and (index - q*nc).
|
||||
query_idx = torch.div(index, self.nc, rounding_mode="floor")
|
||||
boxes = boxes.gather(dim=1, index=query_idx.unsqueeze(-1).expand(-1, -1, 4).long())
|
||||
return torch.cat([boxes, scores[..., None], (index - query_idx * self.nc)[..., None].float()], dim=-1)
|
||||
|
||||
@staticmethod
|
||||
def _generate_anchors(
|
||||
|
||||
@ -234,8 +234,9 @@ class AIFI(TransformerEncoderLayer):
|
||||
omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim
|
||||
omega = 1.0 / (temperature**omega)
|
||||
|
||||
out_w = grid_w.flatten()[..., None] @ omega[None]
|
||||
out_h = grid_h.flatten()[..., None] @ omega[None]
|
||||
# Pin matmul to fp32 for CoreML export: fp16 sin/cos on integer-derived positions accumulates visible error.
|
||||
out_w = grid_w.flatten()[..., None].float() @ omega[None]
|
||||
out_h = grid_h.flatten()[..., None].float() @ omega[None]
|
||||
|
||||
return torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], 1)[None]
|
||||
|
||||
@ -539,8 +540,8 @@ class MSDeformAttn(nn.Module):
|
||||
|
||||
Args:
|
||||
query (torch.Tensor): Query tensor with shape [bs, query_length, C].
|
||||
refer_bbox (torch.Tensor): Reference bounding boxes with shape [bs, query_length, n_levels, 2 or 4], range
|
||||
in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area.
|
||||
refer_bbox (torch.Tensor): Reference bounding boxes with shape [bs, query_length, 1, 2 or 4], range in [0,
|
||||
1], top-left (0,0), bottom-right (1, 1). The size-1 axis broadcasts across n_levels.
|
||||
value (torch.Tensor): Value tensor with shape [bs, value_length, C].
|
||||
value_shapes (list): List with shape [n_levels, 2], [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})].
|
||||
value_mask (torch.Tensor, optional): Mask tensor with shape [bs, value_length], True for padding elements,
|
||||
@ -560,18 +561,21 @@ class MSDeformAttn(nn.Module):
|
||||
if value_mask is not None:
|
||||
value = value.masked_fill(value_mask[..., None], float(0))
|
||||
value = value.view(bs, len_v, self.n_heads, self.d_model // self.n_heads)
|
||||
sampling_offsets = self.sampling_offsets(query).view(bs, len_q, self.n_heads, self.n_levels, self.n_points, 2)
|
||||
attention_weights = self.attention_weights(query).view(bs, len_q, self.n_heads, self.n_levels * self.n_points)
|
||||
attention_weights = F.softmax(attention_weights, -1).view(bs, len_q, self.n_heads, self.n_levels, self.n_points)
|
||||
# N, Len_q, n_heads, n_levels, n_points, 2
|
||||
# Fold (n_levels, n_points) into one axis so every traced tensor stays at rank <= 5 (required for CoreML
|
||||
# export); refer_bbox arrives as (bs, len_q, 1, 2 or 4) and its size-1 axis broadcasts implicitly.
|
||||
n_total_points = self.n_levels * self.n_points
|
||||
sampling_offsets = self.sampling_offsets(query).view(bs, len_q, self.n_heads, n_total_points, 2)
|
||||
attention_weights = self.attention_weights(query).view(bs, len_q, self.n_heads, n_total_points)
|
||||
attention_weights = F.softmax(attention_weights, -1)
|
||||
num_points = refer_bbox.shape[-1]
|
||||
if num_points == 2:
|
||||
offset_normalizer = torch.as_tensor(value_shapes, dtype=query.dtype, device=query.device).flip(-1)
|
||||
add = sampling_offsets / offset_normalizer[None, None, None, :, None, :]
|
||||
sampling_locations = refer_bbox[:, :, None, :, None, :] + add
|
||||
offset_normalizer = offset_normalizer[:, None, :].expand(-1, self.n_points, -1).reshape(n_total_points, 2)
|
||||
sampling_locations = refer_bbox[:, :, None, :, :] + sampling_offsets / offset_normalizer
|
||||
elif num_points == 4:
|
||||
add = sampling_offsets / self.n_points * refer_bbox[:, :, None, :, None, 2:] * 0.5
|
||||
sampling_locations = refer_bbox[:, :, None, :, None, :2] + add
|
||||
sampling_locations = (
|
||||
refer_bbox[:, :, None, :, :2] + sampling_offsets / self.n_points * refer_bbox[:, :, None, :, 2:] * 0.5
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Last dim of reference_points must be 2 or 4, but got {num_points}.")
|
||||
output = multi_scale_deformable_attn_pytorch(value, value_shapes, sampling_locations, attention_weights)
|
||||
|
||||
@ -100,57 +100,48 @@ def inverse_sigmoid(x, eps=1e-5):
|
||||
|
||||
def multi_scale_deformable_attn_pytorch(
|
||||
value: torch.Tensor,
|
||||
value_spatial_shapes: torch.Tensor,
|
||||
value_spatial_shapes: list,
|
||||
sampling_locations: torch.Tensor,
|
||||
attention_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Implement multi-scale deformable attention in PyTorch.
|
||||
|
||||
This function performs deformable attention across multiple feature map scales, allowing the model to attend to
|
||||
different spatial locations with learned offsets.
|
||||
Folds the (num_levels, num_points) axes into a single num_total_points axis so every traced tensor stays at rank <=
|
||||
5, the maximum rank supported by CoreML's MIL converter. Numerically equivalent to the rank-6 reference
|
||||
implementation on CUDA and CPU.
|
||||
|
||||
Args:
|
||||
value (torch.Tensor): The value tensor with shape (bs, num_keys, num_heads, embed_dims).
|
||||
value_spatial_shapes (torch.Tensor): Spatial shapes of the value tensor with shape (num_levels, 2).
|
||||
sampling_locations (torch.Tensor): The sampling locations with shape (bs, num_queries, num_heads, num_levels,
|
||||
value (torch.Tensor): Value tensor with shape (bs, num_keys, num_heads, embed_dims).
|
||||
value_spatial_shapes (list): Per-level spatial shapes as [(H_0, W_0), ..., (H_{L-1}, W_{L-1})].
|
||||
sampling_locations (torch.Tensor): Sampling locations with shape (bs, num_queries, num_heads, num_levels *
|
||||
num_points, 2).
|
||||
attention_weights (torch.Tensor): The attention weights with shape (bs, num_queries, num_heads, num_levels,
|
||||
attention_weights (torch.Tensor): Attention weights with shape (bs, num_queries, num_heads, num_levels *
|
||||
num_points).
|
||||
|
||||
Returns:
|
||||
(torch.Tensor): The output tensor with shape (bs, num_queries, num_heads * embed_dims).
|
||||
(torch.Tensor): Output tensor with shape (bs, num_queries, num_heads * embed_dims).
|
||||
|
||||
References:
|
||||
https://github.com/IDEA-Research/detrex/blob/main/detrex/layers/multi_scale_deform_attn.py
|
||||
"""
|
||||
bs, _, num_heads, embed_dims = value.shape
|
||||
_, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape
|
||||
value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1)
|
||||
sampling_grids = 2 * sampling_locations - 1
|
||||
_, num_queries, _, num_total_points, _ = sampling_locations.shape
|
||||
num_points = num_total_points // len(value_spatial_shapes)
|
||||
|
||||
# (bs, num_keys, num_heads, embed_dims) -> tuple of (bs*num_heads, embed_dims, H*W) per level
|
||||
value_list = value.permute(0, 2, 3, 1).flatten(0, 1).split([h * w for h, w in value_spatial_shapes], dim=-1)
|
||||
# Map to grid_sample coords in [-1, 1] and split per level: tuple of (bs*num_heads, num_queries, num_points, 2)
|
||||
sampling_grids = (2 * sampling_locations - 1).permute(0, 2, 1, 3, 4).flatten(0, 1).split(num_points, dim=-2)
|
||||
|
||||
sampling_value_list = []
|
||||
for level, (H_, W_) in enumerate(value_spatial_shapes):
|
||||
# bs, H_*W_, num_heads, embed_dims ->
|
||||
# bs, H_*W_, num_heads*embed_dims ->
|
||||
# bs, num_heads*embed_dims, H_*W_ ->
|
||||
# bs*num_heads, embed_dims, H_, W_
|
||||
value_l_ = value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_)
|
||||
# bs, num_queries, num_heads, num_points, 2 ->
|
||||
# bs, num_heads, num_queries, num_points, 2 ->
|
||||
# bs*num_heads, num_queries, num_points, 2
|
||||
sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1)
|
||||
# bs*num_heads, embed_dims, num_queries, num_points
|
||||
sampling_value_l_ = F.grid_sample(
|
||||
value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False
|
||||
for level, (h, w) in enumerate(value_spatial_shapes):
|
||||
value_l = value_list[level].reshape(bs * num_heads, embed_dims, h, w)
|
||||
sampling_value_list.append(
|
||||
F.grid_sample(value_l, sampling_grids[level], mode="bilinear", padding_mode="zeros", align_corners=False)
|
||||
)
|
||||
sampling_value_list.append(sampling_value_l_)
|
||||
# (bs, num_queries, num_heads, num_levels, num_points) ->
|
||||
# (bs, num_heads, num_queries, num_levels, num_points) ->
|
||||
# (bs*num_heads, 1, num_queries, num_levels*num_points)
|
||||
attention_weights = attention_weights.transpose(1, 2).reshape(
|
||||
bs * num_heads, 1, num_queries, num_levels * num_points
|
||||
)
|
||||
attention_weights = attention_weights.permute(0, 2, 1, 3).reshape(bs * num_heads, 1, num_queries, num_total_points)
|
||||
output = (
|
||||
(torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights)
|
||||
(torch.cat(sampling_value_list, dim=-1) * attention_weights)
|
||||
.sum(-1)
|
||||
.view(bs, num_heads * embed_dims, num_queries)
|
||||
)
|
||||
|
||||
@ -202,13 +202,23 @@ def torch2coreml(
|
||||
# Internally based on the model conversion and output type.
|
||||
# Setting minimum_deployment_target >= iOS16 will require setting compute_precision=ct.precision.FLOAT32.
|
||||
# iOS16 adds in better support for FP16, but none of the CoreML NMS specifications handle FP16 as input.
|
||||
ct_model = ct.convert(
|
||||
ts,
|
||||
convert_kwargs = dict(
|
||||
inputs=inputs,
|
||||
classifier_config=ct.ClassifierConfig(classifier_names) if classifier_names else None,
|
||||
convert_to="neuralnetwork" if mlmodel else "mlprogram",
|
||||
skip_model_load=True,
|
||||
)
|
||||
if not mlmodel:
|
||||
# RT-DETR decoder class logits and deformable-sampling indices drift in fp16; pin those op types to fp32
|
||||
# only when an RTDETRDecoder is present. YOLO detect/segment/pose/OBB keep mlprogram's fp16 default.
|
||||
from ultralytics.nn.modules.head import RTDETRDecoder
|
||||
|
||||
if any(isinstance(m, RTDETRDecoder) for m in model.modules()):
|
||||
fp32_ops = {"linear", "gather", "gather_nd", "gather_along_axis"}
|
||||
convert_kwargs["compute_precision"] = ct.transform.FP16ComputePrecision(
|
||||
op_selector=lambda op: op.op_type not in fp32_ops
|
||||
)
|
||||
ct_model = ct.convert(ts, **convert_kwargs)
|
||||
bits, mode = (8, "kmeans") if int8 else (16, "linear") if half else (32, None)
|
||||
if bits < 32:
|
||||
if "kmeans" in mode:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user