Enable RT-DETR CoreML export and protect decoder ops from FP16 drift (#24510)

Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: Onuralp SEZER <onuralp@ultralytics.com>
Co-authored-by: Jing Qiu <61612323+Laughing-q@users.noreply.github.com>
Co-authored-by: Laughing-q <1185102784@qq.com>
Co-authored-by: Ultralytics Assistant <135830346+UltralyticsAssistant@users.noreply.github.com>
This commit is contained in:
artest08 2026-05-27 09:57:09 +03:00 committed by GitHub
parent 9a414a82a2
commit ef80b5d08b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 83 additions and 50 deletions

View File

@ -16,7 +16,17 @@ import torch
from tests import SOURCE
from ultralytics import YOLO
from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS
from ultralytics.utils import ARM64, IS_DOCKER, IS_RASPBERRYPI, LINUX, MACOS, MACOS_VERSION, WINDOWS, checks
from ultralytics.utils import (
ARM64,
IS_DOCKER,
IS_RASPBERRYPI,
LINUX,
MACOS,
MACOS_VERSION,
WEIGHTS_DIR,
WINDOWS,
checks,
)
from ultralytics.utils.export.engine import torch2onnx
from ultralytics.utils.torch_utils import (
TORCH_1_10,
@ -273,6 +283,23 @@ def test_export_coreml(isolated_model):
assert "You will not be able to run predict()" not in output, "CoreML export has predict() error"
@pytest.mark.skipif(not TORCH_1_11, reason="RTDETR CoreML export requires torch>=1.11")
@pytest.mark.skipif(WINDOWS, reason="CoreML not supported on Windows")
@pytest.mark.skipif(LINUX and ARM64, reason="CoreML not supported on aarch64 Linux")
@pytest.mark.skipif(checks.IS_PYTHON_3_13, reason="CoreML not supported in Python 3.13")
def test_export_coreml_rtdetr():
"""Test RT-DETR export to CoreML format and check for errors."""
stdout, stderr = io.StringIO(), io.StringIO()
with redirect_stdout(stdout), redirect_stderr(stderr):
file = YOLO(WEIGHTS_DIR / "rtdetr-l.pt").export(format="coreml", imgsz=160)
if MACOS:
YOLO(file)(SOURCE, imgsz=160)
output = stdout.getvalue() + stderr.getvalue()
assert "Error" not in output, f"RTDETR CoreML export produced errors: {output}"
assert "You will not be able to run predict()" not in output, "RTDETR CoreML export has predict() error"
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10, reason="TFLite export requires Python>=3.10")
@pytest.mark.skipif(not TORCH_1_13, reason="TFLite export requires torch>=1.13")
@pytest.mark.skipif(not LINUX, reason="Test disabled as TF suffers from install conflicts on Windows and macOS")

View File

@ -1604,9 +1604,10 @@ class RTDETRDecoder(nn.Module):
cy, w, h, max_class_prob, class_index].
"""
scores, index = scores.flatten(1).topk(self.num_queries)
query_idx = index // self.nc
boxes = boxes.gather(dim=1, index=query_idx.unsqueeze(-1).expand(-1, -1, 4))
return torch.cat([boxes, scores[..., None], (index % self.nc)[..., None].float()], dim=-1)
# CoreML MIL lacks integer floor-div and mod lowering: use torch.div(rounding_mode="floor") and (index - q*nc).
query_idx = torch.div(index, self.nc, rounding_mode="floor")
boxes = boxes.gather(dim=1, index=query_idx.unsqueeze(-1).expand(-1, -1, 4).long())
return torch.cat([boxes, scores[..., None], (index - query_idx * self.nc)[..., None].float()], dim=-1)
@staticmethod
def _generate_anchors(

View File

@ -234,8 +234,9 @@ class AIFI(TransformerEncoderLayer):
omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim
omega = 1.0 / (temperature**omega)
out_w = grid_w.flatten()[..., None] @ omega[None]
out_h = grid_h.flatten()[..., None] @ omega[None]
# Pin matmul to fp32 for CoreML export: fp16 sin/cos on integer-derived positions accumulates visible error.
out_w = grid_w.flatten()[..., None].float() @ omega[None]
out_h = grid_h.flatten()[..., None].float() @ omega[None]
return torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], 1)[None]
@ -539,8 +540,8 @@ class MSDeformAttn(nn.Module):
Args:
query (torch.Tensor): Query tensor with shape [bs, query_length, C].
refer_bbox (torch.Tensor): Reference bounding boxes with shape [bs, query_length, n_levels, 2 or 4], range
in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area.
refer_bbox (torch.Tensor): Reference bounding boxes with shape [bs, query_length, 1, 2 or 4], range in [0,
1], top-left (0,0), bottom-right (1, 1). The size-1 axis broadcasts across n_levels.
value (torch.Tensor): Value tensor with shape [bs, value_length, C].
value_shapes (list): List with shape [n_levels, 2], [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})].
value_mask (torch.Tensor, optional): Mask tensor with shape [bs, value_length], True for padding elements,
@ -560,18 +561,21 @@ class MSDeformAttn(nn.Module):
if value_mask is not None:
value = value.masked_fill(value_mask[..., None], float(0))
value = value.view(bs, len_v, self.n_heads, self.d_model // self.n_heads)
sampling_offsets = self.sampling_offsets(query).view(bs, len_q, self.n_heads, self.n_levels, self.n_points, 2)
attention_weights = self.attention_weights(query).view(bs, len_q, self.n_heads, self.n_levels * self.n_points)
attention_weights = F.softmax(attention_weights, -1).view(bs, len_q, self.n_heads, self.n_levels, self.n_points)
# N, Len_q, n_heads, n_levels, n_points, 2
# Fold (n_levels, n_points) into one axis so every traced tensor stays at rank <= 5 (required for CoreML
# export); refer_bbox arrives as (bs, len_q, 1, 2 or 4) and its size-1 axis broadcasts implicitly.
n_total_points = self.n_levels * self.n_points
sampling_offsets = self.sampling_offsets(query).view(bs, len_q, self.n_heads, n_total_points, 2)
attention_weights = self.attention_weights(query).view(bs, len_q, self.n_heads, n_total_points)
attention_weights = F.softmax(attention_weights, -1)
num_points = refer_bbox.shape[-1]
if num_points == 2:
offset_normalizer = torch.as_tensor(value_shapes, dtype=query.dtype, device=query.device).flip(-1)
add = sampling_offsets / offset_normalizer[None, None, None, :, None, :]
sampling_locations = refer_bbox[:, :, None, :, None, :] + add
offset_normalizer = offset_normalizer[:, None, :].expand(-1, self.n_points, -1).reshape(n_total_points, 2)
sampling_locations = refer_bbox[:, :, None, :, :] + sampling_offsets / offset_normalizer
elif num_points == 4:
add = sampling_offsets / self.n_points * refer_bbox[:, :, None, :, None, 2:] * 0.5
sampling_locations = refer_bbox[:, :, None, :, None, :2] + add
sampling_locations = (
refer_bbox[:, :, None, :, :2] + sampling_offsets / self.n_points * refer_bbox[:, :, None, :, 2:] * 0.5
)
else:
raise ValueError(f"Last dim of reference_points must be 2 or 4, but got {num_points}.")
output = multi_scale_deformable_attn_pytorch(value, value_shapes, sampling_locations, attention_weights)

View File

@ -100,57 +100,48 @@ def inverse_sigmoid(x, eps=1e-5):
def multi_scale_deformable_attn_pytorch(
value: torch.Tensor,
value_spatial_shapes: torch.Tensor,
value_spatial_shapes: list,
sampling_locations: torch.Tensor,
attention_weights: torch.Tensor,
) -> torch.Tensor:
"""Implement multi-scale deformable attention in PyTorch.
This function performs deformable attention across multiple feature map scales, allowing the model to attend to
different spatial locations with learned offsets.
Folds the (num_levels, num_points) axes into a single num_total_points axis so every traced tensor stays at rank <=
5, the maximum rank supported by CoreML's MIL converter. Numerically equivalent to the rank-6 reference
implementation on CUDA and CPU.
Args:
value (torch.Tensor): The value tensor with shape (bs, num_keys, num_heads, embed_dims).
value_spatial_shapes (torch.Tensor): Spatial shapes of the value tensor with shape (num_levels, 2).
sampling_locations (torch.Tensor): The sampling locations with shape (bs, num_queries, num_heads, num_levels,
value (torch.Tensor): Value tensor with shape (bs, num_keys, num_heads, embed_dims).
value_spatial_shapes (list): Per-level spatial shapes as [(H_0, W_0), ..., (H_{L-1}, W_{L-1})].
sampling_locations (torch.Tensor): Sampling locations with shape (bs, num_queries, num_heads, num_levels *
num_points, 2).
attention_weights (torch.Tensor): The attention weights with shape (bs, num_queries, num_heads, num_levels,
attention_weights (torch.Tensor): Attention weights with shape (bs, num_queries, num_heads, num_levels *
num_points).
Returns:
(torch.Tensor): The output tensor with shape (bs, num_queries, num_heads * embed_dims).
(torch.Tensor): Output tensor with shape (bs, num_queries, num_heads * embed_dims).
References:
https://github.com/IDEA-Research/detrex/blob/main/detrex/layers/multi_scale_deform_attn.py
"""
bs, _, num_heads, embed_dims = value.shape
_, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape
value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1)
sampling_grids = 2 * sampling_locations - 1
_, num_queries, _, num_total_points, _ = sampling_locations.shape
num_points = num_total_points // len(value_spatial_shapes)
# (bs, num_keys, num_heads, embed_dims) -> tuple of (bs*num_heads, embed_dims, H*W) per level
value_list = value.permute(0, 2, 3, 1).flatten(0, 1).split([h * w for h, w in value_spatial_shapes], dim=-1)
# Map to grid_sample coords in [-1, 1] and split per level: tuple of (bs*num_heads, num_queries, num_points, 2)
sampling_grids = (2 * sampling_locations - 1).permute(0, 2, 1, 3, 4).flatten(0, 1).split(num_points, dim=-2)
sampling_value_list = []
for level, (H_, W_) in enumerate(value_spatial_shapes):
# bs, H_*W_, num_heads, embed_dims ->
# bs, H_*W_, num_heads*embed_dims ->
# bs, num_heads*embed_dims, H_*W_ ->
# bs*num_heads, embed_dims, H_, W_
value_l_ = value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_)
# bs, num_queries, num_heads, num_points, 2 ->
# bs, num_heads, num_queries, num_points, 2 ->
# bs*num_heads, num_queries, num_points, 2
sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1)
# bs*num_heads, embed_dims, num_queries, num_points
sampling_value_l_ = F.grid_sample(
value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False
for level, (h, w) in enumerate(value_spatial_shapes):
value_l = value_list[level].reshape(bs * num_heads, embed_dims, h, w)
sampling_value_list.append(
F.grid_sample(value_l, sampling_grids[level], mode="bilinear", padding_mode="zeros", align_corners=False)
)
sampling_value_list.append(sampling_value_l_)
# (bs, num_queries, num_heads, num_levels, num_points) ->
# (bs, num_heads, num_queries, num_levels, num_points) ->
# (bs*num_heads, 1, num_queries, num_levels*num_points)
attention_weights = attention_weights.transpose(1, 2).reshape(
bs * num_heads, 1, num_queries, num_levels * num_points
)
attention_weights = attention_weights.permute(0, 2, 1, 3).reshape(bs * num_heads, 1, num_queries, num_total_points)
output = (
(torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights)
(torch.cat(sampling_value_list, dim=-1) * attention_weights)
.sum(-1)
.view(bs, num_heads * embed_dims, num_queries)
)

View File

@ -202,13 +202,23 @@ def torch2coreml(
# Internally based on the model conversion and output type.
# Setting minimum_deployment_target >= iOS16 will require setting compute_precision=ct.precision.FLOAT32.
# iOS16 adds in better support for FP16, but none of the CoreML NMS specifications handle FP16 as input.
ct_model = ct.convert(
ts,
convert_kwargs = dict(
inputs=inputs,
classifier_config=ct.ClassifierConfig(classifier_names) if classifier_names else None,
convert_to="neuralnetwork" if mlmodel else "mlprogram",
skip_model_load=True,
)
if not mlmodel:
# RT-DETR decoder class logits and deformable-sampling indices drift in fp16; pin those op types to fp32
# only when an RTDETRDecoder is present. YOLO detect/segment/pose/OBB keep mlprogram's fp16 default.
from ultralytics.nn.modules.head import RTDETRDecoder
if any(isinstance(m, RTDETRDecoder) for m in model.modules()):
fp32_ops = {"linear", "gather", "gather_nd", "gather_along_axis"}
convert_kwargs["compute_precision"] = ct.transform.FP16ComputePrecision(
op_selector=lambda op: op.op_type not in fp32_ops
)
ct_model = ct.convert(ts, **convert_kwargs)
bits, mode = (8, "kmeans") if int8 else (16, "linear") if half else (32, None)
if bits < 32:
if "kmeans" in mode: