From ef80b5d08b2dde1aa6dbe3483f44681237692a2d Mon Sep 17 00:00:00 2001 From: artest08 <43647848+artest08@users.noreply.github.com> Date: Wed, 27 May 2026 09:57:09 +0300 Subject: [PATCH] Enable RT-DETR CoreML export and protect decoder ops from FP16 drift (#24510) Co-authored-by: UltralyticsAssistant Co-authored-by: Onuralp SEZER Co-authored-by: Jing Qiu <61612323+Laughing-q@users.noreply.github.com> Co-authored-by: Laughing-q <1185102784@qq.com> Co-authored-by: Ultralytics Assistant <135830346+UltralyticsAssistant@users.noreply.github.com> --- tests/test_exports.py | 29 +++++++++++++- ultralytics/nn/modules/head.py | 7 ++-- ultralytics/nn/modules/transformer.py | 28 ++++++++------ ultralytics/nn/modules/utils.py | 55 +++++++++++---------------- ultralytics/utils/export/coreml.py | 14 ++++++- 5 files changed, 83 insertions(+), 50 deletions(-) diff --git a/tests/test_exports.py b/tests/test_exports.py index da492682..daa63da0 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -16,7 +16,17 @@ import torch from tests import SOURCE from ultralytics import YOLO from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS -from ultralytics.utils import ARM64, IS_DOCKER, IS_RASPBERRYPI, LINUX, MACOS, MACOS_VERSION, WINDOWS, checks +from ultralytics.utils import ( + ARM64, + IS_DOCKER, + IS_RASPBERRYPI, + LINUX, + MACOS, + MACOS_VERSION, + WEIGHTS_DIR, + WINDOWS, + checks, +) from ultralytics.utils.export.engine import torch2onnx from ultralytics.utils.torch_utils import ( TORCH_1_10, @@ -273,6 +283,23 @@ def test_export_coreml(isolated_model): assert "You will not be able to run predict()" not in output, "CoreML export has predict() error" +@pytest.mark.skipif(not TORCH_1_11, reason="RTDETR CoreML export requires torch>=1.11") +@pytest.mark.skipif(WINDOWS, reason="CoreML not supported on Windows") +@pytest.mark.skipif(LINUX and ARM64, reason="CoreML not supported on aarch64 Linux") +@pytest.mark.skipif(checks.IS_PYTHON_3_13, reason="CoreML not supported in Python 3.13") +def test_export_coreml_rtdetr(): + """Test RT-DETR export to CoreML format and check for errors.""" + stdout, stderr = io.StringIO(), io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + file = YOLO(WEIGHTS_DIR / "rtdetr-l.pt").export(format="coreml", imgsz=160) + if MACOS: + YOLO(file)(SOURCE, imgsz=160) + + output = stdout.getvalue() + stderr.getvalue() + assert "Error" not in output, f"RTDETR CoreML export produced errors: {output}" + assert "You will not be able to run predict()" not in output, "RTDETR CoreML export has predict() error" + + @pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10, reason="TFLite export requires Python>=3.10") @pytest.mark.skipif(not TORCH_1_13, reason="TFLite export requires torch>=1.13") @pytest.mark.skipif(not LINUX, reason="Test disabled as TF suffers from install conflicts on Windows and macOS") diff --git a/ultralytics/nn/modules/head.py b/ultralytics/nn/modules/head.py index 10296fe1..cb81a0b1 100644 --- a/ultralytics/nn/modules/head.py +++ b/ultralytics/nn/modules/head.py @@ -1604,9 +1604,10 @@ class RTDETRDecoder(nn.Module): cy, w, h, max_class_prob, class_index]. """ scores, index = scores.flatten(1).topk(self.num_queries) - query_idx = index // self.nc - boxes = boxes.gather(dim=1, index=query_idx.unsqueeze(-1).expand(-1, -1, 4)) - return torch.cat([boxes, scores[..., None], (index % self.nc)[..., None].float()], dim=-1) + # CoreML MIL lacks integer floor-div and mod lowering: use torch.div(rounding_mode="floor") and (index - q*nc). + query_idx = torch.div(index, self.nc, rounding_mode="floor") + boxes = boxes.gather(dim=1, index=query_idx.unsqueeze(-1).expand(-1, -1, 4).long()) + return torch.cat([boxes, scores[..., None], (index - query_idx * self.nc)[..., None].float()], dim=-1) @staticmethod def _generate_anchors( diff --git a/ultralytics/nn/modules/transformer.py b/ultralytics/nn/modules/transformer.py index 83e58912..cbf1d9ac 100644 --- a/ultralytics/nn/modules/transformer.py +++ b/ultralytics/nn/modules/transformer.py @@ -234,8 +234,9 @@ class AIFI(TransformerEncoderLayer): omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim omega = 1.0 / (temperature**omega) - out_w = grid_w.flatten()[..., None] @ omega[None] - out_h = grid_h.flatten()[..., None] @ omega[None] + # Pin matmul to fp32 for CoreML export: fp16 sin/cos on integer-derived positions accumulates visible error. + out_w = grid_w.flatten()[..., None].float() @ omega[None] + out_h = grid_h.flatten()[..., None].float() @ omega[None] return torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], 1)[None] @@ -539,8 +540,8 @@ class MSDeformAttn(nn.Module): Args: query (torch.Tensor): Query tensor with shape [bs, query_length, C]. - refer_bbox (torch.Tensor): Reference bounding boxes with shape [bs, query_length, n_levels, 2 or 4], range - in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area. + refer_bbox (torch.Tensor): Reference bounding boxes with shape [bs, query_length, 1, 2 or 4], range in [0, + 1], top-left (0,0), bottom-right (1, 1). The size-1 axis broadcasts across n_levels. value (torch.Tensor): Value tensor with shape [bs, value_length, C]. value_shapes (list): List with shape [n_levels, 2], [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})]. value_mask (torch.Tensor, optional): Mask tensor with shape [bs, value_length], True for padding elements, @@ -560,18 +561,21 @@ class MSDeformAttn(nn.Module): if value_mask is not None: value = value.masked_fill(value_mask[..., None], float(0)) value = value.view(bs, len_v, self.n_heads, self.d_model // self.n_heads) - sampling_offsets = self.sampling_offsets(query).view(bs, len_q, self.n_heads, self.n_levels, self.n_points, 2) - attention_weights = self.attention_weights(query).view(bs, len_q, self.n_heads, self.n_levels * self.n_points) - attention_weights = F.softmax(attention_weights, -1).view(bs, len_q, self.n_heads, self.n_levels, self.n_points) - # N, Len_q, n_heads, n_levels, n_points, 2 + # Fold (n_levels, n_points) into one axis so every traced tensor stays at rank <= 5 (required for CoreML + # export); refer_bbox arrives as (bs, len_q, 1, 2 or 4) and its size-1 axis broadcasts implicitly. + n_total_points = self.n_levels * self.n_points + sampling_offsets = self.sampling_offsets(query).view(bs, len_q, self.n_heads, n_total_points, 2) + attention_weights = self.attention_weights(query).view(bs, len_q, self.n_heads, n_total_points) + attention_weights = F.softmax(attention_weights, -1) num_points = refer_bbox.shape[-1] if num_points == 2: offset_normalizer = torch.as_tensor(value_shapes, dtype=query.dtype, device=query.device).flip(-1) - add = sampling_offsets / offset_normalizer[None, None, None, :, None, :] - sampling_locations = refer_bbox[:, :, None, :, None, :] + add + offset_normalizer = offset_normalizer[:, None, :].expand(-1, self.n_points, -1).reshape(n_total_points, 2) + sampling_locations = refer_bbox[:, :, None, :, :] + sampling_offsets / offset_normalizer elif num_points == 4: - add = sampling_offsets / self.n_points * refer_bbox[:, :, None, :, None, 2:] * 0.5 - sampling_locations = refer_bbox[:, :, None, :, None, :2] + add + sampling_locations = ( + refer_bbox[:, :, None, :, :2] + sampling_offsets / self.n_points * refer_bbox[:, :, None, :, 2:] * 0.5 + ) else: raise ValueError(f"Last dim of reference_points must be 2 or 4, but got {num_points}.") output = multi_scale_deformable_attn_pytorch(value, value_shapes, sampling_locations, attention_weights) diff --git a/ultralytics/nn/modules/utils.py b/ultralytics/nn/modules/utils.py index 491a3096..13eebcc5 100644 --- a/ultralytics/nn/modules/utils.py +++ b/ultralytics/nn/modules/utils.py @@ -100,57 +100,48 @@ def inverse_sigmoid(x, eps=1e-5): def multi_scale_deformable_attn_pytorch( value: torch.Tensor, - value_spatial_shapes: torch.Tensor, + value_spatial_shapes: list, sampling_locations: torch.Tensor, attention_weights: torch.Tensor, ) -> torch.Tensor: """Implement multi-scale deformable attention in PyTorch. - This function performs deformable attention across multiple feature map scales, allowing the model to attend to - different spatial locations with learned offsets. + Folds the (num_levels, num_points) axes into a single num_total_points axis so every traced tensor stays at rank <= + 5, the maximum rank supported by CoreML's MIL converter. Numerically equivalent to the rank-6 reference + implementation on CUDA and CPU. Args: - value (torch.Tensor): The value tensor with shape (bs, num_keys, num_heads, embed_dims). - value_spatial_shapes (torch.Tensor): Spatial shapes of the value tensor with shape (num_levels, 2). - sampling_locations (torch.Tensor): The sampling locations with shape (bs, num_queries, num_heads, num_levels, + value (torch.Tensor): Value tensor with shape (bs, num_keys, num_heads, embed_dims). + value_spatial_shapes (list): Per-level spatial shapes as [(H_0, W_0), ..., (H_{L-1}, W_{L-1})]. + sampling_locations (torch.Tensor): Sampling locations with shape (bs, num_queries, num_heads, num_levels * num_points, 2). - attention_weights (torch.Tensor): The attention weights with shape (bs, num_queries, num_heads, num_levels, + attention_weights (torch.Tensor): Attention weights with shape (bs, num_queries, num_heads, num_levels * num_points). Returns: - (torch.Tensor): The output tensor with shape (bs, num_queries, num_heads * embed_dims). + (torch.Tensor): Output tensor with shape (bs, num_queries, num_heads * embed_dims). References: https://github.com/IDEA-Research/detrex/blob/main/detrex/layers/multi_scale_deform_attn.py """ bs, _, num_heads, embed_dims = value.shape - _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape - value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) - sampling_grids = 2 * sampling_locations - 1 + _, num_queries, _, num_total_points, _ = sampling_locations.shape + num_points = num_total_points // len(value_spatial_shapes) + + # (bs, num_keys, num_heads, embed_dims) -> tuple of (bs*num_heads, embed_dims, H*W) per level + value_list = value.permute(0, 2, 3, 1).flatten(0, 1).split([h * w for h, w in value_spatial_shapes], dim=-1) + # Map to grid_sample coords in [-1, 1] and split per level: tuple of (bs*num_heads, num_queries, num_points, 2) + sampling_grids = (2 * sampling_locations - 1).permute(0, 2, 1, 3, 4).flatten(0, 1).split(num_points, dim=-2) + sampling_value_list = [] - for level, (H_, W_) in enumerate(value_spatial_shapes): - # bs, H_*W_, num_heads, embed_dims -> - # bs, H_*W_, num_heads*embed_dims -> - # bs, num_heads*embed_dims, H_*W_ -> - # bs*num_heads, embed_dims, H_, W_ - value_l_ = value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_) - # bs, num_queries, num_heads, num_points, 2 -> - # bs, num_heads, num_queries, num_points, 2 -> - # bs*num_heads, num_queries, num_points, 2 - sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1) - # bs*num_heads, embed_dims, num_queries, num_points - sampling_value_l_ = F.grid_sample( - value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False + for level, (h, w) in enumerate(value_spatial_shapes): + value_l = value_list[level].reshape(bs * num_heads, embed_dims, h, w) + sampling_value_list.append( + F.grid_sample(value_l, sampling_grids[level], mode="bilinear", padding_mode="zeros", align_corners=False) ) - sampling_value_list.append(sampling_value_l_) - # (bs, num_queries, num_heads, num_levels, num_points) -> - # (bs, num_heads, num_queries, num_levels, num_points) -> - # (bs*num_heads, 1, num_queries, num_levels*num_points) - attention_weights = attention_weights.transpose(1, 2).reshape( - bs * num_heads, 1, num_queries, num_levels * num_points - ) + attention_weights = attention_weights.permute(0, 2, 1, 3).reshape(bs * num_heads, 1, num_queries, num_total_points) output = ( - (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) + (torch.cat(sampling_value_list, dim=-1) * attention_weights) .sum(-1) .view(bs, num_heads * embed_dims, num_queries) ) diff --git a/ultralytics/utils/export/coreml.py b/ultralytics/utils/export/coreml.py index 696207bd..beb2feba 100644 --- a/ultralytics/utils/export/coreml.py +++ b/ultralytics/utils/export/coreml.py @@ -202,13 +202,23 @@ def torch2coreml( # Internally based on the model conversion and output type. # Setting minimum_deployment_target >= iOS16 will require setting compute_precision=ct.precision.FLOAT32. # iOS16 adds in better support for FP16, but none of the CoreML NMS specifications handle FP16 as input. - ct_model = ct.convert( - ts, + convert_kwargs = dict( inputs=inputs, classifier_config=ct.ClassifierConfig(classifier_names) if classifier_names else None, convert_to="neuralnetwork" if mlmodel else "mlprogram", skip_model_load=True, ) + if not mlmodel: + # RT-DETR decoder class logits and deformable-sampling indices drift in fp16; pin those op types to fp32 + # only when an RTDETRDecoder is present. YOLO detect/segment/pose/OBB keep mlprogram's fp16 default. + from ultralytics.nn.modules.head import RTDETRDecoder + + if any(isinstance(m, RTDETRDecoder) for m in model.modules()): + fp32_ops = {"linear", "gather", "gather_nd", "gather_along_axis"} + convert_kwargs["compute_precision"] = ct.transform.FP16ComputePrecision( + op_selector=lambda op: op.op_type not in fp32_ops + ) + ct_model = ct.convert(ts, **convert_kwargs) bits, mode = (8, "kmeans") if int8 else (16, "linear") if half else (32, None) if bits < 32: if "kmeans" in mode: