Onuralp SEZER b73dce8813
refactor: split Exporter export methods into per-format utility modules (#23914)
Signed-off-by: Onuralp SEZER <onuralp@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: Lakshantha Dissanayake <lakshantha@ultralytics.com>
Co-authored-by: Jing Qiu <61612323+Laughing-q@users.noreply.github.com>
Co-authored-by: Laughing-q <1185102784@qq.com>
2026-03-27 18:38:56 +08:00

243 lines
10 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
from __future__ import annotations
from pathlib import Path
from typing import Any
import torch
import torch.nn as nn
from ultralytics.utils import LOGGER
class IOSDetectModel(nn.Module):
"""Wrap an Ultralytics YOLO model for Apple iOS CoreML export."""
def __init__(self, model: nn.Module, im: torch.Tensor, mlprogram: bool = True):
"""Initialize the IOSDetectModel class with a YOLO model and example image.
Args:
model (nn.Module): The YOLO model to wrap.
im (torch.Tensor): Example input tensor with shape (B, C, H, W).
mlprogram (bool): Whether exporting to MLProgram format.
"""
super().__init__()
_, _, h, w = im.shape # batch, channel, height, width
self.model = model
self.nc = len(model.names) # number of classes
self.mlprogram = mlprogram
if w == h:
self.normalize = 1.0 / w # scalar
else:
self.normalize = torch.tensor(
[1.0 / w, 1.0 / h, 1.0 / w, 1.0 / h], # broadcast (slower, smaller)
device=next(model.parameters()).device,
)
def forward(self, x: torch.Tensor):
"""Normalize predictions of object detection model with input size-dependent factors."""
xywh, cls = self.model(x)[0].transpose(0, 1).split((4, self.nc), 1)
if self.mlprogram and self.nc % 80 != 0: # NMS bug https://github.com/ultralytics/ultralytics/issues/22309
pad_length = int(((self.nc + 79) // 80) * 80) - self.nc # pad class length to multiple of 80
cls = torch.nn.functional.pad(cls, (0, pad_length, 0, 0), "constant", 0)
return cls, xywh * self.normalize
def pipeline_coreml(
model: Any,
output_shape: tuple,
metadata: dict,
mlmodel: bool = False,
iou: float = 0.45,
conf: float = 0.25,
agnostic_nms: bool = False,
weights_dir: Path | str | None = None,
prefix: str = "",
):
"""Create CoreML pipeline with NMS for YOLO detection models.
Args:
model: CoreML model.
output_shape (tuple): Output shape tuple from the exporter.
metadata (dict): Model metadata.
mlmodel (bool): Whether the model is an MLModel (vs MLProgram).
iou (float): IoU threshold for NMS.
conf (float): Confidence threshold for NMS.
agnostic_nms (bool): Whether to use class-agnostic NMS.
weights_dir (Path | str | None): Weights directory for MLProgram models.
prefix (str): Prefix for log messages.
Returns:
CoreML pipeline model.
"""
import coremltools as ct
LOGGER.info(f"{prefix} starting pipeline with coremltools {ct.__version__}...")
spec = model.get_spec()
outs = list(iter(spec.description.output))
if mlmodel: # mlmodel doesn't infer shapes automatically
outs[0].type.multiArrayType.shape[:] = output_shape[2], output_shape[1] - 4
outs[1].type.multiArrayType.shape[:] = output_shape[2], 4
names = metadata["names"]
nx = spec.description.input[0].type.imageType.width
ny = spec.description.input[0].type.imageType.height
nc = outs[0].type.multiArrayType.shape[-1]
if len(names) != nc: # Hack fix for MLProgram NMS bug https://github.com/ultralytics/ultralytics/issues/22309
names = {**names, **{i: str(i) for i in range(len(names), nc)}}
model = ct.models.MLModel(spec, weights_dir=weights_dir)
# Create NMS protobuf
nms_spec = ct.proto.Model_pb2.Model()
nms_spec.specificationVersion = spec.specificationVersion
for i in range(len(outs)):
decoder_output = model._spec.description.output[i].SerializeToString()
nms_spec.description.input.add()
nms_spec.description.input[i].ParseFromString(decoder_output)
nms_spec.description.output.add()
nms_spec.description.output[i].ParseFromString(decoder_output)
output_names = ["confidence", "coordinates"]
for i, name in enumerate(output_names):
nms_spec.description.output[i].name = name
for i, out in enumerate(outs):
ma_type = nms_spec.description.output[i].type.multiArrayType
ma_type.shapeRange.sizeRanges.add()
ma_type.shapeRange.sizeRanges[0].lowerBound = 0
ma_type.shapeRange.sizeRanges[0].upperBound = -1
ma_type.shapeRange.sizeRanges.add()
ma_type.shapeRange.sizeRanges[1].lowerBound = out.type.multiArrayType.shape[-1]
ma_type.shapeRange.sizeRanges[1].upperBound = out.type.multiArrayType.shape[-1]
del ma_type.shape[:]
nms = nms_spec.nonMaximumSuppression
nms.confidenceInputFeatureName = outs[0].name # 1x507x80
nms.coordinatesInputFeatureName = outs[1].name # 1x507x4
nms.confidenceOutputFeatureName = output_names[0]
nms.coordinatesOutputFeatureName = output_names[1]
nms.iouThresholdInputFeatureName = "iouThreshold"
nms.confidenceThresholdInputFeatureName = "confidenceThreshold"
nms.iouThreshold = iou
nms.confidenceThreshold = conf
nms.pickTop.perClass = not agnostic_nms
nms.stringClassLabels.vector.extend(names.values())
nms_model = ct.models.MLModel(nms_spec)
# Pipeline models together
pipeline = ct.models.pipeline.Pipeline(
input_features=[
("image", ct.models.datatypes.Array(3, ny, nx)),
("iouThreshold", ct.models.datatypes.Double()),
("confidenceThreshold", ct.models.datatypes.Double()),
],
output_features=output_names,
)
pipeline.add_model(model)
pipeline.add_model(nms_model)
# Correct datatypes
pipeline.spec.description.input[0].ParseFromString(model._spec.description.input[0].SerializeToString())
pipeline.spec.description.output[0].ParseFromString(nms_model._spec.description.output[0].SerializeToString())
pipeline.spec.description.output[1].ParseFromString(nms_model._spec.description.output[1].SerializeToString())
# Update metadata
pipeline.spec.specificationVersion = spec.specificationVersion
pipeline.spec.description.metadata.userDefined.update(
{"IoU threshold": str(nms.iouThreshold), "Confidence threshold": str(nms.confidenceThreshold)}
)
# Save the model
model = ct.models.MLModel(pipeline.spec, weights_dir=weights_dir)
model.input_description["image"] = "Input image"
model.input_description["iouThreshold"] = f"(optional) IoU threshold override (default: {nms.iouThreshold})"
model.input_description["confidenceThreshold"] = (
f"(optional) Confidence threshold override (default: {nms.confidenceThreshold})"
)
model.output_description["confidence"] = 'Boxes × Class confidence (see user-defined metadata "classes")'
model.output_description["coordinates"] = "Boxes × [x, y, width, height] (relative to image size)"
LOGGER.info(f"{prefix} pipeline success")
return model
def torch2coreml(
model: nn.Module,
inputs: list,
im: torch.Tensor,
classifier_names: list[str] | None,
coreml_file: Path | str | None = None,
mlmodel: bool = False,
half: bool = False,
int8: bool = False,
metadata: dict | None = None,
prefix: str = "",
):
"""Export a PyTorch model to CoreML ``.mlpackage`` or ``.mlmodel`` format.
Args:
model (nn.Module): The PyTorch model to export.
inputs (list): CoreML input descriptions for the model.
im (torch.Tensor): Example input tensor for tracing.
classifier_names (list[str] | None): Class names for classifier config, or None if not a classifier.
coreml_file (Path | str | None): Output file path, or None to skip saving.
mlmodel (bool): Whether to export as ``.mlmodel`` (neural network) instead of ``.mlpackage`` (ML program).
half (bool): Whether to quantize to FP16.
int8 (bool): Whether to quantize to INT8.
metadata (dict | None): Metadata to embed in the CoreML model.
prefix (str): Prefix for log messages.
Returns:
(ct.models.MLModel): The converted CoreML model.
"""
import coremltools as ct
LOGGER.info(f"\n{prefix} starting export with coremltools {ct.__version__}...")
ts = torch.jit.trace(model.eval(), im, strict=False) # TorchScript model
# Based on apple's documentation it is better to leave out the minimum_deployment target and let that get set
# Internally based on the model conversion and output type.
# Setting minimum_deployment_target >= iOS16 will require setting compute_precision=ct.precision.FLOAT32.
# iOS16 adds in better support for FP16, but none of the CoreML NMS specifications handle FP16 as input.
ct_model = ct.convert(
ts,
inputs=inputs,
classifier_config=ct.ClassifierConfig(classifier_names) if classifier_names else None,
convert_to="neuralnetwork" if mlmodel else "mlprogram",
)
bits, mode = (8, "kmeans") if int8 else (16, "linear") if half else (32, None)
if bits < 32:
if "kmeans" in mode:
from ultralytics.utils.checks import check_requirements
check_requirements("scikit-learn") # scikit-learn package required for k-means quantization
if mlmodel:
ct_model = ct.models.neural_network.quantization_utils.quantize_weights(ct_model, bits, mode)
elif bits == 8: # mlprogram already quantized to FP16
import coremltools.optimize.coreml as cto
op_config = cto.OpPalettizerConfig(mode="kmeans", nbits=bits, weight_threshold=512)
config = cto.OptimizationConfig(global_config=op_config)
ct_model = cto.palettize_weights(ct_model, config=config)
m = dict(metadata or {}) # copy to avoid mutating original
ct_model.short_description = m.pop("description", "")
ct_model.author = m.pop("author", "")
ct_model.license = m.pop("license", "")
ct_model.version = m.pop("version", "")
ct_model.user_defined_metadata.update({k: str(v) for k, v in m.items()})
if coreml_file is not None:
try:
ct_model.save(str(coreml_file)) # save *.mlpackage
except Exception as e:
LOGGER.warning(
f"{prefix} CoreML export to *.mlpackage failed ({e}), reverting to *.mlmodel export. "
f"Known coremltools Python 3.11 and Windows bugs https://github.com/apple/coremltools/issues/1928."
)
coreml_file = Path(coreml_file).with_suffix(".mlmodel")
ct_model.save(str(coreml_file))
return ct_model