From b5fb207d7af1a52103d0d2f146a3b8d2b7e548da Mon Sep 17 00:00:00 2001 From: Jing Qiu <61612323+Laughing-q@users.noreply.github.com> Date: Tue, 16 Sep 2025 00:16:08 +0800 Subject: [PATCH] `ultralytics 8.3.200` Refactor Sony IMX exports to `imx.py` (#22005) Signed-off-by: Glenn Jocher Co-authored-by: UltralyticsAssistant Co-authored-by: Glenn Jocher Co-authored-by: Francesco Mattioli Co-authored-by: Lakshantha Dissanayake --- docs/en/reference/utils/export.md | 20 -- docs/en/reference/utils/export/__init__.md | 20 ++ docs/en/reference/utils/export/imx.md | 32 ++ docs/en/reference/utils/torch_utils.md | 4 - mkdocs.yml | 4 +- ultralytics/__init__.py | 2 +- ultralytics/engine/exporter.py | 178 +---------- ultralytics/nn/modules/head.py | 6 +- .../utils/{export.py => export/__init__.py} | 6 +- ultralytics/utils/export/imx.py | 289 ++++++++++++++++++ ultralytics/utils/torch_utils.py | 47 --- 11 files changed, 364 insertions(+), 244 deletions(-) delete mode 100644 docs/en/reference/utils/export.md create mode 100644 docs/en/reference/utils/export/__init__.md create mode 100644 docs/en/reference/utils/export/imx.md rename ultralytics/utils/{export.py => export/__init__.py} (99%) create mode 100644 ultralytics/utils/export/imx.py diff --git a/docs/en/reference/utils/export.md b/docs/en/reference/utils/export.md deleted file mode 100644 index f0e22995..00000000 --- a/docs/en/reference/utils/export.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -description: Learn how to export PyTorch models to ONNX and TensorRT formats using Ultralytics utilities. Comprehensive guide for configurations, dynamic shapes, and precision optimizations. -keywords: Ultralytics, YOLO, export, ONNX, TensorRT, PyTorch, model conversion, dynamic shapes, FP16, INT8, machine learning ---- - -# Reference for `ultralytics/utils/export.py` - -!!! note - - This file is available at [https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export.py](https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export.py). If you spot a problem please help fix it by [contributing](https://docs.ultralytics.com/help/contributing/) a [Pull Request](https://github.com/ultralytics/ultralytics/edit/main/ultralytics/utils/export.py) 🛠️. Thank you 🙏! - -
- -## ::: ultralytics.utils.export.export_onnx - -



- -## ::: ultralytics.utils.export.export_engine - -

diff --git a/docs/en/reference/utils/export/__init__.md b/docs/en/reference/utils/export/__init__.md new file mode 100644 index 00000000..5d178ce0 --- /dev/null +++ b/docs/en/reference/utils/export/__init__.md @@ -0,0 +1,20 @@ +--- +description: Learn how to export PyTorch models to ONNX, TensorRT and Sony IMX formats using Ultralytics utilities. Comprehensive guide for configurations, dynamic shapes, and precision optimizations. +keywords: Ultralytics, YOLO, export, ONNX, TensorRT, Sony IMX, PyTorch, model conversion, dynamic shapes, FP16, INT8, machine learning +--- + +# Reference for `ultralytics/utils/export/__init__.py` + +!!! note + + This file is available at [https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/\_\_init\_\_.py](https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/__init__.py). If you spot a problem please help fix it by [contributing](https://docs.ultralytics.com/help/contributing/) a [Pull Request](https://github.com/ultralytics/ultralytics/edit/main/ultralytics/utils/export/__init__.py) 🛠️. Thank you 🙏! + +
+ +## ::: ultralytics.utils.export.torch2onnx + +



+ +## ::: ultralytics.utils.export.onnx2engine + +

diff --git a/docs/en/reference/utils/export/imx.md b/docs/en/reference/utils/export/imx.md new file mode 100644 index 00000000..2aa87e63 --- /dev/null +++ b/docs/en/reference/utils/export/imx.md @@ -0,0 +1,32 @@ +--- +description: Learn how to export PyTorch models to Sony IMX format using Ultralytics utilities. Comprehensive guide for configurations and precision optimizations. +keywords: Ultralytics, YOLO, export, Sony IMX, PyTorch, model conversion, INT8, machine learning, NMS, Detect, Pose +--- + +# Reference for `ultralytics/utils/export/imx.py` + +!!! note + + This file is available at [https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/imx.py](https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/imx.py). If you spot a problem please help fix it by [contributing](https://docs.ultralytics.com/help/contributing/) a [Pull Request](https://github.com/ultralytics/ultralytics/edit/main/ultralytics/utils/export/imx.py) 🛠️. Thank you 🙏! + +
+ +## ::: ultralytics.utils.export.imx.FXModel + +



+ +## ::: ultralytics.utils.export.imx.NMSWrapper + +



+ +## ::: ultralytics.utils.export.imx._inference + +



+ +## ::: ultralytics.utils.export.imx.pose_forward + +



+ +## ::: ultralytics.utils.export.imx.torch2imx + +

diff --git a/docs/en/reference/utils/torch_utils.md b/docs/en/reference/utils/torch_utils.md index d2f2bcc3..aaed4520 100644 --- a/docs/en/reference/utils/torch_utils.md +++ b/docs/en/reference/utils/torch_utils.md @@ -19,10 +19,6 @@ keywords: Ultralytics, torch utils, model optimization, device selection, infere



-## ::: ultralytics.utils.torch_utils.FXModel - -



- ## ::: ultralytics.utils.torch_utils.torch_distributed_zero_first



diff --git a/mkdocs.yml b/mkdocs.yml index ae13ed5b..17aad149 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -685,7 +685,9 @@ nav: - downloads: reference/utils/downloads.md - errors: reference/utils/errors.md - events: reference/utils/events.md - - export: reference/utils/export.md + - export: + - __init__: reference/utils/export/__init__.md + - imx: reference/utils/export/imx.md - files: reference/utils/files.md - git: reference/utils/git.md - instance: reference/utils/instance.md diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index bd2364b6..faa96f33 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license -__version__ = "8.3.199" +__version__ = "8.3.200" import importlib import os diff --git a/ultralytics/engine/exporter.py b/ultralytics/engine/exporter.py index 03515093..5fb73db9 100644 --- a/ultralytics/engine/exporter.py +++ b/ultralytics/engine/exporter.py @@ -106,7 +106,7 @@ from ultralytics.utils.checks import ( is_sudo_available, ) from ultralytics.utils.downloads import attempt_download_asset, get_github_assets, safe_download -from ultralytics.utils.export import export_engine, export_onnx +from ultralytics.utils.export import onnx2engine, torch2imx, torch2onnx from ultralytics.utils.files import file_size, spaces_in_path from ultralytics.utils.metrics import batch_probiou from ultralytics.utils.nms import TorchNMS @@ -409,9 +409,9 @@ class Exporter: model = model.fuse() if imx: - from ultralytics.utils.torch_utils import FXModel + from ultralytics.utils.export.imx import FXModel - model = FXModel(model) + model = FXModel(model, self.imgsz) for m in model.modules(): if isinstance(m, Classify): m.export = True @@ -426,15 +426,6 @@ class Exporter: elif isinstance(m, C2f) and not is_tf_format: # EdgeTPU does not support FlexSplitV while split provides cleaner ONNX graph m.forward = m.forward_split - if isinstance(m, Detect) and imx: - from ultralytics.utils.tal import make_anchors - - m.anchors, m.strides = ( - x.transpose(0, 1) - for x in make_anchors( - torch.cat([s / m.stride.unsqueeze(-1) for s in self.imgsz], dim=1), m.stride, 0.5 - ) - ) y = None for _ in range(2): # dry runs @@ -610,7 +601,7 @@ class Exporter: self.args.opset = opset_version # for NMSModel with arange_patch(self.args): - export_onnx( + torch2onnx( NMSModel(self.model, self.args) if self.args.nms else self.model, self.im, f, @@ -933,7 +924,7 @@ class Exporter: LOGGER.info(f"\n{prefix} starting export with TensorRT {trt.__version__}...") assert Path(f_onnx).exists(), f"failed to export ONNX file: {f_onnx}" f = self.file.with_suffix(".engine") # TensorRT engine file - export_engine( + onnx2engine( f_onnx, f, self.args.workspace, @@ -1169,7 +1160,6 @@ class Exporter: @try_export def export_imx(self, prefix=colorstr("IMX:")): """Export YOLO model to IMX format.""" - gptq = False assert LINUX, ( "export only supported on Linux. " "See https://developer.aitrios.sony-semicon.com/en/raspberrypi-ai-camera/documentation/imx500-converter" @@ -1182,13 +1172,6 @@ class Exporter: check_requirements("imx500-converter[pt]>=3.16.1") # Separate requirements for imx500-converter check_requirements("mct-quantizers>=1.6.0") # Separate for compatibility with model-compression-toolkit - import model_compression_toolkit as mct - import onnx - from edgemdt_tpc import get_target_platform_capabilities - from sony_custom_layers.pytorch import multiclass_nms_with_indices - - LOGGER.info(f"\n{prefix} starting export with model_compression_toolkit {mct.__version__}...") - # Install Java>=17 try: java_output = subprocess.run(["java", "--version"], check=True, capture_output=True).stdout.decode() @@ -1199,150 +1182,17 @@ class Exporter: cmd = (["sudo"] if is_sudo_available() else []) + ["apt", "install", "-y", "openjdk-21-jre"] subprocess.run(cmd, check=True) - def representative_dataset_gen(dataloader=self.get_int8_calibration_dataloader(prefix)): - for batch in dataloader: - img = batch["img"] - img = img / 255.0 - yield [img] - - tpc = get_target_platform_capabilities(tpc_version="4.0", device_type="imx500") - - bit_cfg = mct.core.BitWidthConfig() - if "C2PSA" in self.model.__str__(): # YOLO11 - if self.model.task == "detect": - layer_names = ["sub", "mul_2", "add_14", "cat_21"] - weights_memory = 2585350.2439 - n_layers = 238 # 238 layers for fused YOLO11n - elif self.model.task == "pose": - layer_names = ["sub", "mul_2", "add_14", "cat_22", "cat_23", "mul_4", "add_15"] - weights_memory = 2437771.67 - n_layers = 257 # 257 layers for fused YOLO11n-pose - else: # YOLOv8 - if self.model.task == "detect": - layer_names = ["sub", "mul", "add_6", "cat_17"] - weights_memory = 2550540.8 - n_layers = 168 # 168 layers for fused YOLOv8n - elif self.model.task == "pose": - layer_names = ["add_7", "mul_2", "cat_19", "mul", "sub", "add_6", "cat_18"] - weights_memory = 2482451.85 - n_layers = 187 # 187 layers for fused YOLO11n-pose - - # Check if the model has the expected number of layers - if len(list(self.model.modules())) != n_layers: - raise ValueError("IMX export only supported for YOLOv8n and YOLO11n models.") - - for layer_name in layer_names: - bit_cfg.set_manual_activation_bit_width([mct.core.common.network_editors.NodeNameFilter(layer_name)], 16) - - config = mct.core.CoreConfig( - mixed_precision_config=mct.core.MixedPrecisionQuantizationConfig(num_of_images=10), - quantization_config=mct.core.QuantizationConfig(concat_threshold_update=True), - bit_width_config=bit_cfg, + return torch2imx( + self.model, + self.file, + self.args.conf, + self.args.iou, + self.args.max_det, + metadata=self.metadata, + dataset=self.get_int8_calibration_dataloader(prefix), + prefix=prefix, ) - resource_utilization = mct.core.ResourceUtilization(weights_memory=weights_memory) - - quant_model = ( - mct.gptq.pytorch_gradient_post_training_quantization( # Perform Gradient-Based Post Training Quantization - model=self.model, - representative_data_gen=representative_dataset_gen, - target_resource_utilization=resource_utilization, - gptq_config=mct.gptq.get_pytorch_gptq_config( - n_epochs=1000, use_hessian_based_weights=False, use_hessian_sample_attention=False - ), - core_config=config, - target_platform_capabilities=tpc, - )[0] - if gptq - else mct.ptq.pytorch_post_training_quantization( # Perform post training quantization - in_module=self.model, - representative_data_gen=representative_dataset_gen, - target_resource_utilization=resource_utilization, - core_config=config, - target_platform_capabilities=tpc, - )[0] - ) - - class NMSWrapper(torch.nn.Module): - """Wrap PyTorch Module with multiclass_nms layer from sony_custom_layers.""" - - def __init__( - self, - model: torch.nn.Module, - score_threshold: float = 0.001, - iou_threshold: float = 0.7, - max_detections: int = 300, - task: str = "detect", - ): - """ - Initialize NMSWrapper with PyTorch Module and NMS parameters. - - Args: - model (torch.nn.Module): Model instance. - score_threshold (float): Score threshold for non-maximum suppression. - iou_threshold (float): Intersection over union threshold for non-maximum suppression. - max_detections (int): The number of detections to return. - task (str): Task type, either 'detect' or 'pose'. - """ - super().__init__() - self.model = model - self.score_threshold = score_threshold - self.iou_threshold = iou_threshold - self.max_detections = max_detections - self.task = task - - def forward(self, images): - """Forward pass with model inference and NMS post-processing.""" - # model inference - outputs = self.model(images) - - boxes, scores = outputs[0], outputs[1] - nms_outputs = multiclass_nms_with_indices( - boxes=boxes, - scores=scores, - score_threshold=self.score_threshold, - iou_threshold=self.iou_threshold, - max_detections=self.max_detections, - ) - if self.task == "pose": - kpts = outputs[2] # (bs, max_detections, kpts 17*3) - out_kpts = torch.gather(kpts, 1, nms_outputs.indices.unsqueeze(-1).expand(-1, -1, kpts.size(-1))) - return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, out_kpts - return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, nms_outputs.n_valid - - quant_model = NMSWrapper( - model=quant_model, - score_threshold=self.args.conf or 0.001, - iou_threshold=self.args.iou, - max_detections=self.args.max_det, - task=self.model.task, - ).to(self.device) - - f = Path(str(self.file).replace(self.file.suffix, "_imx_model")) - f.mkdir(exist_ok=True) - onnx_model = f / Path(str(self.file.name).replace(self.file.suffix, "_imx.onnx")) # js dir - mct.exporter.pytorch_export_model( - model=quant_model, save_model_path=onnx_model, repr_dataset=representative_dataset_gen - ) - - model_onnx = onnx.load(onnx_model) # load onnx model - for k, v in self.metadata.items(): - meta = model_onnx.metadata_props.add() - meta.key, meta.value = k, str(v) - - onnx.save(model_onnx, onnx_model) - - subprocess.run( - ["imxconv-pt", "-i", str(onnx_model), "-o", str(f), "--no-input-persistency", "--overwrite-output"], - check=True, - ) - - # Needed for imx models. - with open(f / "labels.txt", "w", encoding="utf-8") as file: - file.writelines([f"{name}\n" for _, name in self.model.names.items()]) - - return f - def _add_tflite_metadata(self, file): """Add metadata to *.tflite models per https://ai.google.dev/edge/litert/models/metadata.""" import zipfile diff --git a/ultralytics/nn/modules/head.py b/ultralytics/nn/modules/head.py index a277aa66..d4c67126 100644 --- a/ultralytics/nn/modules/head.py +++ b/ultralytics/nn/modules/head.py @@ -162,7 +162,7 @@ class Detect(nn.Module): # Inference path shape = x[0].shape # BCHW x_cat = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2) - if self.format != "imx" and (self.dynamic or self.shape != shape): + if self.dynamic or self.shape != shape: self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5)) self.shape = shape @@ -182,8 +182,6 @@ class Detect(nn.Module): dbox = self.decode_bboxes(self.dfl(box) * norm, self.anchors.unsqueeze(0) * norm[:, :2]) else: dbox = self.decode_bboxes(self.dfl(box), self.anchors.unsqueeze(0)) * self.strides - if self.export and self.format == "imx": - return dbox.transpose(1, 2), cls.sigmoid().permute(0, 2, 1) return torch.cat((dbox, cls.sigmoid()), 1) def bias_init(self): @@ -387,8 +385,6 @@ class Pose(Detect): if self.training: return x, kpt pred_kpt = self.kpts_decode(bs, kpt) - if self.export and self.format == "imx": - return (*x, pred_kpt.permute(0, 2, 1)) return torch.cat([x, pred_kpt], 1) if self.export else (torch.cat([x[0], pred_kpt], 1), (x[1], kpt)) def kpts_decode(self, bs: int, kpts: torch.Tensor) -> torch.Tensor: diff --git a/ultralytics/utils/export.py b/ultralytics/utils/export/__init__.py similarity index 99% rename from ultralytics/utils/export.py rename to ultralytics/utils/export/__init__.py index 3c1dde9f..5e028e60 100644 --- a/ultralytics/utils/export.py +++ b/ultralytics/utils/export/__init__.py @@ -9,8 +9,10 @@ import torch from ultralytics.utils import IS_JETSON, LOGGER +from .imx import torch2imx # noqa -def export_onnx( + +def torch2onnx( torch_model: torch.nn.Module, im: torch.Tensor, onnx_file: str, @@ -47,7 +49,7 @@ def export_onnx( ) -def export_engine( +def onnx2engine( onnx_file: str, engine_file: str | None = None, workspace: int | None = None, diff --git a/ultralytics/utils/export/imx.py b/ultralytics/utils/export/imx.py new file mode 100644 index 00000000..a72ea31a --- /dev/null +++ b/ultralytics/utils/export/imx.py @@ -0,0 +1,289 @@ +# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license + +from __future__ import annotations + +import subprocess +import types +from pathlib import Path + +import torch + +from ultralytics.nn.modules import Detect, Pose +from ultralytics.utils import LOGGER +from ultralytics.utils.tal import make_anchors +from ultralytics.utils.torch_utils import copy_attr + + +class FXModel(torch.nn.Module): + """ + A custom model class for torch.fx compatibility. + + This class extends `torch.nn.Module` and is designed to ensure compatibility with torch.fx for tracing and graph + manipulation. It copies attributes from an existing model and explicitly sets the model attribute to ensure proper + copying. + + Attributes: + model (nn.Module): The original model's layers. + """ + + def __init__(self, model, imgsz=(640, 640)): + """ + Initialize the FXModel. + + Args: + model (nn.Module): The original model to wrap for torch.fx compatibility. + imgsz (tuple[int, int]): The input image size (height, width). Default is (640, 640). + """ + super().__init__() + copy_attr(self, model) + # Explicitly set `model` since `copy_attr` somehow does not copy it. + self.model = model.model + self.imgsz = imgsz + + def forward(self, x): + """ + Forward pass through the model. + + This method performs the forward pass through the model, handling the dependencies between layers and saving + intermediate outputs. + + Args: + x (torch.Tensor): The input tensor to the model. + + Returns: + (torch.Tensor): The output tensor from the model. + """ + y = [] # outputs + for m in self.model: + if m.f != -1: # if not from previous layer + # from earlier layers + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] + if isinstance(m, Detect): + m._inference = types.MethodType(_inference, m) # bind method to Detect + m.anchors, m.strides = ( + x.transpose(0, 1) + for x in make_anchors( + torch.cat([s / m.stride.unsqueeze(-1) for s in self.imgsz], dim=1), m.stride, 0.5 + ) + ) + if type(m) is Pose: + m.forward = types.MethodType(pose_forward, m) # bind method to Detect + x = m(x) # run + y.append(x) # save output + return x + + +def _inference(self, x: list[torch.Tensor]) -> tuple[torch.Tensor]: + """Decode boxes and cls scores for imx object detection.""" + x_cat = torch.cat([xi.view(x[0].shape[0], self.no, -1) for xi in x], 2) + box, cls = x_cat.split((self.reg_max * 4, self.nc), 1) + dbox = self.decode_bboxes(self.dfl(box), self.anchors.unsqueeze(0)) * self.strides + return dbox.transpose(1, 2), cls.sigmoid().permute(0, 2, 1) + + +def pose_forward(self, x: list[torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Forward pass for imx pose estimation, including keypoint decoding.""" + bs = x[0].shape[0] # batch size + kpt = torch.cat([self.cv4[i](x[i]).view(bs, self.nk, -1) for i in range(self.nl)], -1) # (bs, 17*3, h*w) + x = Detect.forward(self, x) + pred_kpt = self.kpts_decode(bs, kpt) + return (*x, pred_kpt.permute(0, 2, 1)) + + +class NMSWrapper(torch.nn.Module): + """Wrap PyTorch Module with multiclass_nms layer from sony_custom_layers.""" + + def __init__( + self, + model: torch.nn.Module, + score_threshold: float = 0.001, + iou_threshold: float = 0.7, + max_detections: int = 300, + task: str = "detect", + ): + """ + Initialize NMSWrapper with PyTorch Module and NMS parameters. + + Args: + model (torch.nn.Module): Model instance. + score_threshold (float): Score threshold for non-maximum suppression. + iou_threshold (float): Intersection over union threshold for non-maximum suppression. + max_detections (int): The number of detections to return. + task (str): Task type, either 'detect' or 'pose'. + """ + super().__init__() + self.model = model + self.score_threshold = score_threshold + self.iou_threshold = iou_threshold + self.max_detections = max_detections + self.task = task + + def forward(self, images): + """Forward pass with model inference and NMS post-processing.""" + from sony_custom_layers.pytorch import multiclass_nms_with_indices + + # model inference + outputs = self.model(images) + boxes, scores = outputs[0], outputs[1] + nms_outputs = multiclass_nms_with_indices( + boxes=boxes, + scores=scores, + score_threshold=self.score_threshold, + iou_threshold=self.iou_threshold, + max_detections=self.max_detections, + ) + if self.task == "pose": + kpts = outputs[2] # (bs, max_detections, kpts 17*3) + out_kpts = torch.gather(kpts, 1, nms_outputs.indices.unsqueeze(-1).expand(-1, -1, kpts.size(-1))) + return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, out_kpts + return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, nms_outputs.n_valid + + +def torch2imx( + model: torch.nn.Module, + file: Path | str, + conf: float, + iou: float, + max_det: int, + metadata: dict | None = None, + gptq: bool = False, + dataset=None, + prefix: str = "", +): + """ + Export YOLO model to IMX format for deployment on Sony IMX500 devices. + + This function quantizes a YOLO model using Model Compression Toolkit (MCT) and exports it + to IMX format compatible with Sony IMX500 edge devices. It supports both YOLOv8n and YOLO11n + models for detection and pose estimation tasks. + + Args: + model (torch.nn.Module): The YOLO model to export. Must be YOLOv8n or YOLO11n. + file (Path | str): Output file path for the exported model. + conf (float): Confidence threshold for NMS post-processing. + iou (float): IoU threshold for NMS post-processing. + max_det (int): Maximum number of detections to return. + metadata (dict | None, optional): Metadata to embed in the ONNX model. Defaults to None. + gptq (bool, optional): Whether to use Gradient-Based Post Training Quantization. + If False, uses standard Post Training Quantization. Defaults to False. + dataset (optional): Representative dataset for quantization calibration. Defaults to None. + prefix (str, optional): Logging prefix string. Defaults to "". + + Returns: + f (Path): Path to the exported IMX model directory + + Raises: + ValueError: If the model is not a supported YOLOv8n or YOLO11n variant. + + Example: + >>> from ultralytics import YOLO + >>> model = YOLO("yolo11n.pt") + >>> path, _ = export_imx(model, "model.imx", conf=0.25, iou=0.45, max_det=300) + + Note: + - Requires model_compression_toolkit, onnx, edgemdt_tpc, and sony_custom_layers packages + - Only supports YOLOv8n and YOLO11n models (detection and pose tasks) + - Output includes quantized ONNX model, IMX binary, and labels.txt file + """ + import model_compression_toolkit as mct + import onnx + from edgemdt_tpc import get_target_platform_capabilities + + LOGGER.info(f"\n{prefix} starting export with model_compression_toolkit {mct.__version__}...") + + def representative_dataset_gen(dataloader=dataset): + for batch in dataloader: + img = batch["img"] + img = img / 255.0 + yield [img] + + tpc = get_target_platform_capabilities(tpc_version="4.0", device_type="imx500") + + bit_cfg = mct.core.BitWidthConfig() + if "C2PSA" in model.__str__(): # YOLO11 + if model.task == "detect": + layer_names = ["sub", "mul_2", "add_14", "cat_21"] + weights_memory = 2585350.2439 + n_layers = 238 # 238 layers for fused YOLO11n + elif model.task == "pose": + layer_names = ["sub", "mul_2", "add_14", "cat_22", "cat_23", "mul_4", "add_15"] + weights_memory = 2437771.67 + n_layers = 257 # 257 layers for fused YOLO11n-pose + else: # YOLOv8 + if model.task == "detect": + layer_names = ["sub", "mul", "add_6", "cat_17"] + weights_memory = 2550540.8 + n_layers = 168 # 168 layers for fused YOLOv8n + elif model.task == "pose": + layer_names = ["add_7", "mul_2", "cat_19", "mul", "sub", "add_6", "cat_18"] + weights_memory = 2482451.85 + n_layers = 187 # 187 layers for fused YOLO11n-pose + + # Check if the model has the expected number of layers + if len(list(model.modules())) != n_layers: + raise ValueError("IMX export only supported for YOLOv8n and YOLO11n models.") + + for layer_name in layer_names: + bit_cfg.set_manual_activation_bit_width([mct.core.common.network_editors.NodeNameFilter(layer_name)], 16) + + config = mct.core.CoreConfig( + mixed_precision_config=mct.core.MixedPrecisionQuantizationConfig(num_of_images=10), + quantization_config=mct.core.QuantizationConfig(concat_threshold_update=True), + bit_width_config=bit_cfg, + ) + + resource_utilization = mct.core.ResourceUtilization(weights_memory=weights_memory) + + quant_model = ( + mct.gptq.pytorch_gradient_post_training_quantization( # Perform Gradient-Based Post Training Quantization + model=model, + representative_data_gen=representative_dataset_gen, + target_resource_utilization=resource_utilization, + gptq_config=mct.gptq.get_pytorch_gptq_config( + n_epochs=1000, use_hessian_based_weights=False, use_hessian_sample_attention=False + ), + core_config=config, + target_platform_capabilities=tpc, + )[0] + if gptq + else mct.ptq.pytorch_post_training_quantization( # Perform post training quantization + in_module=model, + representative_data_gen=representative_dataset_gen, + target_resource_utilization=resource_utilization, + core_config=config, + target_platform_capabilities=tpc, + )[0] + ) + + quant_model = NMSWrapper( + model=quant_model, + score_threshold=conf or 0.001, + iou_threshold=iou, + max_detections=max_det, + task=model.task, + ) + + f = Path(str(file).replace(file.suffix, "_imx_model")) + f.mkdir(exist_ok=True) + onnx_model = f / Path(str(file.name).replace(file.suffix, "_imx.onnx")) # js dir + mct.exporter.pytorch_export_model( + model=quant_model, save_model_path=onnx_model, repr_dataset=representative_dataset_gen + ) + + model_onnx = onnx.load(onnx_model) # load onnx model + for k, v in metadata.items(): + meta = model_onnx.metadata_props.add() + meta.key, meta.value = k, str(v) + + onnx.save(model_onnx, onnx_model) + + subprocess.run( + ["imxconv-pt", "-i", str(onnx_model), "-o", str(f), "--no-input-persistency", "--overwrite-output"], + check=True, + ) + + # Needed for imx models. + with open(f / "labels.txt", "w", encoding="utf-8") as file: + file.writelines([f"{name}\n" for _, name in model.names.items()]) + + return f diff --git a/ultralytics/utils/torch_utils.py b/ultralytics/utils/torch_utils.py index dc5a5e90..19860d8f 100644 --- a/ultralytics/utils/torch_utils.py +++ b/ultralytics/utils/torch_utils.py @@ -959,53 +959,6 @@ class EarlyStopping: return stop -class FXModel(nn.Module): - """ - A custom model class for torch.fx compatibility. - - This class extends `torch.nn.Module` and is designed to ensure compatibility with torch.fx for tracing and graph - manipulation. It copies attributes from an existing model and explicitly sets the model attribute to ensure proper - copying. - - Attributes: - model (nn.Module): The original model's layers. - """ - - def __init__(self, model): - """ - Initialize the FXModel. - - Args: - model (nn.Module): The original model to wrap for torch.fx compatibility. - """ - super().__init__() - copy_attr(self, model) - # Explicitly set `model` since `copy_attr` somehow does not copy it. - self.model = model.model - - def forward(self, x): - """ - Forward pass through the model. - - This method performs the forward pass through the model, handling the dependencies between layers and saving - intermediate outputs. - - Args: - x (torch.Tensor): The input tensor to the model. - - Returns: - (torch.Tensor): The output tensor from the model. - """ - y = [] # outputs - for m in self.model: - if m.f != -1: # if not from previous layer - # from earlier layers - x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] - x = m(x) # run - y.append(x) # save output - return x - - def attempt_compile( model: torch.nn.Module, device: torch.device,