ultralytics 8.3.200 Refactor Sony IMX exports to imx.py (#22005)
Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> Co-authored-by: UltralyticsAssistant <web@ultralytics.com> Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com> Co-authored-by: Francesco Mattioli <Francesco.mttl@gmail.com> Co-authored-by: Lakshantha Dissanayake <lakshantha@ultralytics.com>
This commit is contained in:
parent
8691162d3f
commit
b5fb207d7a
@ -1,20 +0,0 @@
|
||||
---
|
||||
description: Learn how to export PyTorch models to ONNX and TensorRT formats using Ultralytics utilities. Comprehensive guide for configurations, dynamic shapes, and precision optimizations.
|
||||
keywords: Ultralytics, YOLO, export, ONNX, TensorRT, PyTorch, model conversion, dynamic shapes, FP16, INT8, machine learning
|
||||
---
|
||||
|
||||
# Reference for `ultralytics/utils/export.py`
|
||||
|
||||
!!! note
|
||||
|
||||
This file is available at [https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export.py](https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export.py). If you spot a problem please help fix it by [contributing](https://docs.ultralytics.com/help/contributing/) a [Pull Request](https://github.com/ultralytics/ultralytics/edit/main/ultralytics/utils/export.py) 🛠️. Thank you 🙏!
|
||||
|
||||
<br>
|
||||
|
||||
## ::: ultralytics.utils.export.export_onnx
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.export.export_engine
|
||||
|
||||
<br><br>
|
||||
20
docs/en/reference/utils/export/__init__.md
Normal file
20
docs/en/reference/utils/export/__init__.md
Normal file
@ -0,0 +1,20 @@
|
||||
---
|
||||
description: Learn how to export PyTorch models to ONNX, TensorRT and Sony IMX formats using Ultralytics utilities. Comprehensive guide for configurations, dynamic shapes, and precision optimizations.
|
||||
keywords: Ultralytics, YOLO, export, ONNX, TensorRT, Sony IMX, PyTorch, model conversion, dynamic shapes, FP16, INT8, machine learning
|
||||
---
|
||||
|
||||
# Reference for `ultralytics/utils/export/__init__.py`
|
||||
|
||||
!!! note
|
||||
|
||||
This file is available at [https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/\_\_init\_\_.py](https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/__init__.py). If you spot a problem please help fix it by [contributing](https://docs.ultralytics.com/help/contributing/) a [Pull Request](https://github.com/ultralytics/ultralytics/edit/main/ultralytics/utils/export/__init__.py) 🛠️. Thank you 🙏!
|
||||
|
||||
<br>
|
||||
|
||||
## ::: ultralytics.utils.export.torch2onnx
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.export.onnx2engine
|
||||
|
||||
<br><br>
|
||||
32
docs/en/reference/utils/export/imx.md
Normal file
32
docs/en/reference/utils/export/imx.md
Normal file
@ -0,0 +1,32 @@
|
||||
---
|
||||
description: Learn how to export PyTorch models to Sony IMX format using Ultralytics utilities. Comprehensive guide for configurations and precision optimizations.
|
||||
keywords: Ultralytics, YOLO, export, Sony IMX, PyTorch, model conversion, INT8, machine learning, NMS, Detect, Pose
|
||||
---
|
||||
|
||||
# Reference for `ultralytics/utils/export/imx.py`
|
||||
|
||||
!!! note
|
||||
|
||||
This file is available at [https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/imx.py](https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/export/imx.py). If you spot a problem please help fix it by [contributing](https://docs.ultralytics.com/help/contributing/) a [Pull Request](https://github.com/ultralytics/ultralytics/edit/main/ultralytics/utils/export/imx.py) 🛠️. Thank you 🙏!
|
||||
|
||||
<br>
|
||||
|
||||
## ::: ultralytics.utils.export.imx.FXModel
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.export.imx.NMSWrapper
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.export.imx._inference
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.export.imx.pose_forward
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.export.imx.torch2imx
|
||||
|
||||
<br><br>
|
||||
@ -19,10 +19,6 @@ keywords: Ultralytics, torch utils, model optimization, device selection, infere
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.torch_utils.FXModel
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.torch_utils.torch_distributed_zero_first
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
@ -685,7 +685,9 @@ nav:
|
||||
- downloads: reference/utils/downloads.md
|
||||
- errors: reference/utils/errors.md
|
||||
- events: reference/utils/events.md
|
||||
- export: reference/utils/export.md
|
||||
- export:
|
||||
- __init__: reference/utils/export/__init__.md
|
||||
- imx: reference/utils/export/imx.md
|
||||
- files: reference/utils/files.md
|
||||
- git: reference/utils/git.md
|
||||
- instance: reference/utils/instance.md
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
__version__ = "8.3.199"
|
||||
__version__ = "8.3.200"
|
||||
|
||||
import importlib
|
||||
import os
|
||||
|
||||
@ -106,7 +106,7 @@ from ultralytics.utils.checks import (
|
||||
is_sudo_available,
|
||||
)
|
||||
from ultralytics.utils.downloads import attempt_download_asset, get_github_assets, safe_download
|
||||
from ultralytics.utils.export import export_engine, export_onnx
|
||||
from ultralytics.utils.export import onnx2engine, torch2imx, torch2onnx
|
||||
from ultralytics.utils.files import file_size, spaces_in_path
|
||||
from ultralytics.utils.metrics import batch_probiou
|
||||
from ultralytics.utils.nms import TorchNMS
|
||||
@ -409,9 +409,9 @@ class Exporter:
|
||||
model = model.fuse()
|
||||
|
||||
if imx:
|
||||
from ultralytics.utils.torch_utils import FXModel
|
||||
from ultralytics.utils.export.imx import FXModel
|
||||
|
||||
model = FXModel(model)
|
||||
model = FXModel(model, self.imgsz)
|
||||
for m in model.modules():
|
||||
if isinstance(m, Classify):
|
||||
m.export = True
|
||||
@ -426,15 +426,6 @@ class Exporter:
|
||||
elif isinstance(m, C2f) and not is_tf_format:
|
||||
# EdgeTPU does not support FlexSplitV while split provides cleaner ONNX graph
|
||||
m.forward = m.forward_split
|
||||
if isinstance(m, Detect) and imx:
|
||||
from ultralytics.utils.tal import make_anchors
|
||||
|
||||
m.anchors, m.strides = (
|
||||
x.transpose(0, 1)
|
||||
for x in make_anchors(
|
||||
torch.cat([s / m.stride.unsqueeze(-1) for s in self.imgsz], dim=1), m.stride, 0.5
|
||||
)
|
||||
)
|
||||
|
||||
y = None
|
||||
for _ in range(2): # dry runs
|
||||
@ -610,7 +601,7 @@ class Exporter:
|
||||
self.args.opset = opset_version # for NMSModel
|
||||
|
||||
with arange_patch(self.args):
|
||||
export_onnx(
|
||||
torch2onnx(
|
||||
NMSModel(self.model, self.args) if self.args.nms else self.model,
|
||||
self.im,
|
||||
f,
|
||||
@ -933,7 +924,7 @@ class Exporter:
|
||||
LOGGER.info(f"\n{prefix} starting export with TensorRT {trt.__version__}...")
|
||||
assert Path(f_onnx).exists(), f"failed to export ONNX file: {f_onnx}"
|
||||
f = self.file.with_suffix(".engine") # TensorRT engine file
|
||||
export_engine(
|
||||
onnx2engine(
|
||||
f_onnx,
|
||||
f,
|
||||
self.args.workspace,
|
||||
@ -1169,7 +1160,6 @@ class Exporter:
|
||||
@try_export
|
||||
def export_imx(self, prefix=colorstr("IMX:")):
|
||||
"""Export YOLO model to IMX format."""
|
||||
gptq = False
|
||||
assert LINUX, (
|
||||
"export only supported on Linux. "
|
||||
"See https://developer.aitrios.sony-semicon.com/en/raspberrypi-ai-camera/documentation/imx500-converter"
|
||||
@ -1182,13 +1172,6 @@ class Exporter:
|
||||
check_requirements("imx500-converter[pt]>=3.16.1") # Separate requirements for imx500-converter
|
||||
check_requirements("mct-quantizers>=1.6.0") # Separate for compatibility with model-compression-toolkit
|
||||
|
||||
import model_compression_toolkit as mct
|
||||
import onnx
|
||||
from edgemdt_tpc import get_target_platform_capabilities
|
||||
from sony_custom_layers.pytorch import multiclass_nms_with_indices
|
||||
|
||||
LOGGER.info(f"\n{prefix} starting export with model_compression_toolkit {mct.__version__}...")
|
||||
|
||||
# Install Java>=17
|
||||
try:
|
||||
java_output = subprocess.run(["java", "--version"], check=True, capture_output=True).stdout.decode()
|
||||
@ -1199,150 +1182,17 @@ class Exporter:
|
||||
cmd = (["sudo"] if is_sudo_available() else []) + ["apt", "install", "-y", "openjdk-21-jre"]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
def representative_dataset_gen(dataloader=self.get_int8_calibration_dataloader(prefix)):
|
||||
for batch in dataloader:
|
||||
img = batch["img"]
|
||||
img = img / 255.0
|
||||
yield [img]
|
||||
|
||||
tpc = get_target_platform_capabilities(tpc_version="4.0", device_type="imx500")
|
||||
|
||||
bit_cfg = mct.core.BitWidthConfig()
|
||||
if "C2PSA" in self.model.__str__(): # YOLO11
|
||||
if self.model.task == "detect":
|
||||
layer_names = ["sub", "mul_2", "add_14", "cat_21"]
|
||||
weights_memory = 2585350.2439
|
||||
n_layers = 238 # 238 layers for fused YOLO11n
|
||||
elif self.model.task == "pose":
|
||||
layer_names = ["sub", "mul_2", "add_14", "cat_22", "cat_23", "mul_4", "add_15"]
|
||||
weights_memory = 2437771.67
|
||||
n_layers = 257 # 257 layers for fused YOLO11n-pose
|
||||
else: # YOLOv8
|
||||
if self.model.task == "detect":
|
||||
layer_names = ["sub", "mul", "add_6", "cat_17"]
|
||||
weights_memory = 2550540.8
|
||||
n_layers = 168 # 168 layers for fused YOLOv8n
|
||||
elif self.model.task == "pose":
|
||||
layer_names = ["add_7", "mul_2", "cat_19", "mul", "sub", "add_6", "cat_18"]
|
||||
weights_memory = 2482451.85
|
||||
n_layers = 187 # 187 layers for fused YOLO11n-pose
|
||||
|
||||
# Check if the model has the expected number of layers
|
||||
if len(list(self.model.modules())) != n_layers:
|
||||
raise ValueError("IMX export only supported for YOLOv8n and YOLO11n models.")
|
||||
|
||||
for layer_name in layer_names:
|
||||
bit_cfg.set_manual_activation_bit_width([mct.core.common.network_editors.NodeNameFilter(layer_name)], 16)
|
||||
|
||||
config = mct.core.CoreConfig(
|
||||
mixed_precision_config=mct.core.MixedPrecisionQuantizationConfig(num_of_images=10),
|
||||
quantization_config=mct.core.QuantizationConfig(concat_threshold_update=True),
|
||||
bit_width_config=bit_cfg,
|
||||
return torch2imx(
|
||||
self.model,
|
||||
self.file,
|
||||
self.args.conf,
|
||||
self.args.iou,
|
||||
self.args.max_det,
|
||||
metadata=self.metadata,
|
||||
dataset=self.get_int8_calibration_dataloader(prefix),
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
resource_utilization = mct.core.ResourceUtilization(weights_memory=weights_memory)
|
||||
|
||||
quant_model = (
|
||||
mct.gptq.pytorch_gradient_post_training_quantization( # Perform Gradient-Based Post Training Quantization
|
||||
model=self.model,
|
||||
representative_data_gen=representative_dataset_gen,
|
||||
target_resource_utilization=resource_utilization,
|
||||
gptq_config=mct.gptq.get_pytorch_gptq_config(
|
||||
n_epochs=1000, use_hessian_based_weights=False, use_hessian_sample_attention=False
|
||||
),
|
||||
core_config=config,
|
||||
target_platform_capabilities=tpc,
|
||||
)[0]
|
||||
if gptq
|
||||
else mct.ptq.pytorch_post_training_quantization( # Perform post training quantization
|
||||
in_module=self.model,
|
||||
representative_data_gen=representative_dataset_gen,
|
||||
target_resource_utilization=resource_utilization,
|
||||
core_config=config,
|
||||
target_platform_capabilities=tpc,
|
||||
)[0]
|
||||
)
|
||||
|
||||
class NMSWrapper(torch.nn.Module):
|
||||
"""Wrap PyTorch Module with multiclass_nms layer from sony_custom_layers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: torch.nn.Module,
|
||||
score_threshold: float = 0.001,
|
||||
iou_threshold: float = 0.7,
|
||||
max_detections: int = 300,
|
||||
task: str = "detect",
|
||||
):
|
||||
"""
|
||||
Initialize NMSWrapper with PyTorch Module and NMS parameters.
|
||||
|
||||
Args:
|
||||
model (torch.nn.Module): Model instance.
|
||||
score_threshold (float): Score threshold for non-maximum suppression.
|
||||
iou_threshold (float): Intersection over union threshold for non-maximum suppression.
|
||||
max_detections (int): The number of detections to return.
|
||||
task (str): Task type, either 'detect' or 'pose'.
|
||||
"""
|
||||
super().__init__()
|
||||
self.model = model
|
||||
self.score_threshold = score_threshold
|
||||
self.iou_threshold = iou_threshold
|
||||
self.max_detections = max_detections
|
||||
self.task = task
|
||||
|
||||
def forward(self, images):
|
||||
"""Forward pass with model inference and NMS post-processing."""
|
||||
# model inference
|
||||
outputs = self.model(images)
|
||||
|
||||
boxes, scores = outputs[0], outputs[1]
|
||||
nms_outputs = multiclass_nms_with_indices(
|
||||
boxes=boxes,
|
||||
scores=scores,
|
||||
score_threshold=self.score_threshold,
|
||||
iou_threshold=self.iou_threshold,
|
||||
max_detections=self.max_detections,
|
||||
)
|
||||
if self.task == "pose":
|
||||
kpts = outputs[2] # (bs, max_detections, kpts 17*3)
|
||||
out_kpts = torch.gather(kpts, 1, nms_outputs.indices.unsqueeze(-1).expand(-1, -1, kpts.size(-1)))
|
||||
return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, out_kpts
|
||||
return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, nms_outputs.n_valid
|
||||
|
||||
quant_model = NMSWrapper(
|
||||
model=quant_model,
|
||||
score_threshold=self.args.conf or 0.001,
|
||||
iou_threshold=self.args.iou,
|
||||
max_detections=self.args.max_det,
|
||||
task=self.model.task,
|
||||
).to(self.device)
|
||||
|
||||
f = Path(str(self.file).replace(self.file.suffix, "_imx_model"))
|
||||
f.mkdir(exist_ok=True)
|
||||
onnx_model = f / Path(str(self.file.name).replace(self.file.suffix, "_imx.onnx")) # js dir
|
||||
mct.exporter.pytorch_export_model(
|
||||
model=quant_model, save_model_path=onnx_model, repr_dataset=representative_dataset_gen
|
||||
)
|
||||
|
||||
model_onnx = onnx.load(onnx_model) # load onnx model
|
||||
for k, v in self.metadata.items():
|
||||
meta = model_onnx.metadata_props.add()
|
||||
meta.key, meta.value = k, str(v)
|
||||
|
||||
onnx.save(model_onnx, onnx_model)
|
||||
|
||||
subprocess.run(
|
||||
["imxconv-pt", "-i", str(onnx_model), "-o", str(f), "--no-input-persistency", "--overwrite-output"],
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Needed for imx models.
|
||||
with open(f / "labels.txt", "w", encoding="utf-8") as file:
|
||||
file.writelines([f"{name}\n" for _, name in self.model.names.items()])
|
||||
|
||||
return f
|
||||
|
||||
def _add_tflite_metadata(self, file):
|
||||
"""Add metadata to *.tflite models per https://ai.google.dev/edge/litert/models/metadata."""
|
||||
import zipfile
|
||||
|
||||
@ -162,7 +162,7 @@ class Detect(nn.Module):
|
||||
# Inference path
|
||||
shape = x[0].shape # BCHW
|
||||
x_cat = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2)
|
||||
if self.format != "imx" and (self.dynamic or self.shape != shape):
|
||||
if self.dynamic or self.shape != shape:
|
||||
self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5))
|
||||
self.shape = shape
|
||||
|
||||
@ -182,8 +182,6 @@ class Detect(nn.Module):
|
||||
dbox = self.decode_bboxes(self.dfl(box) * norm, self.anchors.unsqueeze(0) * norm[:, :2])
|
||||
else:
|
||||
dbox = self.decode_bboxes(self.dfl(box), self.anchors.unsqueeze(0)) * self.strides
|
||||
if self.export and self.format == "imx":
|
||||
return dbox.transpose(1, 2), cls.sigmoid().permute(0, 2, 1)
|
||||
return torch.cat((dbox, cls.sigmoid()), 1)
|
||||
|
||||
def bias_init(self):
|
||||
@ -387,8 +385,6 @@ class Pose(Detect):
|
||||
if self.training:
|
||||
return x, kpt
|
||||
pred_kpt = self.kpts_decode(bs, kpt)
|
||||
if self.export and self.format == "imx":
|
||||
return (*x, pred_kpt.permute(0, 2, 1))
|
||||
return torch.cat([x, pred_kpt], 1) if self.export else (torch.cat([x[0], pred_kpt], 1), (x[1], kpt))
|
||||
|
||||
def kpts_decode(self, bs: int, kpts: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@ -9,8 +9,10 @@ import torch
|
||||
|
||||
from ultralytics.utils import IS_JETSON, LOGGER
|
||||
|
||||
from .imx import torch2imx # noqa
|
||||
|
||||
def export_onnx(
|
||||
|
||||
def torch2onnx(
|
||||
torch_model: torch.nn.Module,
|
||||
im: torch.Tensor,
|
||||
onnx_file: str,
|
||||
@ -47,7 +49,7 @@ def export_onnx(
|
||||
)
|
||||
|
||||
|
||||
def export_engine(
|
||||
def onnx2engine(
|
||||
onnx_file: str,
|
||||
engine_file: str | None = None,
|
||||
workspace: int | None = None,
|
||||
289
ultralytics/utils/export/imx.py
Normal file
289
ultralytics/utils/export/imx.py
Normal file
@ -0,0 +1,289 @@
|
||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from ultralytics.nn.modules import Detect, Pose
|
||||
from ultralytics.utils import LOGGER
|
||||
from ultralytics.utils.tal import make_anchors
|
||||
from ultralytics.utils.torch_utils import copy_attr
|
||||
|
||||
|
||||
class FXModel(torch.nn.Module):
|
||||
"""
|
||||
A custom model class for torch.fx compatibility.
|
||||
|
||||
This class extends `torch.nn.Module` and is designed to ensure compatibility with torch.fx for tracing and graph
|
||||
manipulation. It copies attributes from an existing model and explicitly sets the model attribute to ensure proper
|
||||
copying.
|
||||
|
||||
Attributes:
|
||||
model (nn.Module): The original model's layers.
|
||||
"""
|
||||
|
||||
def __init__(self, model, imgsz=(640, 640)):
|
||||
"""
|
||||
Initialize the FXModel.
|
||||
|
||||
Args:
|
||||
model (nn.Module): The original model to wrap for torch.fx compatibility.
|
||||
imgsz (tuple[int, int]): The input image size (height, width). Default is (640, 640).
|
||||
"""
|
||||
super().__init__()
|
||||
copy_attr(self, model)
|
||||
# Explicitly set `model` since `copy_attr` somehow does not copy it.
|
||||
self.model = model.model
|
||||
self.imgsz = imgsz
|
||||
|
||||
def forward(self, x):
|
||||
"""
|
||||
Forward pass through the model.
|
||||
|
||||
This method performs the forward pass through the model, handling the dependencies between layers and saving
|
||||
intermediate outputs.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): The input tensor to the model.
|
||||
|
||||
Returns:
|
||||
(torch.Tensor): The output tensor from the model.
|
||||
"""
|
||||
y = [] # outputs
|
||||
for m in self.model:
|
||||
if m.f != -1: # if not from previous layer
|
||||
# from earlier layers
|
||||
x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f]
|
||||
if isinstance(m, Detect):
|
||||
m._inference = types.MethodType(_inference, m) # bind method to Detect
|
||||
m.anchors, m.strides = (
|
||||
x.transpose(0, 1)
|
||||
for x in make_anchors(
|
||||
torch.cat([s / m.stride.unsqueeze(-1) for s in self.imgsz], dim=1), m.stride, 0.5
|
||||
)
|
||||
)
|
||||
if type(m) is Pose:
|
||||
m.forward = types.MethodType(pose_forward, m) # bind method to Detect
|
||||
x = m(x) # run
|
||||
y.append(x) # save output
|
||||
return x
|
||||
|
||||
|
||||
def _inference(self, x: list[torch.Tensor]) -> tuple[torch.Tensor]:
|
||||
"""Decode boxes and cls scores for imx object detection."""
|
||||
x_cat = torch.cat([xi.view(x[0].shape[0], self.no, -1) for xi in x], 2)
|
||||
box, cls = x_cat.split((self.reg_max * 4, self.nc), 1)
|
||||
dbox = self.decode_bboxes(self.dfl(box), self.anchors.unsqueeze(0)) * self.strides
|
||||
return dbox.transpose(1, 2), cls.sigmoid().permute(0, 2, 1)
|
||||
|
||||
|
||||
def pose_forward(self, x: list[torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Forward pass for imx pose estimation, including keypoint decoding."""
|
||||
bs = x[0].shape[0] # batch size
|
||||
kpt = torch.cat([self.cv4[i](x[i]).view(bs, self.nk, -1) for i in range(self.nl)], -1) # (bs, 17*3, h*w)
|
||||
x = Detect.forward(self, x)
|
||||
pred_kpt = self.kpts_decode(bs, kpt)
|
||||
return (*x, pred_kpt.permute(0, 2, 1))
|
||||
|
||||
|
||||
class NMSWrapper(torch.nn.Module):
|
||||
"""Wrap PyTorch Module with multiclass_nms layer from sony_custom_layers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: torch.nn.Module,
|
||||
score_threshold: float = 0.001,
|
||||
iou_threshold: float = 0.7,
|
||||
max_detections: int = 300,
|
||||
task: str = "detect",
|
||||
):
|
||||
"""
|
||||
Initialize NMSWrapper with PyTorch Module and NMS parameters.
|
||||
|
||||
Args:
|
||||
model (torch.nn.Module): Model instance.
|
||||
score_threshold (float): Score threshold for non-maximum suppression.
|
||||
iou_threshold (float): Intersection over union threshold for non-maximum suppression.
|
||||
max_detections (int): The number of detections to return.
|
||||
task (str): Task type, either 'detect' or 'pose'.
|
||||
"""
|
||||
super().__init__()
|
||||
self.model = model
|
||||
self.score_threshold = score_threshold
|
||||
self.iou_threshold = iou_threshold
|
||||
self.max_detections = max_detections
|
||||
self.task = task
|
||||
|
||||
def forward(self, images):
|
||||
"""Forward pass with model inference and NMS post-processing."""
|
||||
from sony_custom_layers.pytorch import multiclass_nms_with_indices
|
||||
|
||||
# model inference
|
||||
outputs = self.model(images)
|
||||
boxes, scores = outputs[0], outputs[1]
|
||||
nms_outputs = multiclass_nms_with_indices(
|
||||
boxes=boxes,
|
||||
scores=scores,
|
||||
score_threshold=self.score_threshold,
|
||||
iou_threshold=self.iou_threshold,
|
||||
max_detections=self.max_detections,
|
||||
)
|
||||
if self.task == "pose":
|
||||
kpts = outputs[2] # (bs, max_detections, kpts 17*3)
|
||||
out_kpts = torch.gather(kpts, 1, nms_outputs.indices.unsqueeze(-1).expand(-1, -1, kpts.size(-1)))
|
||||
return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, out_kpts
|
||||
return nms_outputs.boxes, nms_outputs.scores, nms_outputs.labels, nms_outputs.n_valid
|
||||
|
||||
|
||||
def torch2imx(
|
||||
model: torch.nn.Module,
|
||||
file: Path | str,
|
||||
conf: float,
|
||||
iou: float,
|
||||
max_det: int,
|
||||
metadata: dict | None = None,
|
||||
gptq: bool = False,
|
||||
dataset=None,
|
||||
prefix: str = "",
|
||||
):
|
||||
"""
|
||||
Export YOLO model to IMX format for deployment on Sony IMX500 devices.
|
||||
|
||||
This function quantizes a YOLO model using Model Compression Toolkit (MCT) and exports it
|
||||
to IMX format compatible with Sony IMX500 edge devices. It supports both YOLOv8n and YOLO11n
|
||||
models for detection and pose estimation tasks.
|
||||
|
||||
Args:
|
||||
model (torch.nn.Module): The YOLO model to export. Must be YOLOv8n or YOLO11n.
|
||||
file (Path | str): Output file path for the exported model.
|
||||
conf (float): Confidence threshold for NMS post-processing.
|
||||
iou (float): IoU threshold for NMS post-processing.
|
||||
max_det (int): Maximum number of detections to return.
|
||||
metadata (dict | None, optional): Metadata to embed in the ONNX model. Defaults to None.
|
||||
gptq (bool, optional): Whether to use Gradient-Based Post Training Quantization.
|
||||
If False, uses standard Post Training Quantization. Defaults to False.
|
||||
dataset (optional): Representative dataset for quantization calibration. Defaults to None.
|
||||
prefix (str, optional): Logging prefix string. Defaults to "".
|
||||
|
||||
Returns:
|
||||
f (Path): Path to the exported IMX model directory
|
||||
|
||||
Raises:
|
||||
ValueError: If the model is not a supported YOLOv8n or YOLO11n variant.
|
||||
|
||||
Example:
|
||||
>>> from ultralytics import YOLO
|
||||
>>> model = YOLO("yolo11n.pt")
|
||||
>>> path, _ = export_imx(model, "model.imx", conf=0.25, iou=0.45, max_det=300)
|
||||
|
||||
Note:
|
||||
- Requires model_compression_toolkit, onnx, edgemdt_tpc, and sony_custom_layers packages
|
||||
- Only supports YOLOv8n and YOLO11n models (detection and pose tasks)
|
||||
- Output includes quantized ONNX model, IMX binary, and labels.txt file
|
||||
"""
|
||||
import model_compression_toolkit as mct
|
||||
import onnx
|
||||
from edgemdt_tpc import get_target_platform_capabilities
|
||||
|
||||
LOGGER.info(f"\n{prefix} starting export with model_compression_toolkit {mct.__version__}...")
|
||||
|
||||
def representative_dataset_gen(dataloader=dataset):
|
||||
for batch in dataloader:
|
||||
img = batch["img"]
|
||||
img = img / 255.0
|
||||
yield [img]
|
||||
|
||||
tpc = get_target_platform_capabilities(tpc_version="4.0", device_type="imx500")
|
||||
|
||||
bit_cfg = mct.core.BitWidthConfig()
|
||||
if "C2PSA" in model.__str__(): # YOLO11
|
||||
if model.task == "detect":
|
||||
layer_names = ["sub", "mul_2", "add_14", "cat_21"]
|
||||
weights_memory = 2585350.2439
|
||||
n_layers = 238 # 238 layers for fused YOLO11n
|
||||
elif model.task == "pose":
|
||||
layer_names = ["sub", "mul_2", "add_14", "cat_22", "cat_23", "mul_4", "add_15"]
|
||||
weights_memory = 2437771.67
|
||||
n_layers = 257 # 257 layers for fused YOLO11n-pose
|
||||
else: # YOLOv8
|
||||
if model.task == "detect":
|
||||
layer_names = ["sub", "mul", "add_6", "cat_17"]
|
||||
weights_memory = 2550540.8
|
||||
n_layers = 168 # 168 layers for fused YOLOv8n
|
||||
elif model.task == "pose":
|
||||
layer_names = ["add_7", "mul_2", "cat_19", "mul", "sub", "add_6", "cat_18"]
|
||||
weights_memory = 2482451.85
|
||||
n_layers = 187 # 187 layers for fused YOLO11n-pose
|
||||
|
||||
# Check if the model has the expected number of layers
|
||||
if len(list(model.modules())) != n_layers:
|
||||
raise ValueError("IMX export only supported for YOLOv8n and YOLO11n models.")
|
||||
|
||||
for layer_name in layer_names:
|
||||
bit_cfg.set_manual_activation_bit_width([mct.core.common.network_editors.NodeNameFilter(layer_name)], 16)
|
||||
|
||||
config = mct.core.CoreConfig(
|
||||
mixed_precision_config=mct.core.MixedPrecisionQuantizationConfig(num_of_images=10),
|
||||
quantization_config=mct.core.QuantizationConfig(concat_threshold_update=True),
|
||||
bit_width_config=bit_cfg,
|
||||
)
|
||||
|
||||
resource_utilization = mct.core.ResourceUtilization(weights_memory=weights_memory)
|
||||
|
||||
quant_model = (
|
||||
mct.gptq.pytorch_gradient_post_training_quantization( # Perform Gradient-Based Post Training Quantization
|
||||
model=model,
|
||||
representative_data_gen=representative_dataset_gen,
|
||||
target_resource_utilization=resource_utilization,
|
||||
gptq_config=mct.gptq.get_pytorch_gptq_config(
|
||||
n_epochs=1000, use_hessian_based_weights=False, use_hessian_sample_attention=False
|
||||
),
|
||||
core_config=config,
|
||||
target_platform_capabilities=tpc,
|
||||
)[0]
|
||||
if gptq
|
||||
else mct.ptq.pytorch_post_training_quantization( # Perform post training quantization
|
||||
in_module=model,
|
||||
representative_data_gen=representative_dataset_gen,
|
||||
target_resource_utilization=resource_utilization,
|
||||
core_config=config,
|
||||
target_platform_capabilities=tpc,
|
||||
)[0]
|
||||
)
|
||||
|
||||
quant_model = NMSWrapper(
|
||||
model=quant_model,
|
||||
score_threshold=conf or 0.001,
|
||||
iou_threshold=iou,
|
||||
max_detections=max_det,
|
||||
task=model.task,
|
||||
)
|
||||
|
||||
f = Path(str(file).replace(file.suffix, "_imx_model"))
|
||||
f.mkdir(exist_ok=True)
|
||||
onnx_model = f / Path(str(file.name).replace(file.suffix, "_imx.onnx")) # js dir
|
||||
mct.exporter.pytorch_export_model(
|
||||
model=quant_model, save_model_path=onnx_model, repr_dataset=representative_dataset_gen
|
||||
)
|
||||
|
||||
model_onnx = onnx.load(onnx_model) # load onnx model
|
||||
for k, v in metadata.items():
|
||||
meta = model_onnx.metadata_props.add()
|
||||
meta.key, meta.value = k, str(v)
|
||||
|
||||
onnx.save(model_onnx, onnx_model)
|
||||
|
||||
subprocess.run(
|
||||
["imxconv-pt", "-i", str(onnx_model), "-o", str(f), "--no-input-persistency", "--overwrite-output"],
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Needed for imx models.
|
||||
with open(f / "labels.txt", "w", encoding="utf-8") as file:
|
||||
file.writelines([f"{name}\n" for _, name in model.names.items()])
|
||||
|
||||
return f
|
||||
@ -959,53 +959,6 @@ class EarlyStopping:
|
||||
return stop
|
||||
|
||||
|
||||
class FXModel(nn.Module):
|
||||
"""
|
||||
A custom model class for torch.fx compatibility.
|
||||
|
||||
This class extends `torch.nn.Module` and is designed to ensure compatibility with torch.fx for tracing and graph
|
||||
manipulation. It copies attributes from an existing model and explicitly sets the model attribute to ensure proper
|
||||
copying.
|
||||
|
||||
Attributes:
|
||||
model (nn.Module): The original model's layers.
|
||||
"""
|
||||
|
||||
def __init__(self, model):
|
||||
"""
|
||||
Initialize the FXModel.
|
||||
|
||||
Args:
|
||||
model (nn.Module): The original model to wrap for torch.fx compatibility.
|
||||
"""
|
||||
super().__init__()
|
||||
copy_attr(self, model)
|
||||
# Explicitly set `model` since `copy_attr` somehow does not copy it.
|
||||
self.model = model.model
|
||||
|
||||
def forward(self, x):
|
||||
"""
|
||||
Forward pass through the model.
|
||||
|
||||
This method performs the forward pass through the model, handling the dependencies between layers and saving
|
||||
intermediate outputs.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): The input tensor to the model.
|
||||
|
||||
Returns:
|
||||
(torch.Tensor): The output tensor from the model.
|
||||
"""
|
||||
y = [] # outputs
|
||||
for m in self.model:
|
||||
if m.f != -1: # if not from previous layer
|
||||
# from earlier layers
|
||||
x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f]
|
||||
x = m(x) # run
|
||||
y.append(x) # save output
|
||||
return x
|
||||
|
||||
|
||||
def attempt_compile(
|
||||
model: torch.nn.Module,
|
||||
device: torch.device,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user