ultralytics 8.4.58 Isolated CI export environments (#24649)

Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
This commit is contained in:
Glenn Jocher 2026-05-31 19:23:43 +08:00 committed by GitHub
parent 0f01bcd739
commit 158ced9c17
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 452 additions and 108 deletions

86
.github/scripts/create-export-env.py vendored Normal file
View File

@ -0,0 +1,86 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
"""Build and smoke-validate export test virtual environments from the exporter registry."""
from __future__ import annotations
import argparse
import os
import shutil
import subprocess
from pathlib import Path
from ultralytics.engine.exporter import EXPORT_ENVS
def isolated_env_ids():
"""Return export environments that should run outside the shared base CI environment."""
return sorted(
(env for env, recipe in EXPORT_ENVS.items() if recipe["python"]), key=lambda env: env != "isolated-deepx"
)
def build_env(env_id, root):
"""Build one export environment and run its smoke export commands."""
recipe = EXPORT_ENVS[env_id]
venv = root / env_id
repo = Path(__file__).resolve().parents[2]
shutil.rmtree(venv, ignore_errors=True)
subprocess.run(["uv", "venv", str(venv), "--python", recipe["python"]], check=True)
python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
package = f"{repo}[{','.join(recipe['extras'])}]"
indexes = [token for flag, url in recipe["indexes"] for token in (flag, url)]
torch = [f"torch{recipe['torch']}"] if recipe["torch"] else []
subprocess.run(
[
"uv",
"pip",
"install",
"--python",
str(python),
"-e",
package,
"pytest",
*torch,
*recipe["requirements"],
*indexes,
"--index-strategy",
"unsafe-best-match",
],
check=True,
)
if recipe["env"]:
site_packages = next(venv.glob("lib/python*/site-packages"))
site_packages.joinpath("sitecustomize.py").write_text(
"\n".join(f'import os; os.environ.setdefault("{k}", "{v}")' for k, v in recipe["env"].items()) + "\n"
)
env = {**os.environ, "YOLO_AUTOINSTALL": "false", **recipe["env"]}
yolo = venv / ("Scripts/yolo.exe" if os.name == "nt" else "bin/yolo")
for command in recipe["smoke"]:
cmd = command.split()
executable = yolo if cmd[0] == "yolo" else venv / "bin" / cmd[0]
subprocess.run([str(executable), *cmd[1:]], check=True, cwd=venv, env=env)
def main():
"""Parse arguments and build requested export environments."""
parser = argparse.ArgumentParser()
parser.add_argument("--env", action="append", choices=isolated_env_ids())
parser.add_argument("--list", action="store_true", help="Print isolated export environment ids and exit.")
args = parser.parse_args()
envs = args.env or isolated_env_ids()
if args.list:
print("\n".join(envs))
return
root = Path(os.environ.get("ULTRALYTICS_ISOLATED_VENVS", "/opt/venvs"))
root.mkdir(parents=True, exist_ok=True)
for env_id in envs:
build_env(env_id, root)
if __name__ == "__main__":
main()

View File

@ -1,62 +0,0 @@
#!/usr/bin/env bash
# Run slow tests with enough diagnostics for silent runner failures.
set -euo pipefail
uv_system_args=()
[[ "${MATRIX_OS:-}" == "cpu-latest" ]] || uv_system_args=(--system)
snapshot() {
echo "Resource snapshot: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
df -h .
command -v free > /dev/null 2>&1 && free -h
[[ "$(uname -s)" == "Linux" ]] && ps -eo pid,ppid,pcpu,pmem,rss,vsz,comm --sort=-rss | head -15 || true
}
restore_torch() {
[[ "${MATRIX_TORCH:-latest}" == "latest" ]] && return
echo "Restoring torch==${MATRIX_TORCH} torchvision==${MATRIX_TORCHVISION}"
uv pip install "${uv_system_args[@]}" "torch==${MATRIX_TORCH}" "torchvision==${MATRIX_TORCHVISION}" \
--index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
}
snapshot
if [[ "$(uname -s)" == "Linux" ]]; then
while sleep 60; do snapshot; done &
trap 'kill "$!" 2>/dev/null || true' EXIT
fi
restore_torch
pytest_workers=2
pytest_cmd=(pytest -n "$pytest_workers" --dist=loadfile)
if [[ "${MATRIX_OS:-}" == "ubuntu-24.04-arm" ]]; then
pytest_workers=1
pytest_cmd=(pytest -n "$pytest_workers" --dist=loadfile --forked)
fi
main_args=(
-vv -s --slow --cov=ultralytics/ --cov-report=xml --durations=50 --durations-min=1 tests/
--deselect tests/test_exports.py::test_export_imx
--deselect tests/test_exports.py::test_export_mnn
--deselect tests/test_exports.py::test_export_ncnn
--deselect tests/test_exports.py::test_export_executorch
--deselect tests/test_exports.py::test_export_axelera
--deselect tests/test_exports.py::test_export_deepx
)
PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 "${pytest_cmd[@]}" "${main_args[@]}"
for export_test in tests/test_exports.py::test_export_axelera tests/test_exports.py::test_export_deepx; do
rm -rf "${TMPDIR:-/tmp}"/pytest-of-* ~/.cache/pip || true
uv cache prune --ci || true
python - << 'PY'
import os
from pathlib import Path
if cache_dir := os.environ.get("UV_CACHE_DIR"):
Path(cache_dir).mkdir(parents=True, exist_ok=True)
PY
snapshot
restore_torch
PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 pytest -vv -s --slow --cov=ultralytics/ --cov-append \
--cov-report=xml --durations=10 --durations-min=1 "$export_test"
done

View File

@ -145,7 +145,9 @@ jobs:
if [ "${{ matrix.torch }}" != "latest" ]; then
torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}"
fi
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export-base,solutions]" \
$torch aiohttp pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu \
--index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
@ -165,7 +167,9 @@ jobs:
run: python tests/cache_test_assets.py
- name: Pytest tests
shell: bash # for Windows compatibility
run: pytest -n auto --dist=loadfile --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx
run: pytest -n auto --dist=loadfile --cov=ultralytics/ --cov-report=xml tests/ --export-env base
env:
YOLO_AUTOINSTALL: "false"
- name: Upload Coverage Reports to CodeCov
if: github.repository == 'ultralytics/ultralytics' # && matrix.os == 'ubuntu-latest' && matrix.python == '3.13'
uses: codecov/codecov-action@v6
@ -176,6 +180,7 @@ jobs:
- name: Prune uv Cache
run: uv cache prune --ci
# Shared runner CI keeps current stacks on the main path; legacy torch shards are isolated only where needed.
SlowTests:
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.tests == 'true') || github.event_name == 'schedule'
timeout-minutes: 360
@ -187,6 +192,7 @@ jobs:
python: ["3.13"]
torch: [latest]
include:
# Legacy PyTorch shards keep NumPy <2 only where the older torch builds actually need it.
- os: ubuntu-latest
python: "3.8" # torch 1.8.0 requires python >=3.6, <=3.9
torch: "1.8.0"
@ -195,34 +201,42 @@ jobs:
python: "3.9"
torch: "1.9.0"
torchvision: "0.10.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.9"
torch: "1.10.0"
torchvision: "0.11.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.10"
torch: "1.11.0"
torchvision: "0.12.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.10"
torch: "1.12.0"
torchvision: "0.13.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.10"
torch: "1.13.0"
torchvision: "0.14.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.11"
torch: "2.0.0"
torchvision: "0.15.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.11"
torch: "2.1.0"
torchvision: "0.16.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.12"
torch: "2.2.0"
torchvision: "0.17.0"
numpy_extra: export-legacy-torch
- os: ubuntu-latest
python: "3.12"
torch: "2.3.0"
@ -276,6 +290,10 @@ jobs:
if [ "${{ matrix.os }}" != "cpu-latest" ]; then
uv_system_args=(--system)
fi
legacy_extra=""
if [ -n "${{ matrix.numpy_extra }}" ]; then
legacy_extra=",${{ matrix.numpy_extra }}"
fi
if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
torch_packages=(torch torchvision)
if [ "${{ matrix.torch }}" != "latest" ]; then
@ -283,15 +301,15 @@ jobs:
fi
uv pip install "${uv_system_args[@]}" "${torch_packages[@]}" \
--index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
uv pip install "${uv_system_args[@]}" -e ".[export,solutions]" \
faster-coco-eval mlflow pytest-cov pytest-forked pytest-xdist
uv pip install "${uv_system_args[@]}" -e ".[export-base,solutions${legacy_extra}]" \
aiohttp faster-coco-eval mlflow pytest-cov pytest-xdist
else
torch=""
if [ "${{ matrix.torch }}" != "latest" ]; then
torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}"
fi
uv pip install "${uv_system_args[@]}" -e ".[export,solutions]" $torch \
faster-coco-eval mlflow pytest-cov pytest-forked pytest-xdist \
uv pip install "${uv_system_args[@]}" -e ".[export-base,solutions${legacy_extra}]" $torch \
aiohttp faster-coco-eval mlflow pytest-cov pytest-xdist \
--extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
fi
- name: Check environment
@ -315,15 +333,75 @@ jobs:
uses: ultralytics/actions/retry@main
with:
shell: bash # for Windows compatibility
# -n 2 limits parallelism for resource-heavy export tests on shared runners
run: .github/scripts/run-slow-tests.sh
# -n 2 limits parallelism for resource-heavy base export tests on shared runners. The ARM runner is
# single-process to avoid xdist worker overhead and runner disconnects during long slow-test runs.
run: |
pytest_args=(-vv -s --slow --durations=50 --durations-min=1 tests/ --export-env base)
if [[ "${{ matrix.os }}" == "ubuntu-24.04-arm" ]]; then
PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 pytest "${pytest_args[@]}"
else
PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 pytest -n 2 --dist=loadfile "${pytest_args[@]}"
fi
retries: 1 # Retry once after initial attempt (2 total runs)
timeout_minutes: 180
retry_delay_seconds: 60
env:
MATRIX_OS: ${{ matrix.os }}
MATRIX_TORCH: ${{ matrix.torch }}
MATRIX_TORCHVISION: ${{ matrix.torchvision }}
YOLO_AUTOINSTALL: "false"
- name: Prune uv Cache
run: uv cache prune --ci
# Export recipes that need their own environments are validated here so the shared CI jobs stay lean.
IsolatedExports:
if: github.event_name == 'pull_request' || github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tests == 'true') || github.event_name == 'schedule'
timeout-minutes: 360
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.13"
- uses: astral-sh/setup-uv@v7
with:
python-version: "3.13"
- name: Install export system dependencies
run: sudo apt-get update && sudo apt-get install -y openjdk-21-jre flatbuffers-compiler
- name: Cache uv wheels
uses: actions/cache@v5
with:
path: ~/.cache/uv
key: isolated-exports-uv-${{ hashFiles('ultralytics/engine/exporter.py', 'pyproject.toml') }}
restore-keys: |
isolated-exports-uv-
- name: Install base package
run: uv pip install --system -e ".[export-base]" pytest
- name: Configure Ultralytics cache paths
run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets"
- name: Cache test assets
uses: actions/cache@v5
with:
path: ${{ github.workspace }}/.ultralytics_cache
key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }}
restore-keys: |
ultralytics-test-assets-${{ runner.os }}-
- name: Pre-cache test assets
run: python tests/cache_test_assets.py --slow
- name: Build and smoke-validate export environments
env:
ULTRALYTICS_ISOLATED_VENVS: ${{ runner.temp }}/venvs
YOLO_AUTOINSTALL: "false"
run: python .github/scripts/create-export-env.py
- name: Run export tests in their isolated environments
env:
ULTRALYTICS_ISOLATED_VENVS: ${{ runner.temp }}/venvs
YOLO_AUTOINSTALL: "false"
run: |
slow_arg=""
if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
slow_arg="--slow"
fi
for env in $(python .github/scripts/create-export-env.py --list); do
"${ULTRALYTICS_ISOLATED_VENVS}/${env}/bin/python" -m pytest -vv -s $slow_arg tests/test_exports.py --export-env "$env"
done
- name: Prune uv Cache
run: uv cache prune --ci
@ -362,6 +440,7 @@ jobs:
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Raspberry Pi stays separate because it is a long-tail self-hosted runner with its own setup and cleanup flow.
RaspberryPi:
if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.raspberrypi == 'true')
timeout-minutes: 120
@ -395,7 +474,7 @@ jobs:
- name: Pre-cache test assets
run: python tests/cache_test_assets.py --slow
- name: Pytest tests
run: pytest -n auto --dist=loadfile --slow tests/ --deselect tests/test_exports.py::test_export_imx
run: pytest -n auto --dist=loadfile --slow tests/ --export-env base
- name: Activate Virtual Environment for Benchmarks
run: |
python3.11 -m venv env-benchmarks
@ -530,7 +609,7 @@ jobs:
- name: test_engine.py
run: pytest ultralytics/tests/test_engine.py -v -s
- name: test_exports.py
run: pytest ultralytics/tests/test_exports.py -v -s --deselect ultralytics/tests/test_exports.py::test_export_imx
run: pytest ultralytics/tests/test_exports.py -v -s
- name: test_integrations.py
run: pytest ultralytics/tests/test_integrations.py -v -s
- name: test_solutions.py
@ -541,11 +620,11 @@ jobs:
Summary:
runs-on: ubuntu-latest
needs: [Benchmarks, Tests, SlowTests, GPU, RaspberryPi, NVIDIA_Jetson, Conda]
needs: [Benchmarks, Tests, SlowTests, IsolatedExports, GPU, RaspberryPi, NVIDIA_Jetson, Conda]
if: always()
steps:
- name: Check for failure and notify
if: (needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1'
if: (needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.IsolatedExports.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1'
uses: slackapi/slack-github-action@v3.0.3
with:
webhook-type: incoming-webhook

View File

@ -239,6 +239,10 @@ jobs:
if: (github.event_name == 'push' || github.event.inputs[matrix.dockerfile] == 'true') && (matrix.platforms == 'linux/amd64' || matrix.platforms == 'linux/arm64') && matrix.dockerfile != 'Dockerfile-conda'
run: docker run ultralytics/ultralytics:${{ (matrix.tags == 'latest-python' && 'latest-python-export') || (matrix.tags == 'latest' && 'latest-export') || matrix.tags }} /bin/bash -c "yolo checks && uv pip list"
- name: Prune Docker Build Cache
if: (github.event_name == 'push' || github.event.inputs[matrix.dockerfile] == 'true') && (matrix.platforms == 'linux/amd64' || matrix.platforms == 'linux/arm64') && matrix.dockerfile != 'Dockerfile-conda'
run: docker builder prune -af
- name: Run Tests
if: (github.event_name == 'push' || github.event.inputs[matrix.dockerfile] == 'true') && (matrix.platforms == 'linux/amd64' || matrix.platforms == 'linux/arm64') && matrix.dockerfile != 'Dockerfile-conda'
run: docker run ultralytics/ultralytics:${{ (matrix.tags == 'latest-python' && 'latest-python-export') || (matrix.tags == 'latest' && 'latest-export') || matrix.tags }} /bin/bash -c "uv pip install --system --break-system-packages pytest && pytest tests"

View File

@ -14,7 +14,8 @@ RUN uv pip install --system -e ".[export]" "onnxruntime-gpu" paddlepaddle x2padd
yolo export model=tmp/yolo26n.pt format=edgetpu imgsz=32 && \
yolo export model=tmp/yolo26n.pt format=ncnn imgsz=32 && \
# Remove temporary files \
rm -rf tmp /root/.config/Ultralytics/persistent_cache.json
uv cache clean && \
rm -rf tmp ~/.cache /root/.config/Ultralytics/persistent_cache.json
# Usage --------------------------------------------------------------------------------------------------------------

View File

@ -22,7 +22,8 @@ RUN uv pip install --system -e ".[export]" && \
yolo export model=tmp/yolo11n.pt format=imx imgsz=32 && \
uv pip install --system paddlepaddle x2paddle && \
# Remove extra build files
rm -rf tmp /root/.config/Ultralytics/persistent_cache.json
uv cache clean && \
rm -rf tmp ~/.cache /root/.config/Ultralytics/persistent_cache.json
# Usage --------------------------------------------------------------------------------------------------------------

View File

@ -89,16 +89,23 @@ dev = [
"mkdocs-ultralytics-plugin>=0.2.4", # for meta descriptions and images, dates and authors
"minijinja>=2.0.0", # render docs macros without mkdocs-macros-plugin
]
export = [
"numpy<2.0.0; python_version < '3.13'", # TF 2.20 compatibility
"numpy<=2.3.5; python_version >= '3.13'", # pin until CoreML>9.0 new release
export-base = [
"onnx>=1.12.0; platform_system != 'Darwin'", # ONNX export
"onnx>=1.12.0,<1.18.0; platform_system == 'Darwin' and python_version < '3.13'", # TF inference hanging on MacOS (tested up to onnx==1.20.0)
"onnx>=1.20.0; platform_system == 'Darwin' and python_version >= '3.13'",
"onnxslim>=0.1.82",
"coremltools>=9.0; platform_system != 'Windows' and python_version <= '3.13'", # CoreML supported on macOS and Linux
"scikit-learn>=1.3.2; platform_system != 'Windows' and python_version <= '3.13'", # CoreML k-means quantization
# ONNX export validation and inference. Jetson jobs provide custom aarch64 onnxruntime-gpu wheels.
"onnxruntime<1.20.0; python_version < '3.11' and (platform_machine != 'aarch64' or platform_system != 'Linux')",
"onnxruntime>=1.20.0; python_version >= '3.11'",
"nncf>=2.14.0,<3.0.0; python_version >= '3.9'", # OpenVINO INT8 export
"openvino>=2024.0.0", # OpenVINO export
"packaging>=26.0; platform_machine == 'aarch64' and platform_system == 'Linux' and python_version >= '3.9'", # IMX export bug
]
export-legacy-torch = [
"numpy<2",
]
export-tensorflow = [
"numpy<2.0.0; python_version < '3.13'", # TF 2.20 compatibility
"tensorflow>=2.0.0,<=2.19.0; python_version < '3.13'", # TF bug https://github.com/ultralytics/ultralytics/issues/5161
"tensorflow>2.19.0; python_version >= '3.13'",
"tensorflowjs>=2.0.0", # TF.js export, automatically installs tensorflow
@ -107,10 +114,31 @@ export = [
"tensorstore>=0.1.63; platform_machine == 'aarch64' and python_version >= '3.9'", # for TF Raspberry Pi exports
"h5py!=3.11.0; platform_machine == 'aarch64'", # fix h5py build issues due to missing aarch64 wheels in 3.11 release
"setuptools<=81.0.0", # pin due to >=82.0.0 breaking tensorflow.js package
"packaging>=26.0; platform_machine == 'aarch64' and platform_system == 'Linux' and python_version >= '3.9'", # IMX export bug
]
export-coreml = [
"numpy<=2.3.5; python_version >= '3.13'", # pin until CoreML>9.0 new release
"coremltools>=9.0; platform_system != 'Windows' and python_version <= '3.13'", # CoreML supported on macOS and Linux
"scikit-learn>=1.3.2; platform_system != 'Windows' and python_version <= '3.13'", # CoreML k-means quantization
]
export-executorch = [
"executorch",
"numpy<=2.3.5",
]
export-deepx = [
"dx_com",
"numpy<2",
]
export = [
"ultralytics[export-base]",
"ultralytics[export-tensorflow]",
"ultralytics[export-coreml]",
]
solutions = [
"shapely>=2.0.0", # shapely for point and polygon data matching
"lap>=0.5.12", # for solution tracking linear assignment
"faiss-cpu", # for similarity search
"openai-clip>=1.0.1", # for similarity search text/image embeddings
"setuptools>=80.0.0,<81.0.0; python_version >= '3.9'", # openai-clip imports pkg_resources.packaging, removed in setuptools>=81
"streamlit>=1.51.0; python_version >= '3.10'", # for live inference on web browser, i.e `yolo streamlit-predict`
"streamlit>=1.29.0,<1.51.0; python_version < '3.10' and (python_version < '3.9' or platform_machine != 'aarch64' or platform_system != 'Linux')",
"flask>=3.0.1", # for similarity search solution

View File

@ -29,6 +29,27 @@ def solution_assets():
def pytest_addoption(parser):
"""Add custom command-line options to pytest."""
parser.addoption("--slow", action="store_true", default=False, help="Run slow tests")
parser.addoption(
"--export-env",
default=None,
help="Run only export tests assigned to this export environment id.",
)
def _export_format_from_item(item, formats):
"""Infer the export format covered by a tests/test_exports.py item."""
if Path(str(item.fspath)).name != "test_exports.py":
return None
name = getattr(item, "originalname", None) or item.name.split("[", 1)[0]
if name == "test_torch2onnx_serializes_concurrent_exports":
return "onnx"
if not name.startswith("test_export_"):
return None
suffix = name[len("test_export_") :]
return next(
(fmt for fmt in sorted(formats, key=len, reverse=True) if suffix == fmt or suffix.startswith(f"{fmt}_")), None
)
def pytest_collection_modifyitems(config, items):
@ -42,6 +63,18 @@ def pytest_collection_modifyitems(config, items):
# Remove the item entirely from the list of test items if it's marked as 'slow'
items[:] = [item for item in items if "slow" not in item.keywords]
export_env = config.getoption("--export-env")
if not export_env:
return
from ultralytics.engine.exporter import export_formats
env_by_format = dict(zip(export_formats()["Argument"], export_formats()["Env"]))
for item in items:
fmt = _export_format_from_item(item, env_by_format)
if fmt and env_by_format.get(fmt) != export_env:
item.add_marker(pytest.mark.skip(reason=f"export format '{fmt}' belongs to env '{env_by_format[fmt]}'"))
def pytest_sessionstart(session):
"""Initialize session configurations for pytest.

View File

@ -8,7 +8,7 @@ import pytest
from PIL import Image
from tests import CUDA_DEVICE_COUNT, CUDA_IS_AVAILABLE, MODELS, TASK_MODEL_DATA
from ultralytics.utils import ARM64, ASSETS, IS_RASPBERRYPI, LINUX, WEIGHTS_DIR, checks
from ultralytics.utils import ARM64, ASSETS, DATASETS_DIR, IS_RASPBERRYPI, LINUX, WEIGHTS_DIR, checks
from ultralytics.utils.torch_utils import TORCH_1_11
@ -23,6 +23,7 @@ def test_special_modes() -> None:
run("yolo checks")
run("yolo version")
run("yolo settings reset")
run(f"yolo settings weights_dir={WEIGHTS_DIR} datasets_dir={DATASETS_DIR}")
run("yolo cfg")

View File

@ -16,6 +16,7 @@ import torch
from tests import SOURCE
from ultralytics import YOLO
from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS
from ultralytics.engine.exporter import EXPORT_ENVS, export_formats
from ultralytics.utils import (
ARM64,
IS_DOCKER,
@ -406,6 +407,15 @@ def test_export_imx(isolated_model):
YOLO(file)(SOURCE, imgsz=32)
@pytest.mark.slow
@pytest.mark.skipif(not LINUX or ARM64, reason="RKNN export only supported on non-aarch64 Linux")
def test_export_rknn(isolated_model):
"""Test YOLO export to RKNN format."""
file = YOLO(isolated_model).export(format="rknn", imgsz=32)
assert next(Path(file).rglob("*.rknn"), None), f"RKNN export failed, no RKNN model found in: {file}"
shutil.rmtree(file, ignore_errors=True)
# @pytest.mark.skipif(True, reason="Disabled for debugging ruamel.yaml installation required by executorch")
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0")
@pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows")
@ -496,3 +506,15 @@ def test_export_qnn(isolated_model):
assert next(Path(file).rglob("*_qnn.onnx"), None), f"QNN export failed, no context binary found in: {file}"
# Note: on-device inference is not exercised here as it requires Qualcomm Snapdragon hardware
shutil.rmtree(file, ignore_errors=True) # cleanup
@pytest.mark.parametrize("env", [k for k, v in EXPORT_ENVS.items() if k != "base" or v["smoke"]])
def test_export_env_has_smoke(env):
"""Ensure every non-base export environment declares a build-time smoke export."""
assert EXPORT_ENVS[env]["smoke"], f"export env '{env}' has no smoke command"
def test_every_format_env_is_registered():
"""Ensure every export format points at a registered export environment."""
for fmt, env in zip(export_formats()["Argument"], export_formats()["Env"]):
assert env in EXPORT_ENVS, f"format '{fmt}' references unknown env '{env}'"

View File

@ -1,6 +1,6 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
__version__ = "8.4.57"
__version__ = "8.4.58"
import importlib
import os

View File

@ -131,10 +131,19 @@ from ultralytics.utils.torch_utils import (
def export_formats():
"""Return a dictionary of Ultralytics YOLO export formats."""
# Format, Argument, Suffix, CPU, GPU, Arguments, Env
x = [
["PyTorch", "-", ".pt", True, True, []],
["TorchScript", "torchscript", ".torchscript", True, True, ["batch", "optimize", "half", "nms", "dynamic"]],
["ONNX", "onnx", ".onnx", True, True, ["batch", "dynamic", "half", "opset", "simplify", "nms"]],
["PyTorch", "-", ".pt", True, True, [], "base"],
[
"TorchScript",
"torchscript",
".torchscript",
True,
True,
["batch", "optimize", "half", "nms", "dynamic"],
"base",
],
["ONNX", "onnx", ".onnx", True, True, ["batch", "dynamic", "half", "opset", "simplify", "nms"], "base"],
[
"OpenVINO",
"openvino",
@ -142,6 +151,7 @@ def export_formats():
True,
False,
["batch", "data", "dynamic", "half", "int8", "nms", "fraction"],
"base",
],
[
"TensorRT",
@ -150,8 +160,9 @@ def export_formats():
False,
True,
["batch", "data", "dynamic", "half", "int8", "simplify", "nms", "fraction"],
"base",
],
["CoreML", "coreml", ".mlpackage", True, False, ["batch", "dynamic", "half", "int8", "nms"]],
["CoreML", "coreml", ".mlpackage", True, False, ["batch", "dynamic", "half", "int8", "nms"], "coreml"],
[
"TensorFlow SavedModel",
"saved_model",
@ -159,22 +170,164 @@ def export_formats():
True,
True,
["batch", "data", "fraction", "int8", "keras", "nms"],
"tensorflow",
],
["TensorFlow GraphDef", "pb", ".pb", True, True, ["batch"]],
["TensorFlow Lite", "tflite", ".tflite", True, False, ["batch", "data", "half", "int8", "nms", "fraction"]],
["TensorFlow Edge TPU", "edgetpu", "_edgetpu.tflite", True, False, ["data", "fraction", "int8"]],
["TensorFlow.js", "tfjs", "_web_model", True, False, ["batch", "data", "fraction", "half", "int8", "nms"]],
["PaddlePaddle", "paddle", "_paddle_model", True, True, ["batch"]],
["MNN", "mnn", ".mnn", True, True, ["batch", "half", "int8"]],
["NCNN", "ncnn", "_ncnn_model", True, True, ["batch", "half"]],
["IMX", "imx", "_imx_model", True, True, ["data", "int8", "fraction", "nms"]],
["RKNN", "rknn", "_rknn_model", False, False, ["batch", "name"]],
["ExecuTorch", "executorch", "_executorch_model", True, False, ["batch"]],
["Axelera AI", "axelera", "_axelera_model", False, False, ["batch", "int8", "fraction", "data"]],
["DEEPX", "deepx", "_deepx_model", False, False, ["data", "int8", "optimize"]],
["Qualcomm QNN", "qnn", "_qnn_model", False, False, ["batch", "name", "int8", "fraction", "data"]],
["TensorFlow GraphDef", "pb", ".pb", True, True, ["batch"], "tensorflow"],
[
"TensorFlow Lite",
"tflite",
".tflite",
True,
False,
["batch", "data", "half", "int8", "nms", "fraction"],
"tensorflow",
],
["TensorFlow Edge TPU", "edgetpu", "_edgetpu.tflite", True, False, ["data", "fraction", "int8"], "tensorflow"],
[
"TensorFlow.js",
"tfjs",
"_web_model",
True,
False,
["batch", "data", "fraction", "half", "int8", "nms"],
"tensorflow",
],
["PaddlePaddle", "paddle", "_paddle_model", True, True, ["batch"], "base"],
["MNN", "mnn", ".mnn", True, True, ["batch", "half", "int8"], "mnn"],
["NCNN", "ncnn", "_ncnn_model", True, True, ["batch", "half"], "ncnn"],
["IMX", "imx", "_imx_model", True, True, ["data", "int8", "fraction", "nms"], "isolated-imx"],
["RKNN", "rknn", "_rknn_model", False, False, ["batch", "name"], "isolated-rknn"],
["ExecuTorch", "executorch", "_executorch_model", True, False, ["batch"], "executorch"],
[
"Axelera AI",
"axelera",
"_axelera_model",
False,
False,
["batch", "int8", "fraction", "data"],
"isolated-axelera",
],
["DEEPX", "deepx", "_deepx_model", False, False, ["data", "int8", "optimize"], "isolated-deepx"],
["Qualcomm QNN", "qnn", "_qnn_model", False, False, ["batch", "name", "int8", "fraction", "data"], "base"],
]
return dict(zip(["Format", "Argument", "Suffix", "CPU", "GPU", "Arguments"], zip(*x)))
return dict(zip(["Format", "Argument", "Suffix", "CPU", "GPU", "Arguments", "Env"], zip(*x)))
EXPORT_ENVS = {
"base": {
"python": None,
"extras": ["export-base"],
"torch": None,
"requirements": [],
"indexes": [],
"env": {},
"smoke": [],
},
"tensorflow": {
"python": "3.12",
"extras": ["export-base", "export-tensorflow"],
"torch": None,
"requirements": [
"onnx2tf>=1.26.3,<1.29.0",
"tf_keras<=2.19.0",
"sng4onnx>=1.0.1",
"onnx_graphsurgeon>=0.3.26",
"ai-edge-litert>=1.2.0",
"onnxruntime",
"protobuf>=5",
],
"indexes": [
("--extra-index-url", "https://download.pytorch.org/whl/cpu"),
("--extra-index-url", "https://pypi.ngc.nvidia.com"),
],
"env": {},
"smoke": ["yolo export format=tflite model=yolo11n.pt imgsz=32"],
},
"coreml": {
"python": "3.13",
"extras": ["export-base", "export-coreml"],
"torch": ">=2.12",
"requirements": [],
"indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")],
"env": {},
"smoke": ["yolo export format=coreml model=yolo11n.pt imgsz=32"],
},
"mnn": {
"python": "3.13",
"extras": ["export-base"],
"torch": None,
"requirements": ["MNN>=2.9.6"],
"indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")],
"env": {},
"smoke": ["yolo export format=mnn model=yolo11n.pt imgsz=32"],
},
"ncnn": {
"python": "3.13",
"extras": ["export-base"],
"torch": None,
"requirements": ["ncnn", "pnnx"],
"indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")],
"env": {},
"smoke": ["yolo export format=ncnn model=yolo11n.pt imgsz=32"],
},
"executorch": {
"python": "3.13",
"extras": ["export-base", "export-executorch"],
"torch": ">=2.12",
"requirements": [],
"indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")],
"env": {},
"smoke": ["yolo export format=executorch model=yolo11n.pt imgsz=32"],
},
"isolated-imx": {
"python": "3.11",
"extras": ["export-base"],
"torch": ">=2.9,<2.12",
"requirements": [
"model-compression-toolkit>=2.4.1",
"edge-mdt-cl<1.1.0",
"edge-mdt-tpc>=1.2.0",
"pydantic<2.12",
"imx500-converter[pt]>=3.17.3",
],
"indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")],
"env": {},
"smoke": ["yolo export format=imx model=yolo11n.pt imgsz=32"],
},
"isolated-rknn": {
"python": "3.11",
"extras": ["export-base"],
"torch": "==2.4",
"requirements": ["rknn-toolkit2>=2.3.2", "onnx>=1.16.1,<1.19.0", "setuptools<82"],
"indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")],
"env": {},
"smoke": ["yolo export format=rknn model=yolo11n.pt imgsz=32"],
},
"isolated-axelera": {
"python": "3.12",
"extras": ["export-base"],
"torch": ">=2.8,<2.12",
"requirements": ["axelera-devkit==1.6.0", "onnx>=1.12.0,<2.0.0", "onnxslim>=0.1.71"],
"indexes": [
("--extra-index-url", "https://download.pytorch.org/whl/cpu"),
("--extra-index-url", "https://software.axelera.ai/artifactory/api/pypi/axelera-pypi/simple"),
],
"env": {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "python"},
"smoke": ["yolo export format=axelera model=yolo11n.pt imgsz=64 data=coco8.yaml"],
},
"isolated-deepx": {
"python": "3.12",
"extras": ["export-base", "export-deepx"],
"torch": ">=2.8,<2.12",
"requirements": [],
"indexes": [
("--extra-index-url", "https://download.pytorch.org/whl/cpu"),
("--find-links", "https://sdk.deepx.ai/release/dxcom/v2.3.0/index.html"),
],
"env": {},
"smoke": ["yolo export format=deepx model=yolo11n.pt imgsz=32"],
},
}
def validate_args(format, passed_args, valid_args):

View File

@ -126,7 +126,7 @@ def benchmark(
if format_arg:
formats = frozenset(export_formats()["Argument"])
assert format in formats, f"Expected format to be one of {formats}, but got '{format_arg}'."
for name, format, suffix, cpu, gpu, valid_args in zip(*export_formats().values()):
for name, format, suffix, cpu, gpu, valid_args, _ in zip(*export_formats().values()):
emoji, filename = "❌", None # export defaults
try:
if format_arg and format_arg != format:

View File

@ -525,8 +525,6 @@ def check_executorch_requirements():
check_requirements("packaging>=22.0")
check_requirements("executorch", cmds=f"torch=={TORCH_VERSION.split('+')[0]}")
# Pin numpy to avoid coremltools errors with numpy>=2.4.0, must be separate
check_requirements("numpy<=2.3.5")
def check_tensorrt(min_version: str = "7.0.0"):