From 158ced9c17df98a4f9c404e2c7dc4a8db32d3aff Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 31 May 2026 19:23:43 +0800 Subject: [PATCH] `ultralytics 8.4.58` Isolated CI export environments (#24649) Co-authored-by: UltralyticsAssistant --- .github/scripts/create-export-env.py | 86 ++++++++++++ .github/scripts/run-slow-tests.sh | 62 --------- .github/workflows/ci.yml | 109 ++++++++++++--- .github/workflows/docker.yml | 4 + docker/Dockerfile-export | 3 +- docker/Dockerfile-python-export | 3 +- pyproject.toml | 40 +++++- tests/conftest.py | 33 +++++ tests/test_cli.py | 3 +- tests/test_exports.py | 22 ++++ ultralytics/__init__.py | 2 +- ultralytics/engine/exporter.py | 189 ++++++++++++++++++++++++--- ultralytics/utils/benchmarks.py | 2 +- ultralytics/utils/checks.py | 2 - 14 files changed, 452 insertions(+), 108 deletions(-) create mode 100644 .github/scripts/create-export-env.py delete mode 100755 .github/scripts/run-slow-tests.sh diff --git a/.github/scripts/create-export-env.py b/.github/scripts/create-export-env.py new file mode 100644 index 00000000..111b25ce --- /dev/null +++ b/.github/scripts/create-export-env.py @@ -0,0 +1,86 @@ +# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license +"""Build and smoke-validate export test virtual environments from the exporter registry.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +from pathlib import Path + +from ultralytics.engine.exporter import EXPORT_ENVS + + +def isolated_env_ids(): + """Return export environments that should run outside the shared base CI environment.""" + return sorted( + (env for env, recipe in EXPORT_ENVS.items() if recipe["python"]), key=lambda env: env != "isolated-deepx" + ) + + +def build_env(env_id, root): + """Build one export environment and run its smoke export commands.""" + recipe = EXPORT_ENVS[env_id] + venv = root / env_id + repo = Path(__file__).resolve().parents[2] + + shutil.rmtree(venv, ignore_errors=True) + subprocess.run(["uv", "venv", str(venv), "--python", recipe["python"]], check=True) + python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + package = f"{repo}[{','.join(recipe['extras'])}]" + indexes = [token for flag, url in recipe["indexes"] for token in (flag, url)] + torch = [f"torch{recipe['torch']}"] if recipe["torch"] else [] + subprocess.run( + [ + "uv", + "pip", + "install", + "--python", + str(python), + "-e", + package, + "pytest", + *torch, + *recipe["requirements"], + *indexes, + "--index-strategy", + "unsafe-best-match", + ], + check=True, + ) + + if recipe["env"]: + site_packages = next(venv.glob("lib/python*/site-packages")) + site_packages.joinpath("sitecustomize.py").write_text( + "\n".join(f'import os; os.environ.setdefault("{k}", "{v}")' for k, v in recipe["env"].items()) + "\n" + ) + + env = {**os.environ, "YOLO_AUTOINSTALL": "false", **recipe["env"]} + yolo = venv / ("Scripts/yolo.exe" if os.name == "nt" else "bin/yolo") + for command in recipe["smoke"]: + cmd = command.split() + executable = yolo if cmd[0] == "yolo" else venv / "bin" / cmd[0] + subprocess.run([str(executable), *cmd[1:]], check=True, cwd=venv, env=env) + + +def main(): + """Parse arguments and build requested export environments.""" + parser = argparse.ArgumentParser() + parser.add_argument("--env", action="append", choices=isolated_env_ids()) + parser.add_argument("--list", action="store_true", help="Print isolated export environment ids and exit.") + args = parser.parse_args() + envs = args.env or isolated_env_ids() + + if args.list: + print("\n".join(envs)) + return + + root = Path(os.environ.get("ULTRALYTICS_ISOLATED_VENVS", "/opt/venvs")) + root.mkdir(parents=True, exist_ok=True) + for env_id in envs: + build_env(env_id, root) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/run-slow-tests.sh b/.github/scripts/run-slow-tests.sh deleted file mode 100755 index ca3b489e..00000000 --- a/.github/scripts/run-slow-tests.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# Run slow tests with enough diagnostics for silent runner failures. - -set -euo pipefail - -uv_system_args=() -[[ "${MATRIX_OS:-}" == "cpu-latest" ]] || uv_system_args=(--system) - -snapshot() { - echo "Resource snapshot: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" - df -h . - command -v free > /dev/null 2>&1 && free -h - [[ "$(uname -s)" == "Linux" ]] && ps -eo pid,ppid,pcpu,pmem,rss,vsz,comm --sort=-rss | head -15 || true -} - -restore_torch() { - [[ "${MATRIX_TORCH:-latest}" == "latest" ]] && return - echo "Restoring torch==${MATRIX_TORCH} torchvision==${MATRIX_TORCHVISION}" - uv pip install "${uv_system_args[@]}" "torch==${MATRIX_TORCH}" "torchvision==${MATRIX_TORCHVISION}" \ - --index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match -} - -snapshot -if [[ "$(uname -s)" == "Linux" ]]; then - while sleep 60; do snapshot; done & - trap 'kill "$!" 2>/dev/null || true' EXIT -fi - -restore_torch -pytest_workers=2 -pytest_cmd=(pytest -n "$pytest_workers" --dist=loadfile) -if [[ "${MATRIX_OS:-}" == "ubuntu-24.04-arm" ]]; then - pytest_workers=1 - pytest_cmd=(pytest -n "$pytest_workers" --dist=loadfile --forked) -fi - -main_args=( - -vv -s --slow --cov=ultralytics/ --cov-report=xml --durations=50 --durations-min=1 tests/ - --deselect tests/test_exports.py::test_export_imx - --deselect tests/test_exports.py::test_export_mnn - --deselect tests/test_exports.py::test_export_ncnn - --deselect tests/test_exports.py::test_export_executorch - --deselect tests/test_exports.py::test_export_axelera - --deselect tests/test_exports.py::test_export_deepx -) -PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 "${pytest_cmd[@]}" "${main_args[@]}" - -for export_test in tests/test_exports.py::test_export_axelera tests/test_exports.py::test_export_deepx; do - rm -rf "${TMPDIR:-/tmp}"/pytest-of-* ~/.cache/pip || true - uv cache prune --ci || true - python - << 'PY' -import os -from pathlib import Path - -if cache_dir := os.environ.get("UV_CACHE_DIR"): - Path(cache_dir).mkdir(parents=True, exist_ok=True) -PY - snapshot - restore_torch - PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 pytest -vv -s --slow --cov=ultralytics/ --cov-append \ - --cov-report=xml --durations=10 --durations-min=1 "$export_test" -done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a206b50a..fc02bcc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,7 +145,9 @@ jobs: if [ "${{ matrix.torch }}" != "latest" ]; then torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}" fi - uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match + uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export-base,solutions]" \ + $torch aiohttp pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu \ + --index-strategy unsafe-best-match - name: Check environment run: | yolo checks @@ -165,7 +167,9 @@ jobs: run: python tests/cache_test_assets.py - name: Pytest tests shell: bash # for Windows compatibility - run: pytest -n auto --dist=loadfile --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx + run: pytest -n auto --dist=loadfile --cov=ultralytics/ --cov-report=xml tests/ --export-env base + env: + YOLO_AUTOINSTALL: "false" - name: Upload Coverage Reports to CodeCov if: github.repository == 'ultralytics/ultralytics' # && matrix.os == 'ubuntu-latest' && matrix.python == '3.13' uses: codecov/codecov-action@v6 @@ -176,6 +180,7 @@ jobs: - name: Prune uv Cache run: uv cache prune --ci + # Shared runner CI keeps current stacks on the main path; legacy torch shards are isolated only where needed. SlowTests: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.tests == 'true') || github.event_name == 'schedule' timeout-minutes: 360 @@ -187,6 +192,7 @@ jobs: python: ["3.13"] torch: [latest] include: + # Legacy PyTorch shards keep NumPy <2 only where the older torch builds actually need it. - os: ubuntu-latest python: "3.8" # torch 1.8.0 requires python >=3.6, <=3.9 torch: "1.8.0" @@ -195,34 +201,42 @@ jobs: python: "3.9" torch: "1.9.0" torchvision: "0.10.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.9" torch: "1.10.0" torchvision: "0.11.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.10" torch: "1.11.0" torchvision: "0.12.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.10" torch: "1.12.0" torchvision: "0.13.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.10" torch: "1.13.0" torchvision: "0.14.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.11" torch: "2.0.0" torchvision: "0.15.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.11" torch: "2.1.0" torchvision: "0.16.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.12" torch: "2.2.0" torchvision: "0.17.0" + numpy_extra: export-legacy-torch - os: ubuntu-latest python: "3.12" torch: "2.3.0" @@ -276,6 +290,10 @@ jobs: if [ "${{ matrix.os }}" != "cpu-latest" ]; then uv_system_args=(--system) fi + legacy_extra="" + if [ -n "${{ matrix.numpy_extra }}" ]; then + legacy_extra=",${{ matrix.numpy_extra }}" + fi if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then torch_packages=(torch torchvision) if [ "${{ matrix.torch }}" != "latest" ]; then @@ -283,15 +301,15 @@ jobs: fi uv pip install "${uv_system_args[@]}" "${torch_packages[@]}" \ --index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match - uv pip install "${uv_system_args[@]}" -e ".[export,solutions]" \ - faster-coco-eval mlflow pytest-cov pytest-forked pytest-xdist + uv pip install "${uv_system_args[@]}" -e ".[export-base,solutions${legacy_extra}]" \ + aiohttp faster-coco-eval mlflow pytest-cov pytest-xdist else torch="" if [ "${{ matrix.torch }}" != "latest" ]; then torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}" fi - uv pip install "${uv_system_args[@]}" -e ".[export,solutions]" $torch \ - faster-coco-eval mlflow pytest-cov pytest-forked pytest-xdist \ + uv pip install "${uv_system_args[@]}" -e ".[export-base,solutions${legacy_extra}]" $torch \ + aiohttp faster-coco-eval mlflow pytest-cov pytest-xdist \ --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match fi - name: Check environment @@ -315,15 +333,75 @@ jobs: uses: ultralytics/actions/retry@main with: shell: bash # for Windows compatibility - # -n 2 limits parallelism for resource-heavy export tests on shared runners - run: .github/scripts/run-slow-tests.sh + # -n 2 limits parallelism for resource-heavy base export tests on shared runners. The ARM runner is + # single-process to avoid xdist worker overhead and runner disconnects during long slow-test runs. + run: | + pytest_args=(-vv -s --slow --durations=50 --durations-min=1 tests/ --export-env base) + if [[ "${{ matrix.os }}" == "ubuntu-24.04-arm" ]]; then + PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 pytest "${pytest_args[@]}" + else + PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 pytest -n 2 --dist=loadfile "${pytest_args[@]}" + fi retries: 1 # Retry once after initial attempt (2 total runs) timeout_minutes: 180 retry_delay_seconds: 60 env: - MATRIX_OS: ${{ matrix.os }} - MATRIX_TORCH: ${{ matrix.torch }} - MATRIX_TORCHVISION: ${{ matrix.torchvision }} + YOLO_AUTOINSTALL: "false" + - name: Prune uv Cache + run: uv cache prune --ci + + # Export recipes that need their own environments are validated here so the shared CI jobs stay lean. + IsolatedExports: + if: github.event_name == 'pull_request' || github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.tests == 'true') || github.event_name == 'schedule' + timeout-minutes: 360 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + - name: Install export system dependencies + run: sudo apt-get update && sudo apt-get install -y openjdk-21-jre flatbuffers-compiler + - name: Cache uv wheels + uses: actions/cache@v5 + with: + path: ~/.cache/uv + key: isolated-exports-uv-${{ hashFiles('ultralytics/engine/exporter.py', 'pyproject.toml') }} + restore-keys: | + isolated-exports-uv- + - name: Install base package + run: uv pip install --system -e ".[export-base]" pytest + - name: Configure Ultralytics cache paths + run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets" + - name: Cache test assets + uses: actions/cache@v5 + with: + path: ${{ github.workspace }}/.ultralytics_cache + key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }} + restore-keys: | + ultralytics-test-assets-${{ runner.os }}- + - name: Pre-cache test assets + run: python tests/cache_test_assets.py --slow + - name: Build and smoke-validate export environments + env: + ULTRALYTICS_ISOLATED_VENVS: ${{ runner.temp }}/venvs + YOLO_AUTOINSTALL: "false" + run: python .github/scripts/create-export-env.py + - name: Run export tests in their isolated environments + env: + ULTRALYTICS_ISOLATED_VENVS: ${{ runner.temp }}/venvs + YOLO_AUTOINSTALL: "false" + run: | + slow_arg="" + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + slow_arg="--slow" + fi + for env in $(python .github/scripts/create-export-env.py --list); do + "${ULTRALYTICS_ISOLATED_VENVS}/${env}/bin/python" -m pytest -vv -s $slow_arg tests/test_exports.py --export-env "$env" + done - name: Prune uv Cache run: uv cache prune --ci @@ -362,6 +440,7 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # Raspberry Pi stays separate because it is a long-tail self-hosted runner with its own setup and cleanup flow. RaspberryPi: if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.raspberrypi == 'true') timeout-minutes: 120 @@ -395,7 +474,7 @@ jobs: - name: Pre-cache test assets run: python tests/cache_test_assets.py --slow - name: Pytest tests - run: pytest -n auto --dist=loadfile --slow tests/ --deselect tests/test_exports.py::test_export_imx + run: pytest -n auto --dist=loadfile --slow tests/ --export-env base - name: Activate Virtual Environment for Benchmarks run: | python3.11 -m venv env-benchmarks @@ -530,7 +609,7 @@ jobs: - name: test_engine.py run: pytest ultralytics/tests/test_engine.py -v -s - name: test_exports.py - run: pytest ultralytics/tests/test_exports.py -v -s --deselect ultralytics/tests/test_exports.py::test_export_imx + run: pytest ultralytics/tests/test_exports.py -v -s - name: test_integrations.py run: pytest ultralytics/tests/test_integrations.py -v -s - name: test_solutions.py @@ -541,11 +620,11 @@ jobs: Summary: runs-on: ubuntu-latest - needs: [Benchmarks, Tests, SlowTests, GPU, RaspberryPi, NVIDIA_Jetson, Conda] + needs: [Benchmarks, Tests, SlowTests, IsolatedExports, GPU, RaspberryPi, NVIDIA_Jetson, Conda] if: always() steps: - name: Check for failure and notify - if: (needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1' + if: (needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.IsolatedExports.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1' uses: slackapi/slack-github-action@v3.0.3 with: webhook-type: incoming-webhook diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 80b6ea38..0b8d7fac 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -239,6 +239,10 @@ jobs: if: (github.event_name == 'push' || github.event.inputs[matrix.dockerfile] == 'true') && (matrix.platforms == 'linux/amd64' || matrix.platforms == 'linux/arm64') && matrix.dockerfile != 'Dockerfile-conda' run: docker run ultralytics/ultralytics:${{ (matrix.tags == 'latest-python' && 'latest-python-export') || (matrix.tags == 'latest' && 'latest-export') || matrix.tags }} /bin/bash -c "yolo checks && uv pip list" + - name: Prune Docker Build Cache + if: (github.event_name == 'push' || github.event.inputs[matrix.dockerfile] == 'true') && (matrix.platforms == 'linux/amd64' || matrix.platforms == 'linux/arm64') && matrix.dockerfile != 'Dockerfile-conda' + run: docker builder prune -af + - name: Run Tests if: (github.event_name == 'push' || github.event.inputs[matrix.dockerfile] == 'true') && (matrix.platforms == 'linux/amd64' || matrix.platforms == 'linux/arm64') && matrix.dockerfile != 'Dockerfile-conda' run: docker run ultralytics/ultralytics:${{ (matrix.tags == 'latest-python' && 'latest-python-export') || (matrix.tags == 'latest' && 'latest-export') || matrix.tags }} /bin/bash -c "uv pip install --system --break-system-packages pytest && pytest tests" diff --git a/docker/Dockerfile-export b/docker/Dockerfile-export index e3850940..f2c4aa26 100644 --- a/docker/Dockerfile-export +++ b/docker/Dockerfile-export @@ -14,7 +14,8 @@ RUN uv pip install --system -e ".[export]" "onnxruntime-gpu" paddlepaddle x2padd yolo export model=tmp/yolo26n.pt format=edgetpu imgsz=32 && \ yolo export model=tmp/yolo26n.pt format=ncnn imgsz=32 && \ # Remove temporary files \ - rm -rf tmp /root/.config/Ultralytics/persistent_cache.json + uv cache clean && \ + rm -rf tmp ~/.cache /root/.config/Ultralytics/persistent_cache.json # Usage -------------------------------------------------------------------------------------------------------------- diff --git a/docker/Dockerfile-python-export b/docker/Dockerfile-python-export index 0c8040af..534b256e 100644 --- a/docker/Dockerfile-python-export +++ b/docker/Dockerfile-python-export @@ -22,7 +22,8 @@ RUN uv pip install --system -e ".[export]" && \ yolo export model=tmp/yolo11n.pt format=imx imgsz=32 && \ uv pip install --system paddlepaddle x2paddle && \ # Remove extra build files - rm -rf tmp /root/.config/Ultralytics/persistent_cache.json + uv cache clean && \ + rm -rf tmp ~/.cache /root/.config/Ultralytics/persistent_cache.json # Usage -------------------------------------------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 2685a7bd..19f4a23d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,16 +89,23 @@ dev = [ "mkdocs-ultralytics-plugin>=0.2.4", # for meta descriptions and images, dates and authors "minijinja>=2.0.0", # render docs macros without mkdocs-macros-plugin ] -export = [ - "numpy<2.0.0; python_version < '3.13'", # TF 2.20 compatibility - "numpy<=2.3.5; python_version >= '3.13'", # pin until CoreML>9.0 new release +export-base = [ "onnx>=1.12.0; platform_system != 'Darwin'", # ONNX export "onnx>=1.12.0,<1.18.0; platform_system == 'Darwin' and python_version < '3.13'", # TF inference hanging on MacOS (tested up to onnx==1.20.0) "onnx>=1.20.0; platform_system == 'Darwin' and python_version >= '3.13'", "onnxslim>=0.1.82", - "coremltools>=9.0; platform_system != 'Windows' and python_version <= '3.13'", # CoreML supported on macOS and Linux - "scikit-learn>=1.3.2; platform_system != 'Windows' and python_version <= '3.13'", # CoreML k-means quantization + # ONNX export validation and inference. Jetson jobs provide custom aarch64 onnxruntime-gpu wheels. + "onnxruntime<1.20.0; python_version < '3.11' and (platform_machine != 'aarch64' or platform_system != 'Linux')", + "onnxruntime>=1.20.0; python_version >= '3.11'", + "nncf>=2.14.0,<3.0.0; python_version >= '3.9'", # OpenVINO INT8 export "openvino>=2024.0.0", # OpenVINO export + "packaging>=26.0; platform_machine == 'aarch64' and platform_system == 'Linux' and python_version >= '3.9'", # IMX export bug +] +export-legacy-torch = [ + "numpy<2", +] +export-tensorflow = [ + "numpy<2.0.0; python_version < '3.13'", # TF 2.20 compatibility "tensorflow>=2.0.0,<=2.19.0; python_version < '3.13'", # TF bug https://github.com/ultralytics/ultralytics/issues/5161 "tensorflow>2.19.0; python_version >= '3.13'", "tensorflowjs>=2.0.0", # TF.js export, automatically installs tensorflow @@ -107,10 +114,31 @@ export = [ "tensorstore>=0.1.63; platform_machine == 'aarch64' and python_version >= '3.9'", # for TF Raspberry Pi exports "h5py!=3.11.0; platform_machine == 'aarch64'", # fix h5py build issues due to missing aarch64 wheels in 3.11 release "setuptools<=81.0.0", # pin due to >=82.0.0 breaking tensorflow.js package - "packaging>=26.0; platform_machine == 'aarch64' and platform_system == 'Linux' and python_version >= '3.9'", # IMX export bug +] +export-coreml = [ + "numpy<=2.3.5; python_version >= '3.13'", # pin until CoreML>9.0 new release + "coremltools>=9.0; platform_system != 'Windows' and python_version <= '3.13'", # CoreML supported on macOS and Linux + "scikit-learn>=1.3.2; platform_system != 'Windows' and python_version <= '3.13'", # CoreML k-means quantization +] +export-executorch = [ + "executorch", + "numpy<=2.3.5", +] +export-deepx = [ + "dx_com", + "numpy<2", +] +export = [ + "ultralytics[export-base]", + "ultralytics[export-tensorflow]", + "ultralytics[export-coreml]", ] solutions = [ "shapely>=2.0.0", # shapely for point and polygon data matching + "lap>=0.5.12", # for solution tracking linear assignment + "faiss-cpu", # for similarity search + "openai-clip>=1.0.1", # for similarity search text/image embeddings + "setuptools>=80.0.0,<81.0.0; python_version >= '3.9'", # openai-clip imports pkg_resources.packaging, removed in setuptools>=81 "streamlit>=1.51.0; python_version >= '3.10'", # for live inference on web browser, i.e `yolo streamlit-predict` "streamlit>=1.29.0,<1.51.0; python_version < '3.10' and (python_version < '3.9' or platform_machine != 'aarch64' or platform_system != 'Linux')", "flask>=3.0.1", # for similarity search solution diff --git a/tests/conftest.py b/tests/conftest.py index afcb955b..679db92f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,27 @@ def solution_assets(): def pytest_addoption(parser): """Add custom command-line options to pytest.""" parser.addoption("--slow", action="store_true", default=False, help="Run slow tests") + parser.addoption( + "--export-env", + default=None, + help="Run only export tests assigned to this export environment id.", + ) + + +def _export_format_from_item(item, formats): + """Infer the export format covered by a tests/test_exports.py item.""" + if Path(str(item.fspath)).name != "test_exports.py": + return None + name = getattr(item, "originalname", None) or item.name.split("[", 1)[0] + if name == "test_torch2onnx_serializes_concurrent_exports": + return "onnx" + if not name.startswith("test_export_"): + return None + + suffix = name[len("test_export_") :] + return next( + (fmt for fmt in sorted(formats, key=len, reverse=True) if suffix == fmt or suffix.startswith(f"{fmt}_")), None + ) def pytest_collection_modifyitems(config, items): @@ -42,6 +63,18 @@ def pytest_collection_modifyitems(config, items): # Remove the item entirely from the list of test items if it's marked as 'slow' items[:] = [item for item in items if "slow" not in item.keywords] + export_env = config.getoption("--export-env") + if not export_env: + return + + from ultralytics.engine.exporter import export_formats + + env_by_format = dict(zip(export_formats()["Argument"], export_formats()["Env"])) + for item in items: + fmt = _export_format_from_item(item, env_by_format) + if fmt and env_by_format.get(fmt) != export_env: + item.add_marker(pytest.mark.skip(reason=f"export format '{fmt}' belongs to env '{env_by_format[fmt]}'")) + def pytest_sessionstart(session): """Initialize session configurations for pytest. diff --git a/tests/test_cli.py b/tests/test_cli.py index 901d6bef..ba5060ee 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,7 +8,7 @@ import pytest from PIL import Image from tests import CUDA_DEVICE_COUNT, CUDA_IS_AVAILABLE, MODELS, TASK_MODEL_DATA -from ultralytics.utils import ARM64, ASSETS, IS_RASPBERRYPI, LINUX, WEIGHTS_DIR, checks +from ultralytics.utils import ARM64, ASSETS, DATASETS_DIR, IS_RASPBERRYPI, LINUX, WEIGHTS_DIR, checks from ultralytics.utils.torch_utils import TORCH_1_11 @@ -23,6 +23,7 @@ def test_special_modes() -> None: run("yolo checks") run("yolo version") run("yolo settings reset") + run(f"yolo settings weights_dir={WEIGHTS_DIR} datasets_dir={DATASETS_DIR}") run("yolo cfg") diff --git a/tests/test_exports.py b/tests/test_exports.py index dc417dee..9dfa9dbd 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -16,6 +16,7 @@ import torch from tests import SOURCE from ultralytics import YOLO from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS +from ultralytics.engine.exporter import EXPORT_ENVS, export_formats from ultralytics.utils import ( ARM64, IS_DOCKER, @@ -406,6 +407,15 @@ def test_export_imx(isolated_model): YOLO(file)(SOURCE, imgsz=32) +@pytest.mark.slow +@pytest.mark.skipif(not LINUX or ARM64, reason="RKNN export only supported on non-aarch64 Linux") +def test_export_rknn(isolated_model): + """Test YOLO export to RKNN format.""" + file = YOLO(isolated_model).export(format="rknn", imgsz=32) + assert next(Path(file).rglob("*.rknn"), None), f"RKNN export failed, no RKNN model found in: {file}" + shutil.rmtree(file, ignore_errors=True) + + # @pytest.mark.skipif(True, reason="Disabled for debugging ruamel.yaml installation required by executorch") @pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0") @pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows") @@ -496,3 +506,15 @@ def test_export_qnn(isolated_model): assert next(Path(file).rglob("*_qnn.onnx"), None), f"QNN export failed, no context binary found in: {file}" # Note: on-device inference is not exercised here as it requires Qualcomm Snapdragon hardware shutil.rmtree(file, ignore_errors=True) # cleanup + + +@pytest.mark.parametrize("env", [k for k, v in EXPORT_ENVS.items() if k != "base" or v["smoke"]]) +def test_export_env_has_smoke(env): + """Ensure every non-base export environment declares a build-time smoke export.""" + assert EXPORT_ENVS[env]["smoke"], f"export env '{env}' has no smoke command" + + +def test_every_format_env_is_registered(): + """Ensure every export format points at a registered export environment.""" + for fmt, env in zip(export_formats()["Argument"], export_formats()["Env"]): + assert env in EXPORT_ENVS, f"format '{fmt}' references unknown env '{env}'" diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index 2dcb3e93..c47c31ea 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license -__version__ = "8.4.57" +__version__ = "8.4.58" import importlib import os diff --git a/ultralytics/engine/exporter.py b/ultralytics/engine/exporter.py index 78456eb0..5c385317 100644 --- a/ultralytics/engine/exporter.py +++ b/ultralytics/engine/exporter.py @@ -131,10 +131,19 @@ from ultralytics.utils.torch_utils import ( def export_formats(): """Return a dictionary of Ultralytics YOLO export formats.""" + # Format, Argument, Suffix, CPU, GPU, Arguments, Env x = [ - ["PyTorch", "-", ".pt", True, True, []], - ["TorchScript", "torchscript", ".torchscript", True, True, ["batch", "optimize", "half", "nms", "dynamic"]], - ["ONNX", "onnx", ".onnx", True, True, ["batch", "dynamic", "half", "opset", "simplify", "nms"]], + ["PyTorch", "-", ".pt", True, True, [], "base"], + [ + "TorchScript", + "torchscript", + ".torchscript", + True, + True, + ["batch", "optimize", "half", "nms", "dynamic"], + "base", + ], + ["ONNX", "onnx", ".onnx", True, True, ["batch", "dynamic", "half", "opset", "simplify", "nms"], "base"], [ "OpenVINO", "openvino", @@ -142,6 +151,7 @@ def export_formats(): True, False, ["batch", "data", "dynamic", "half", "int8", "nms", "fraction"], + "base", ], [ "TensorRT", @@ -150,8 +160,9 @@ def export_formats(): False, True, ["batch", "data", "dynamic", "half", "int8", "simplify", "nms", "fraction"], + "base", ], - ["CoreML", "coreml", ".mlpackage", True, False, ["batch", "dynamic", "half", "int8", "nms"]], + ["CoreML", "coreml", ".mlpackage", True, False, ["batch", "dynamic", "half", "int8", "nms"], "coreml"], [ "TensorFlow SavedModel", "saved_model", @@ -159,22 +170,164 @@ def export_formats(): True, True, ["batch", "data", "fraction", "int8", "keras", "nms"], + "tensorflow", ], - ["TensorFlow GraphDef", "pb", ".pb", True, True, ["batch"]], - ["TensorFlow Lite", "tflite", ".tflite", True, False, ["batch", "data", "half", "int8", "nms", "fraction"]], - ["TensorFlow Edge TPU", "edgetpu", "_edgetpu.tflite", True, False, ["data", "fraction", "int8"]], - ["TensorFlow.js", "tfjs", "_web_model", True, False, ["batch", "data", "fraction", "half", "int8", "nms"]], - ["PaddlePaddle", "paddle", "_paddle_model", True, True, ["batch"]], - ["MNN", "mnn", ".mnn", True, True, ["batch", "half", "int8"]], - ["NCNN", "ncnn", "_ncnn_model", True, True, ["batch", "half"]], - ["IMX", "imx", "_imx_model", True, True, ["data", "int8", "fraction", "nms"]], - ["RKNN", "rknn", "_rknn_model", False, False, ["batch", "name"]], - ["ExecuTorch", "executorch", "_executorch_model", True, False, ["batch"]], - ["Axelera AI", "axelera", "_axelera_model", False, False, ["batch", "int8", "fraction", "data"]], - ["DEEPX", "deepx", "_deepx_model", False, False, ["data", "int8", "optimize"]], - ["Qualcomm QNN", "qnn", "_qnn_model", False, False, ["batch", "name", "int8", "fraction", "data"]], + ["TensorFlow GraphDef", "pb", ".pb", True, True, ["batch"], "tensorflow"], + [ + "TensorFlow Lite", + "tflite", + ".tflite", + True, + False, + ["batch", "data", "half", "int8", "nms", "fraction"], + "tensorflow", + ], + ["TensorFlow Edge TPU", "edgetpu", "_edgetpu.tflite", True, False, ["data", "fraction", "int8"], "tensorflow"], + [ + "TensorFlow.js", + "tfjs", + "_web_model", + True, + False, + ["batch", "data", "fraction", "half", "int8", "nms"], + "tensorflow", + ], + ["PaddlePaddle", "paddle", "_paddle_model", True, True, ["batch"], "base"], + ["MNN", "mnn", ".mnn", True, True, ["batch", "half", "int8"], "mnn"], + ["NCNN", "ncnn", "_ncnn_model", True, True, ["batch", "half"], "ncnn"], + ["IMX", "imx", "_imx_model", True, True, ["data", "int8", "fraction", "nms"], "isolated-imx"], + ["RKNN", "rknn", "_rknn_model", False, False, ["batch", "name"], "isolated-rknn"], + ["ExecuTorch", "executorch", "_executorch_model", True, False, ["batch"], "executorch"], + [ + "Axelera AI", + "axelera", + "_axelera_model", + False, + False, + ["batch", "int8", "fraction", "data"], + "isolated-axelera", + ], + ["DEEPX", "deepx", "_deepx_model", False, False, ["data", "int8", "optimize"], "isolated-deepx"], + ["Qualcomm QNN", "qnn", "_qnn_model", False, False, ["batch", "name", "int8", "fraction", "data"], "base"], ] - return dict(zip(["Format", "Argument", "Suffix", "CPU", "GPU", "Arguments"], zip(*x))) + return dict(zip(["Format", "Argument", "Suffix", "CPU", "GPU", "Arguments", "Env"], zip(*x))) + + +EXPORT_ENVS = { + "base": { + "python": None, + "extras": ["export-base"], + "torch": None, + "requirements": [], + "indexes": [], + "env": {}, + "smoke": [], + }, + "tensorflow": { + "python": "3.12", + "extras": ["export-base", "export-tensorflow"], + "torch": None, + "requirements": [ + "onnx2tf>=1.26.3,<1.29.0", + "tf_keras<=2.19.0", + "sng4onnx>=1.0.1", + "onnx_graphsurgeon>=0.3.26", + "ai-edge-litert>=1.2.0", + "onnxruntime", + "protobuf>=5", + ], + "indexes": [ + ("--extra-index-url", "https://download.pytorch.org/whl/cpu"), + ("--extra-index-url", "https://pypi.ngc.nvidia.com"), + ], + "env": {}, + "smoke": ["yolo export format=tflite model=yolo11n.pt imgsz=32"], + }, + "coreml": { + "python": "3.13", + "extras": ["export-base", "export-coreml"], + "torch": ">=2.12", + "requirements": [], + "indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")], + "env": {}, + "smoke": ["yolo export format=coreml model=yolo11n.pt imgsz=32"], + }, + "mnn": { + "python": "3.13", + "extras": ["export-base"], + "torch": None, + "requirements": ["MNN>=2.9.6"], + "indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")], + "env": {}, + "smoke": ["yolo export format=mnn model=yolo11n.pt imgsz=32"], + }, + "ncnn": { + "python": "3.13", + "extras": ["export-base"], + "torch": None, + "requirements": ["ncnn", "pnnx"], + "indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")], + "env": {}, + "smoke": ["yolo export format=ncnn model=yolo11n.pt imgsz=32"], + }, + "executorch": { + "python": "3.13", + "extras": ["export-base", "export-executorch"], + "torch": ">=2.12", + "requirements": [], + "indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")], + "env": {}, + "smoke": ["yolo export format=executorch model=yolo11n.pt imgsz=32"], + }, + "isolated-imx": { + "python": "3.11", + "extras": ["export-base"], + "torch": ">=2.9,<2.12", + "requirements": [ + "model-compression-toolkit>=2.4.1", + "edge-mdt-cl<1.1.0", + "edge-mdt-tpc>=1.2.0", + "pydantic<2.12", + "imx500-converter[pt]>=3.17.3", + ], + "indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")], + "env": {}, + "smoke": ["yolo export format=imx model=yolo11n.pt imgsz=32"], + }, + "isolated-rknn": { + "python": "3.11", + "extras": ["export-base"], + "torch": "==2.4", + "requirements": ["rknn-toolkit2>=2.3.2", "onnx>=1.16.1,<1.19.0", "setuptools<82"], + "indexes": [("--extra-index-url", "https://download.pytorch.org/whl/cpu")], + "env": {}, + "smoke": ["yolo export format=rknn model=yolo11n.pt imgsz=32"], + }, + "isolated-axelera": { + "python": "3.12", + "extras": ["export-base"], + "torch": ">=2.8,<2.12", + "requirements": ["axelera-devkit==1.6.0", "onnx>=1.12.0,<2.0.0", "onnxslim>=0.1.71"], + "indexes": [ + ("--extra-index-url", "https://download.pytorch.org/whl/cpu"), + ("--extra-index-url", "https://software.axelera.ai/artifactory/api/pypi/axelera-pypi/simple"), + ], + "env": {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "python"}, + "smoke": ["yolo export format=axelera model=yolo11n.pt imgsz=64 data=coco8.yaml"], + }, + "isolated-deepx": { + "python": "3.12", + "extras": ["export-base", "export-deepx"], + "torch": ">=2.8,<2.12", + "requirements": [], + "indexes": [ + ("--extra-index-url", "https://download.pytorch.org/whl/cpu"), + ("--find-links", "https://sdk.deepx.ai/release/dxcom/v2.3.0/index.html"), + ], + "env": {}, + "smoke": ["yolo export format=deepx model=yolo11n.pt imgsz=32"], + }, +} def validate_args(format, passed_args, valid_args): diff --git a/ultralytics/utils/benchmarks.py b/ultralytics/utils/benchmarks.py index d99c1d92..cc44155b 100644 --- a/ultralytics/utils/benchmarks.py +++ b/ultralytics/utils/benchmarks.py @@ -126,7 +126,7 @@ def benchmark( if format_arg: formats = frozenset(export_formats()["Argument"]) assert format in formats, f"Expected format to be one of {formats}, but got '{format_arg}'." - for name, format, suffix, cpu, gpu, valid_args in zip(*export_formats().values()): + for name, format, suffix, cpu, gpu, valid_args, _ in zip(*export_formats().values()): emoji, filename = "❌", None # export defaults try: if format_arg and format_arg != format: diff --git a/ultralytics/utils/checks.py b/ultralytics/utils/checks.py index 8572fc0e..c06e4925 100644 --- a/ultralytics/utils/checks.py +++ b/ultralytics/utils/checks.py @@ -525,8 +525,6 @@ def check_executorch_requirements(): check_requirements("packaging>=22.0") check_requirements("executorch", cmds=f"torch=={TORCH_VERSION.split('+')[0]}") - # Pin numpy to avoid coremltools errors with numpy>=2.4.0, must be separate - check_requirements("numpy<=2.3.5") def check_tensorrt(min_version: str = "7.0.0"):