Glenn Jocher 3950b59458
Change CI OS from cpu-latest to ubuntu-latest (#24641)
Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
2026-05-29 19:48:39 +08:00

534 lines
22 KiB
YAML

# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
# Ultralytics YOLO Continuous Integration (CI) GitHub Actions tests
name: CI
permissions:
contents: read
env:
PYTHONFAULTHANDLER: 1
on:
push:
branches: [main]
pull_request:
schedule:
- cron: "0 8 * * *" # runs at 08:00 UTC every day
workflow_dispatch:
inputs:
benchmarks:
description: "Run Benchmarks"
default: true
type: boolean
tests:
description: "Run Tests"
default: true
type: boolean
gpu:
description: "Run GPU"
default: true
type: boolean
raspberrypi:
description: "Run Raspberry Pi"
default: true
type: boolean
nvidia-jetson:
description: "Run NVIDIA Jetson"
default: true
type: boolean
conda:
description: "Run Conda"
default: true
type: boolean
jobs:
Benchmarks:
if: github.event_name != 'workflow_dispatch' || github.event.inputs.benchmarks == 'true'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
# Temporarily disable windows-latest due to https://github.com/ultralytics/ultralytics/actions/runs/13020330819/job/36319338854?pr=18921
os: [ubuntu-latest, macos-26, ubuntu-24.04-arm]
python: ["3.12"]
model: [yolo26n]
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
if: matrix.os != 'cpu-latest'
with:
python-version: ${{ matrix.python }}
- uses: astral-sh/setup-uv@v7
with:
python-version: ${{ matrix.python }}
activate-environment: ${{ matrix.os == 'cpu-latest' }}
- name: Install requirements
shell: bash # for Windows compatibility
run: |
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export]" "coverage[toml]" --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Benchmark DetectionModel
shell: bash
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}.pt' imgsz=160 verbose=0.216
- name: Benchmark ClassificationModel
shell: bash
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-cls.pt' imgsz=160 verbose=0.249
- name: Benchmark YOLOWorld DetectionModel
shell: bash
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/yolov8s-worldv2.pt' imgsz=160 verbose=0.335
- name: Benchmark SegmentationModel
shell: bash
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-seg.pt' imgsz=160 verbose=0.229
- name: Benchmark PoseModel
shell: bash
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-pose.pt' imgsz=160 verbose=0.185
- name: Benchmark OBBModel
shell: bash
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-obb.pt' imgsz=160 verbose=0.371
- name: Benchmark SemanticSegmentationModel
shell: bash
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-sem.pt' imgsz=160 verbose=0.189
- name: Merge Coverage Reports
run: |
coverage xml -o coverage-benchmarks.xml
- name: Upload Coverage Reports to CodeCov
if: github.repository == 'ultralytics/ultralytics'
uses: codecov/codecov-action@v6
with:
flags: Benchmarks
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Prune uv Cache
run: uv cache prune --ci
- name: Benchmark Summary
run: |
cat benchmarks.log
echo '```' >> $GITHUB_STEP_SUMMARY
cat benchmarks.log >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
Tests:
if: github.event_name == 'pull_request' || github.event_name == 'push'
timeout-minutes: 60
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-26, windows-latest, ubuntu-24.04-arm]
python: ["3.12"]
torch: [latest]
include:
- os: ubuntu-latest
python: "3.8" # torch 1.8.0 requires python >=3.6, <=3.9
torch: "1.8.0"
torchvision: "0.9.0"
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
if: matrix.os != 'cpu-latest'
with:
python-version: ${{ matrix.python }}
- uses: astral-sh/setup-uv@v7
with:
python-version: ${{ matrix.python }}
activate-environment: ${{ matrix.os == 'cpu-latest' }}
- name: Install requirements
shell: bash # for Windows compatibility
run: |
slow=""
torch=""
if [ "${{ matrix.torch }}" != "latest" ]; then
torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}"
fi
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Configure Ultralytics cache paths
shell: bash # for Windows compatibility
run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets"
- name: Cache test assets
uses: actions/cache@v5
with:
path: ${{ github.workspace }}/.ultralytics_cache
key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }}
restore-keys: |
ultralytics-test-assets-${{ runner.os }}-
- name: Pre-cache test assets
shell: bash # for Windows compatibility
run: python tests/cache_test_assets.py
- name: Pytest tests
shell: bash # for Windows compatibility
run: pytest -n auto --dist=loadfile --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx
- name: Upload Coverage Reports to CodeCov
if: github.repository == 'ultralytics/ultralytics' # && matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
uses: codecov/codecov-action@v6
with:
flags: Tests
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Prune uv Cache
run: uv cache prune --ci
SlowTests:
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.tests == 'true') || github.event_name == 'schedule'
timeout-minutes: 360
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-26, windows-latest, ubuntu-24.04-arm]
python: ["3.12"]
torch: [latest]
include:
- os: ubuntu-latest
python: "3.8" # torch 1.8.0 requires python >=3.6, <=3.9
torch: "1.8.0"
torchvision: "0.9.0"
- os: ubuntu-latest
python: "3.9"
torch: "1.9.0"
torchvision: "0.10.0"
- os: ubuntu-latest
python: "3.9"
torch: "1.10.0"
torchvision: "0.11.0"
- os: ubuntu-latest
python: "3.10"
torch: "1.11.0"
torchvision: "0.12.0"
- os: ubuntu-latest
python: "3.10"
torch: "1.12.0"
torchvision: "0.13.0"
- os: ubuntu-latest
python: "3.10"
torch: "1.13.0"
torchvision: "0.14.0"
- os: ubuntu-latest
python: "3.11"
torch: "2.0.0"
torchvision: "0.15.0"
- os: ubuntu-latest
python: "3.11"
torch: "2.1.0"
torchvision: "0.16.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.2.0"
torchvision: "0.17.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.3.0"
torchvision: "0.18.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.4.0"
torchvision: "0.19.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.5.0"
torchvision: "0.20.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.6.0"
torchvision: "0.21.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.7.0"
torchvision: "0.22.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.8.0"
torchvision: "0.23.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.9.0"
torchvision: "0.24.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.10.0"
torchvision: "0.25.0"
- os: ubuntu-latest
python: "3.12"
torch: "2.11.0"
torchvision: "0.26.0"
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
if: matrix.os != 'cpu-latest'
with:
python-version: ${{ matrix.python }}
- uses: astral-sh/setup-uv@v7
with:
python-version: ${{ matrix.python }}
activate-environment: ${{ matrix.os == 'cpu-latest' }}
- name: Install requirements
shell: bash # for Windows compatibility
run: |
torch=""
if [ "${{ matrix.torch }}" != "latest" ]; then
torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}"
fi
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch faster-coco-eval mlflow pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Configure Ultralytics cache paths
shell: bash # for Windows compatibility
run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets"
- name: Cache test assets
uses: actions/cache@v5
with:
path: ${{ github.workspace }}/.ultralytics_cache
key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }}
restore-keys: |
ultralytics-test-assets-${{ runner.os }}-
- name: Pre-cache test assets
shell: bash # for Windows compatibility
run: python tests/cache_test_assets.py
- name: Pytest tests
uses: ultralytics/actions/retry@main
with:
shell: bash # for Windows compatibility
# -n 2 limits parallelism for resource-heavy export tests on shared runners
run: pytest -vv -s -n 2 --dist=loadfile --slow --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx --deselect tests/test_exports.py::test_export_mnn --deselect tests/test_exports.py::test_export_ncnn --deselect tests/test_exports.py::test_export_executorch
retries: 1 # Retry once after initial attempt (2 total runs)
timeout_minutes: 180
retry_delay_seconds: 60
- name: Prune uv Cache
run: uv cache prune --ci
GPU:
if: github.repository == 'ultralytics/ultralytics' && (github.event_name != 'workflow_dispatch' || github.event.inputs.gpu == 'true')
timeout-minutes: 60
runs-on: gpu-latest
steps:
- uses: actions/checkout@v6
- uses: astral-sh/setup-uv@v7
with:
activate-environment: true
- name: Install requirements
run: |
# Install TensorRT into the job venv so uv can reuse the runner image cache.
uv pip install -e . pytest-cov nvidia-ml-py "torch<2.11" "tensorrt-cu12>=7.0.0,!=10.2.0" # TODO: remove torch pin after GPU runner driver update to CUDA 13.0
env:
PIP_BREAK_SYSTEM_PACKAGES: 1
- name: Check environment
run: |
yolo checks
uv pip list
- name: Pytest tests
run: |
slow=""
if [[ "${{ github.event_name }}" =~ ^(schedule|workflow_dispatch)$ ]]; then
slow="--slow"
fi
pytest $slow --cov=ultralytics/ --cov-report xml tests/test_cuda.py -sv
env:
PIP_BREAK_SYSTEM_PACKAGES: 1
- name: Upload Coverage Reports to CodeCov
uses: codecov/codecov-action@v6
with:
flags: GPU
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
RaspberryPi:
if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.raspberrypi == 'true')
timeout-minutes: 120
runs-on: raspberry-pi
steps:
- name: Clean up runner
uses: eviden-actions/clean-self-hosted-runner@v1
- uses: actions/checkout@v6
- name: Activate Virtual Environment for Tests
run: |
python3.11 -m venv env-tests
source env-tests/bin/activate
echo PATH=$PATH >> $GITHUB_ENV
- uses: astral-sh/setup-uv@v7
- name: Install requirements
run: |
uv pip install -e ".[export]" pytest pytest-xdist mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Configure Ultralytics cache paths
run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets"
- name: Cache test assets
uses: actions/cache@v5
with:
path: ${{ github.workspace }}/.ultralytics_cache
key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }}
restore-keys: |
ultralytics-test-assets-${{ runner.os }}-
- name: Pre-cache test assets
run: python tests/cache_test_assets.py
- name: Pytest tests
run: pytest -n auto --dist=loadfile --slow tests/ --deselect tests/test_exports.py::test_export_imx
- name: Activate Virtual Environment for Benchmarks
run: |
python3.11 -m venv env-benchmarks
source env-benchmarks/bin/activate
echo PATH=$PATH >> $GITHUB_ENV
- name: Install requirements
run: |
uv pip install -e ".[export]" pytest mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Benchmark DetectionModel
run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n.pt' imgsz=160 verbose=0.216
- name: Benchmark ClassificationModel
run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-cls.pt' imgsz=160 verbose=0.249
- name: Benchmark YOLOWorld DetectionModel
run: python -m ultralytics.cfg.__init__ benchmark model='yolov8s-worldv2.pt' imgsz=160 verbose=0.335
- name: Benchmark SegmentationModel
run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-seg.pt' imgsz=160 verbose=0.229
- name: Benchmark PoseModel
run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-pose.pt' imgsz=160 verbose=0.185
- name: Benchmark OBBModel
run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-obb.pt' imgsz=160 verbose=0.371
- name: Benchmark SemanticSegmentationModel
run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-sem.pt' imgsz=160 verbose=0.189
- name: Benchmark Summary
run: |
cat benchmarks.log
echo "$(cat benchmarks.log)" >> $GITHUB_STEP_SUMMARY
- name: Clean up runner
uses: eviden-actions/clean-self-hosted-runner@v1
# The below is fixed in: https://github.com/ultralytics/ultralytics/pull/15987
# - name: Reboot # run a reboot command in the background to free resources for next run and not crash main thread
# run: sudo bash -c "sleep 10; reboot" &
NVIDIA_Jetson:
if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.nvidia-jetson == 'true')
timeout-minutes: 120
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
name: [JetPack6.2, JetPack5.1.2]
include:
- name: JetPack6.2
python: "3.10"
runner: jetson-jp62
numpy: "1.26.4"
torch_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torch-2.5.0a0+872d972e41.nv24.08-cp310-cp310-linux_aarch64.whl"
torchvision_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torchvision-0.20.0a0+afc54f7-cp310-cp310-linux_aarch64.whl"
onnxruntime_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/onnxruntime_gpu-1.20.0-cp310-cp310-linux_aarch64.whl"
- name: JetPack5.1.2
python: "3.8"
runner: jetson-jp512
numpy: "1.23.5"
torch_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torch-2.2.0-cp38-cp38-linux_aarch64.whl"
torchvision_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torchvision-0.17.2+c1d70fe-cp38-cp38-linux_aarch64.whl"
onnxruntime_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/onnxruntime_gpu-1.16.3-cp38-cp38-linux_aarch64.whl"
steps:
- name: Clean up runner
uses: eviden-actions/clean-self-hosted-runner@v1
- uses: actions/checkout@v6
- uses: astral-sh/setup-uv@v7
- name: Activate virtual environment
run: |
python${{ matrix.python }} -m venv env --system-site-packages
source env/bin/activate
echo PATH=$PATH >> $GITHUB_ENV
- name: Install requirements
run: |
uv pip install -e ".[export]" pytest \
"${{ matrix.torch_whl }}" "${{ matrix.torchvision_whl }}" "${{ matrix.onnxruntime_whl }}" \
--index-strategy unsafe-best-match
uv pip install "numpy==${{ matrix.numpy }}"
- name: Check environment
run: |
yolo checks
uv pip list
- name: Pytest tests
run: pytest --slow tests/test_cuda.py
- name: Clean up runner
uses: eviden-actions/clean-self-hosted-runner@v1
Conda:
if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.conda == 'true')
timeout-minutes: 120
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
python: ["3.12"]
defaults:
run:
shell: bash -el {0}
steps:
- uses: astral-sh/setup-uv@v7
- uses: conda-incubator/setup-miniconda@v4
with:
python-version: ${{ matrix.python }}
channels: conda-forge,defaults
channel-priority: true
activate-environment: anaconda-client-env
- name: Install Ultralytics package from conda-forge
run: conda install -c pytorch -c conda-forge pytorch-cpu torchvision ultralytics "openvino<2026"
- name: Install pip packages
run: uv pip install pytest
- name: Check environment
run: conda list
- name: Test benchmark CLI
run: yolo benchmark model=yolo26n.pt data='coco8.yaml' imgsz=640 format=onnx
- name: Test Python
# Note this step must use the updated default bash environment, not a Python environment
run: |
python -c "
from ultralytics import YOLO
model = YOLO('yolo26n.pt')
results = model.train(data='coco8.yaml', epochs=3, imgsz=160)
results = model.val(imgsz=160)
results = model.predict(imgsz=160)
results = model.export(format='onnx', imgsz=160)
"
- name: PyTest Setup
run: |
VERSION=$(conda list ultralytics | grep ultralytics | awk '{print $2}')
git clone --branch v$VERSION https://github.com/ultralytics/ultralytics.git
- name: test_cli.py
run: pytest ultralytics/tests/test_cli.py -v -s
- name: test_cuda.py
run: pytest ultralytics/tests/test_cuda.py -v -s
- name: test_engine.py
run: pytest ultralytics/tests/test_engine.py -v -s
- name: test_exports.py
run: pytest ultralytics/tests/test_exports.py -v -s --deselect ultralytics/tests/test_exports.py::test_export_imx
- name: test_integrations.py
run: pytest ultralytics/tests/test_integrations.py -v -s
- name: test_solutions.py
run: pytest ultralytics/tests/test_solutions.py -v -s
# WARNING: tests hang here for unknown reasons https://github.com/ultralytics/ultralytics/pull/21577
# - name: test_python.py
# run: pytest ultralytics/tests/test_python.py -vv -s
Summary:
runs-on: ubuntu-latest
needs: [Benchmarks, Tests, SlowTests, GPU, RaspberryPi, NVIDIA_Jetson, Conda]
if: always()
steps:
- name: Check for failure and notify
if: (needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1'
uses: slackapi/slack-github-action@v3.0.3
with:
webhook-type: incoming-webhook
webhook: ${{ secrets.SLACK_WEBHOOK_URL_YOLO }}
payload: |
text: "<!channel> GitHub Actions error for ${{ github.workflow }} ❌\n\n\n*Repository:* https://github.com/${{ github.repository }}\n*Action:* https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Author:* ${{ github.actor }}\n*Event:* ${{ github.event_name }}\n"