Enable parallel CI tests via pytest-xdist and fix export/train race conditions (#24565)

Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
Jing Qiu 2026-05-26 03:07:18 +08:00 committed by GitHub
parent b30c4d97c4
commit b97bfe2997
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 266 additions and 129 deletions

View File

@ -145,14 +145,27 @@ jobs:
if [ "${{ matrix.torch }}" != "latest" ]; then
torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}"
fi
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Configure Ultralytics cache paths
shell: bash # for Windows compatibility
run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets"
- name: Cache test assets
uses: actions/cache@v4
with:
path: ${{ github.workspace }}/.ultralytics_cache
key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }}
restore-keys: |
ultralytics-test-assets-${{ runner.os }}-
- name: Pre-cache test assets
shell: bash # for Windows compatibility
run: python tests/cache_test_assets.py
- name: Pytest tests
shell: bash # for Windows compatibility
run: pytest --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx
run: pytest -n auto --dist=loadfile --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx
- name: Upload Coverage Reports to CodeCov
if: github.repository == 'ultralytics/ultralytics' # && matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
uses: codecov/codecov-action@v6
@ -263,16 +276,30 @@ jobs:
if [ "${{ matrix.torch }}" != "latest" ]; then
torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}"
fi
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch faster-coco-eval mlflow pytest-cov --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch faster-coco-eval mlflow pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Configure Ultralytics cache paths
shell: bash # for Windows compatibility
run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets"
- name: Cache test assets
uses: actions/cache@v4
with:
path: ${{ github.workspace }}/.ultralytics_cache
key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }}
restore-keys: |
ultralytics-test-assets-${{ runner.os }}-
- name: Pre-cache test assets
shell: bash # for Windows compatibility
run: python tests/cache_test_assets.py
- name: Pytest tests
uses: ultralytics/actions/retry@main
with:
shell: bash # for Windows compatibility
run: pytest --slow --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx
# -n 2 limits parallelism for resource-heavy export tests on shared runners
run: pytest -vv -s -n 2 --dist=loadfile --slow --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx --deselect tests/test_exports.py::test_export_mnn --deselect tests/test_exports.py::test_export_ncnn --deselect tests/test_exports.py::test_export_executorch
retries: 1 # Retry once after initial attempt (2 total runs)
timeout_minutes: 180
retry_delay_seconds: 60
@ -330,13 +357,24 @@ jobs:
- uses: astral-sh/setup-uv@v7
- name: Install requirements
run: |
uv pip install -e ".[export]" pytest mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
uv pip install -e ".[export]" pytest pytest-xdist mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match
- name: Check environment
run: |
yolo checks
uv pip list
- name: Configure Ultralytics cache paths
run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets"
- name: Cache test assets
uses: actions/cache@v4
with:
path: ${{ github.workspace }}/.ultralytics_cache
key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }}
restore-keys: |
ultralytics-test-assets-${{ runner.os }}-
- name: Pre-cache test assets
run: python tests/cache_test_assets.py
- name: Pytest tests
run: pytest --slow tests/ --deselect tests/test_exports.py::test_export_imx
run: pytest -n auto --dist=loadfile --slow tests/ --deselect tests/test_exports.py::test_export_imx
- name: Activate Virtual Environment for Benchmarks
run: |
python3.11 -m venv env-benchmarks
@ -447,14 +485,8 @@ jobs:
run: uv pip install pytest
- name: Check environment
run: conda list
- name: Test CLI
run: |
yolo predict model=yolo26n.pt imgsz=320
yolo train model=yolo26n.pt data=coco8.yaml epochs=1 imgsz=32
yolo val model=yolo26n.pt data=coco8.yaml imgsz=32
yolo export model=yolo26n.pt format=torchscript imgsz=160
yolo benchmark model=yolo26n.pt data='coco8.yaml' imgsz=640 format=onnx
yolo solutions
- name: Test benchmark CLI
run: yolo benchmark model=yolo26n.pt data='coco8.yaml' imgsz=640 format=onnx
- name: Test Python
# Note this step must use the updated default bash environment, not a Python environment
run: |

3
.gitignore vendored
View File

@ -203,3 +203,6 @@ calibration_*.npy
# Training logs
args.yaml
results.csv
# CI test asset cache
.ultralytics_cache/

View File

@ -82,6 +82,7 @@ dev = [
"ipython",
"pytest",
"pytest-cov",
"pytest-xdist>=3.0.0",
"coverage[toml]",
"zensical>=0.0.15; python_version >= '3.10'",
"mkdocs-ultralytics-plugin>=0.2.4", # for meta descriptions and images, dates and authors

View File

@ -10,14 +10,24 @@ SOURCE = ASSETS / "bus.jpg"
SOURCES_LIST = [ASSETS / "bus.jpg", ASSETS, ASSETS / "*", ASSETS / "**/*.jpg"]
CUDA_IS_AVAILABLE = checks.cuda_is_available()
CUDA_DEVICE_COUNT = checks.cuda_device_count()
TASK_MODEL_DATA = [(task, WEIGHTS_DIR / TASK2MODEL[task], TASK2DATA[task]) for task in TASKS]
MODELS = frozenset([*list(TASK2MODEL.values()), "yolo11n-grayscale.pt"])
TASK_MODEL_DATA = sorted([(task, WEIGHTS_DIR / TASK2MODEL[task], TASK2DATA[task]) for task in TASKS])
MODELS = sorted([*list(TASK2MODEL.values()), "yolo11n-grayscale.pt"])
SOLUTION_ASSETS = {
"demo_video": "solutions_ci_demo.mp4",
"crop_video": "decelera_landscape_min.mov",
"pose_video": "solution_ci_pose_demo.mp4",
"parking_video": "solution_ci_parking_demo.mp4",
"vertical_video": "solution_vertical_demo.mp4",
"parking_areas": "solution_ci_parking_areas.json",
"parking_model": "solutions_ci_parking_model.pt",
}
__all__ = (
"CFG",
"CUDA_DEVICE_COUNT",
"CUDA_IS_AVAILABLE",
"MODEL",
"SOLUTION_ASSETS",
"SOURCE",
"SOURCES_LIST",
)

View File

@ -0,0 +1,85 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
"""Pre-download shared test assets to avoid race conditions under pytest-xdist.
Run this script once before `pytest -n auto` to ensure all model weights,
datasets, and solution assets are already cached locally. Each xdist worker
can then reuse existing files instead of competing to download the same remote resources.
"""
import shutil
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests import MODEL, SOLUTION_ASSETS
from ultralytics.cfg import TASK2CALIBRATIONDATA, TASK2DATA, TASK2MODEL
from ultralytics.data.utils import check_cls_dataset, check_det_dataset
from ultralytics.utils import ASSETS_URL, LOGGER, WEIGHTS_DIR
from ultralytics.utils.downloads import attempt_download_asset, safe_download
WEIGHTS = [
*TASK2MODEL.values(),
"yolo11n-grayscale.pt",
"rtdetr-l.pt",
"FastSAM-s.pt",
"mobile_sam.pt",
"sam2.1_b.pt",
"yolov8s-world.pt",
"yolov8s-worldv2.pt",
"yoloe-11s-seg.pt",
"yoloe-11s-seg-pf.pt",
]
DATASETS = [
*TASK2DATA.values(),
*TASK2CALIBRATIONDATA.values(),
"coco8-grayscale.yaml",
"coco8-multispectral.yaml",
"coco12-formats.yaml",
]
def cache_weights() -> None:
"""Download all model weights used by tests."""
LOGGER.info("[cache] Downloading model weights ...")
for w in WEIGHTS:
attempt_download_asset(WEIGHTS_DIR / w)
if not MODEL.exists():
MODEL.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(WEIGHTS_DIR / "yolo26n.pt", MODEL)
LOGGER.info("[cache] Weights done.")
def cache_datasets() -> None:
"""Download / extract all datasets used by tests."""
LOGGER.info("[cache] Downloading datasets ...")
for ds in DATASETS:
if ds.startswith("imagenet"):
check_cls_dataset(ds)
else:
check_det_dataset(ds, autodownload=True)
LOGGER.info("[cache] Datasets done.")
def cache_solution_assets() -> None:
"""Download solution test assets (videos, parking json, etc.)."""
LOGGER.info("[cache] Downloading solution assets ...")
cache_dir = WEIGHTS_DIR / "solution_assets"
cache_dir.mkdir(parents=True, exist_ok=True)
for asset in SOLUTION_ASSETS.values():
dst = cache_dir / asset
if not dst.exists():
safe_download(url=f"{ASSETS_URL}/{asset}", dir=cache_dir)
LOGGER.info("[cache] Solution assets done.")
def main() -> None:
"""Main function to orchestrate caching of all test assets."""
cache_weights()
cache_datasets()
cache_solution_assets()
LOGGER.info("[cache] All test assets are ready.")
if __name__ == "__main__":
main()

View File

@ -3,46 +3,25 @@
import shutil
from pathlib import Path
import numpy.testing # noqa: F401 # Pre-import before any test can corrupt numpy via in-place upgrade
import pytest
@pytest.fixture(scope="session")
def solution_assets():
"""Session-scoped fixture to cache solution test assets.
Lazily downloads solution assets into a persistent directory (WEIGHTS_DIR/solution_assets) and returns a callable
that resolves asset names to cached paths.
"""
"""Return cached solution asset paths by name."""
from tests import SOLUTION_ASSETS
from ultralytics.utils import ASSETS_URL, WEIGHTS_DIR
from ultralytics.utils.downloads import safe_download
# Use persistent directory alongside weights
cache_dir = WEIGHTS_DIR / "solution_assets"
cache_dir.mkdir(parents=True, exist_ok=True)
# Define all assets needed for solution tests
assets = {
# Videos
"demo_video": "solutions_ci_demo.mp4",
"crop_video": "decelera_landscape_min.mov",
"pose_video": "solution_ci_pose_demo.mp4",
"parking_video": "solution_ci_parking_demo.mp4",
"vertical_video": "solution_vertical_demo.mp4",
# Parking manager files
"parking_areas": "solution_ci_parking_areas.json",
"parking_model": "solutions_ci_parking_model.pt",
}
asset_paths = {}
def get_asset(name):
"""Return the cached path for a named solution asset, downloading it on first use."""
if name not in asset_paths:
asset_path = cache_dir / assets[name]
if not asset_path.exists():
safe_download(url=f"{ASSETS_URL}/{asset_path.name}", dir=cache_dir)
asset_paths[name] = asset_path
return asset_paths[name]
asset_path = cache_dir / SOLUTION_ASSETS[name]
if not asset_path.exists():
safe_download(url=f"{ASSETS_URL}/{asset_path.name}", dir=cache_dir)
return asset_path
return get_asset
@ -78,17 +57,38 @@ def pytest_sessionstart(session):
init_seeds()
def pytest_terminal_summary(terminalreporter, exitstatus, config):
@pytest.fixture
def isolated_model(tmp_path):
"""Provide an isolated copy of the test model to prevent export file races under pytest-xdist.
When multiple xdist workers run export tests simultaneously, they derive output filenames from the model path (e.g.,
model.onnx, model.torchscript). Using the same MODEL path causes workers to overwrite each other's
intermediate/export files. This fixture copies the shared model to a per-test temporary directory so each test
exports to a unique path.
"""
from tests import MODEL
if not Path(MODEL).exists():
from ultralytics.utils.downloads import attempt_download_asset
MODEL.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(attempt_download_asset("yolo26n.pt"), MODEL)
dst = tmp_path / "model.pt"
shutil.copy(MODEL, dst)
return str(dst)
def pytest_sessionfinish(session, exitstatus):
"""Cleanup operations after pytest session.
This function is automatically called by pytest at the end of the entire test session. It removes certain files and
directories used during testing.
Args:
terminalreporter: The terminal reporter object used for terminal output.
exitstatus (int): The exit status of the test run.
config: The pytest config object.
Runs only on the pytest controller (or serial run), skipping xdist workers to avoid race conditions where one worker
deletes shared assets while another is still reading them.
"""
# Skip on xdist workers - only the controller should clean up shared resources
if hasattr(session.config, "workerinput"):
return
from ultralytics.utils import WEIGHTS_DIR
# Remove files
@ -97,6 +97,5 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
Path(file).unlink(missing_ok=True)
# Remove directories
models = [path for x in {"*.mlpackage", "*_openvino_model"} for path in WEIGHTS_DIR.rglob(x)]
for directory in [WEIGHTS_DIR / "solution_assets", WEIGHTS_DIR / "path with spaces", *models]:
for directory in [path for x in {"*.mlpackage", "*_openvino_model"} for path in WEIGHTS_DIR.rglob(x)]:
shutil.rmtree(directory, ignore_errors=True)

View File

@ -36,35 +36,34 @@ def test_train(task: str, model: str, data: str) -> None:
def test_val(task: str, model: str, data: str) -> None:
"""Test YOLO validation process for specified task, model, and data using a shell command."""
for end2end in {False, True}:
run(
f"yolo val {task} model={model} data={data} imgsz=32 save_txt save_json visualize end2end={end2end} max_det=100 agnostic_nms"
)
run(f"yolo val {task} model={model} data={data} imgsz=32 end2end={end2end} max_det=100 agnostic_nms")
@pytest.mark.parametrize("task,model,data", TASK_MODEL_DATA)
def test_predict(task: str, model: str, data: str) -> None:
"""Test YOLO prediction on provided sample assets for specified task and model."""
for end2end in {False, True}:
run(
f"yolo {task} predict model={model} source={ASSETS} imgsz=32 save save_crop save_txt visualize end2end={end2end} max_det=100"
)
run(f"yolo {task} predict model={model} source={ASSETS} imgsz=32 save end2end={end2end} max_det=100")
@pytest.mark.parametrize("model", MODELS)
def test_export(model: str) -> None:
def test_export(model: str, tmp_path: Path) -> None: # use tmp_path to prevent the race condition with test_exports.py
"""Test exporting a YOLO model to TorchScript format."""
for end2end in {False, True}:
run(f"yolo export model={model} format=torchscript imgsz=32 end2end={end2end} max_det=100")
run(
f"yolo export model={model} format=torchscript imgsz=32 end2end={end2end} max_det=100 project={tmp_path} name=export"
)
@pytest.mark.skipif(not TORCH_1_11, reason="RTDETR requires torch>=1.11")
def test_rtdetr(task: str = "detect", model: Path = WEIGHTS_DIR / "rtdetr-l.pt", data: str = "coco8.yaml") -> None:
"""Test the RTDETR functionality within Ultralytics for detection tasks using specified model and data."""
# Add comma, spaces, fraction=0.25 args to test single-image training
run(f"yolo predict {task} model={model} source={ASSETS / 'bus.jpg'} imgsz=160 save save_crop save_txt")
run(f"yolo predict {task} model={model} source={ASSETS / 'bus.jpg'} imgsz=160 save")
run(f"yolo train {task} model={model} data={data} --imgsz= 160 epochs =1, cache = disk fraction=0.25")
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Edge devices not intended for heavy FastSAM tests")
@pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="MobileSAM with CLIP is not supported in Python 3.12")
@pytest.mark.skipif(
checks.IS_PYTHON_3_8 and LINUX and ARM64,
@ -77,7 +76,7 @@ def test_fastsam(
source = ASSETS / "bus.jpg"
run(f"yolo segment val {task} model={model} data={data} imgsz=32")
run(f"yolo segment predict model={model} source={source} imgsz=32 save save_crop save_txt")
run(f"yolo segment predict model={model} source={source} imgsz=32 save")
from ultralytics import FastSAM
from ultralytics.models.sam import Predictor
@ -87,7 +86,7 @@ def test_fastsam(
# Run inference on an image
for s in (source, Image.open(source)):
everything_results = sam_model(s, device="cpu", retina_masks=True, imgsz=320, conf=0.4, iou=0.9)
everything_results = sam_model(s, device="cpu", retina_masks=True, imgsz=160, conf=0.4, iou=0.9)
# Remove small regions
_new_masks, _ = Predictor.remove_small_regions(everything_results[0].masks.data, min_area=20)

View File

@ -52,7 +52,7 @@ def test_amp():
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, simplify, nms)
for task, dynamic, int8, half, batch, simplify, nms in product(
TASKS, [True, False], [False], [False], [1, 2], [True, False], [True, False]
sorted(TASKS), [True, False], [False], [False], [1, 2], [True, False], [True, False]
)
if not (
(int8 and half) or (task == "classify" and nms) or (task == "obb" and nms and (not TORCH_1_13 or IS_JETSON))
@ -85,7 +85,7 @@ def test_export_onnx_matrix(task, dynamic, int8, half, batch, simplify, nms):
(task, dynamic, int8, half, batch)
# Note: tests reduced below pending compute availability expansion as GPU CI runner utilization is high
# for task, dynamic, int8, half, batch in product(TASKS, [True, False], [True, False], [True, False], [1, 2])
for task, dynamic, int8, half, batch in product(TASKS, [True], [True], [False], [2])
for task, dynamic, int8, half, batch in product(sorted(TASKS), [True], [True], [False], [2])
if not (int8 and half) # exclude cases where both int8 and half are True
],
)
@ -168,19 +168,19 @@ def test_autobatch():
"""Check optimal batch size for YOLO model training using autobatch utility."""
from ultralytics.utils.autobatch import check_train_batch_size
check_train_batch_size(YOLO(MODEL).model.to(f"cuda:{DEVICES[0]}"), imgsz=128, amp=True)
check_train_batch_size(YOLO(MODEL).model.to(f"cuda:{DEVICES[0]}"), imgsz=64, amp=True)
@pytest.mark.slow
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
def test_utils_benchmarks():
def test_utils_benchmarks(isolated_model):
"""Profile YOLO models for performance benchmarks."""
from ultralytics.utils.benchmarks import ProfileModels
# Pre-export a dynamic engine model to use dynamic inference
YOLO(MODEL).export(format="engine", imgsz=32, dynamic=True, batch=1, device=DEVICES[0])
YOLO(isolated_model).export(format="engine", imgsz=32, dynamic=True, batch=1, device=DEVICES[0])
ProfileModels(
[MODEL],
[isolated_model],
imgsz=32,
half=False,
min_time=1,

View File

@ -13,7 +13,7 @@ from pathlib import Path
import pytest
import torch
from tests import MODEL, SOURCE
from tests import SOURCE
from ultralytics import YOLO
from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS
from ultralytics.utils import ARM64, IS_DOCKER, IS_RASPBERRYPI, LINUX, MACOS, MACOS_VERSION, WINDOWS, checks
@ -37,16 +37,16 @@ def skip_rpi_semantic(task):
@pytest.mark.parametrize("end2end", [False, True])
def test_export_torchscript(end2end):
def test_export_torchscript(end2end, isolated_model):
"""Test YOLO model export to TorchScript format for compatibility and correctness."""
file = YOLO(MODEL).export(format="torchscript", optimize=False, imgsz=32, end2end=end2end)
file = YOLO(isolated_model).export(format="torchscript", optimize=False, imgsz=32, end2end=end2end)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@pytest.mark.parametrize("end2end", [False, True])
def test_export_onnx(end2end):
def test_export_onnx(end2end, isolated_model):
"""Test YOLO model export to ONNX format with dynamic axes."""
file = YOLO(MODEL).export(format="onnx", dynamic=True, imgsz=32, end2end=end2end)
file = YOLO(isolated_model).export(format="onnx", dynamic=True, imgsz=32, end2end=end2end)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@ -86,9 +86,9 @@ def test_torch2onnx_serializes_concurrent_exports(monkeypatch, tmp_path):
@pytest.mark.skipif(not TORCH_2_1, reason="OpenVINO requires torch>=2.1")
@pytest.mark.parametrize("end2end", [False, True])
def test_export_openvino(end2end):
def test_export_openvino(end2end, isolated_model):
"""Test YOLO export to OpenVINO format for model inference compatibility."""
file = YOLO(MODEL).export(format="openvino", imgsz=32, end2end=end2end)
file = YOLO(isolated_model).export(format="openvino", imgsz=32, end2end=end2end)
if WINDOWS:
# Ensure a unique export path per test to prevent OpenVINO file writes
file = Path(file)
@ -103,7 +103,7 @@ def test_export_openvino(end2end):
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, nms, end2end)
for task, dynamic, int8, half, batch, nms, end2end in product(
TASKS, [True, False], [True, False], [True, False], [1, 2], [True, False], [True]
sorted(TASKS), [True, False], [True, False], [True, False], [1, 2], [True, False], [True]
)
if not ((int8 and half) or (task == "classify" and nms) or (end2end and nms))
],
@ -137,7 +137,7 @@ def test_export_openvino_matrix(task, dynamic, int8, half, batch, nms, end2end):
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, simplify, nms, end2end)
for task, dynamic, int8, half, batch, simplify, nms, end2end in product(
TASKS, [True, False], [False], [False], [1, 2], [True, False], [True, False], [True, False]
sorted(TASKS), [True, False], [False], [False], [1, 2], [True, False], [True, False], [True, False]
)
if not ((int8 and half) or (task == "classify" and nms) or (nms and not TORCH_1_13) or (end2end and nms))
],
@ -166,7 +166,7 @@ def test_export_onnx_matrix(task, dynamic, int8, half, batch, simplify, nms, end
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, nms, end2end)
for task, dynamic, int8, half, batch, nms, end2end in product(
TASKS, [False, True], [False], [False, True], [1, 2], [True, False], [True, False]
sorted(TASKS), [False, True], [False], [False, True], [1, 2], [True, False], [True, False]
)
if not ((task == "classify" and nms) or (end2end and nms))
],
@ -193,7 +193,7 @@ def test_export_torchscript_matrix(task, dynamic, int8, half, batch, nms, end2en
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, nms, batch, end2end)
for task, dynamic, int8, half, nms, batch, end2end in product(
TASKS, [True, False], [True, False], [True, False], [True, False], [1], [True, False]
sorted(TASKS), [True, False], [True, False], [True, False], [True, False], [1], [True, False]
)
if not (int8 and half)
and not (task != "detect" and nms)
@ -232,7 +232,7 @@ def test_export_coreml_matrix(task, dynamic, int8, half, nms, batch, end2end):
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, nms, end2end)
for task, dynamic, int8, half, batch, nms, end2end in product(
TASKS, [False], [True, False], [True, False], [1], [True, False], [True, False]
sorted(TASKS), [False], [True, False], [True, False], [1], [True, False], [True, False]
)
if not (
(int8 and half)
@ -257,14 +257,14 @@ def test_export_tflite_matrix(task, dynamic, int8, half, batch, nms, end2end):
@pytest.mark.skipif(WINDOWS, reason="CoreML not supported on Windows") # RuntimeError: BlobWriter not loaded
@pytest.mark.skipif(LINUX and ARM64, reason="CoreML not supported on aarch64 Linux")
@pytest.mark.skipif(checks.IS_PYTHON_3_13, reason="CoreML not supported in Python 3.13")
def test_export_coreml():
def test_export_coreml(isolated_model):
"""Test YOLO export to CoreML format and check for errors."""
# Capture stdout and stderr
stdout, stderr = io.StringIO(), io.StringIO()
with redirect_stdout(stdout), redirect_stderr(stderr):
YOLO(MODEL).export(format="coreml", nms=True, imgsz=32)
YOLO(isolated_model).export(format="coreml", nms=True, imgsz=32)
if MACOS:
file = YOLO(MODEL).export(format="coreml", imgsz=32)
file = YOLO(isolated_model).export(format="coreml", imgsz=32)
YOLO(file)(SOURCE, imgsz=32) # model prediction only supported on macOS for nms=False models
# Check captured output for errors
@ -276,33 +276,32 @@ def test_export_coreml():
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10, reason="TFLite export requires Python>=3.10")
@pytest.mark.skipif(not TORCH_1_13, reason="TFLite export requires torch>=1.13")
@pytest.mark.skipif(not LINUX, reason="Test disabled as TF suffers from install conflicts on Windows and macOS")
def test_export_tflite():
def test_export_tflite(isolated_model):
"""Test YOLO export to TFLite format under specific OS and Python version conditions."""
model = YOLO(MODEL)
model = YOLO(isolated_model)
file = model.export(format="tflite", imgsz=32)
YOLO(file)(SOURCE, imgsz=32)
@pytest.mark.skipif(True, reason="Test disabled")
@pytest.mark.skipif(not LINUX, reason="TF suffers from install conflicts on Windows and macOS")
def test_export_pb():
def test_export_pb(isolated_model):
"""Test YOLO export to TensorFlow's Protobuf (*.pb) format."""
model = YOLO(MODEL)
model = YOLO(isolated_model)
file = model.export(format="pb", imgsz=32)
YOLO(file)(SOURCE, imgsz=32)
@pytest.mark.skipif(True, reason="Test disabled as Paddle protobuf and ONNX protobuf requirements conflict.")
def test_export_paddle():
def test_export_paddle(isolated_model):
"""Test YOLO export to Paddle format, noting protobuf conflicts with ONNX."""
YOLO(MODEL).export(format="paddle", imgsz=32)
YOLO(isolated_model).export(format="paddle", imgsz=32)
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_1_10, reason="MNN export requires torch>=1.10")
def test_export_mnn():
def test_export_mnn(isolated_model):
"""Test YOLO export to MNN format (WARNING: MNN test must precede NCNN test or CI error on Windows)."""
file = YOLO(MODEL).export(format="mnn", imgsz=32)
file = YOLO(isolated_model).export(format="mnn", imgsz=32)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@ -312,7 +311,9 @@ def test_export_mnn():
"task, int8, half, batch, end2end",
[ # generate all combinations except for exclusion cases
(task, int8, half, batch, end2end)
for task, int8, half, batch, end2end in product(TASKS, [True, False], [True, False], [1, 2], [True, False])
for task, int8, half, batch, end2end in product(
sorted(TASKS), [True, False], [True, False], [1, 2], [True, False]
)
if not (int8 and half)
],
)
@ -324,17 +325,16 @@ def test_export_mnn_matrix(task, int8, half, batch, end2end):
Path(file).unlink() # cleanup
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_2_0, reason="NCNN inference causes segfault on PyTorch<2.0")
def test_export_ncnn():
def test_export_ncnn(isolated_model):
"""Test YOLO export to NCNN format."""
file = YOLO(MODEL).export(format="ncnn", imgsz=32)
file = YOLO(isolated_model).export(format="ncnn", imgsz=32)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_2_0, reason="NCNN inference causes segfault on PyTorch<2.0")
@pytest.mark.parametrize("task, half, batch", list(product(TASKS, [True, False], [1])))
@pytest.mark.parametrize("task, half, batch", list(product(sorted(TASKS), [True, False], [1])))
def test_export_ncnn_matrix(task, half, batch):
"""Test YOLO export to NCNN format considering various export configurations."""
skip_rpi_semantic(task)
@ -349,7 +349,7 @@ def test_export_ncnn_matrix(task, half, batch):
@pytest.mark.skipif(
IS_RASPBERRYPI, reason="Test disabled as IMX export suffers from OOM (Out of Memory) on Raspberry Pi 5 16GB"
)
def test_export_imx():
def test_export_imx(isolated_model):
"""Test YOLO export to IMX format."""
model = YOLO("yolo11n.pt") # IMX export only supports YOLO11
file = model.export(format="imx", imgsz=32)
@ -359,9 +359,9 @@ def test_export_imx():
# @pytest.mark.skipif(True, reason="Disabled for debugging ruamel.yaml installation required by executorch")
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0")
@pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows")
def test_export_executorch():
def test_export_executorch(isolated_model):
"""Test YOLO model export to ExecuTorch format."""
file = YOLO(MODEL).export(format="executorch", imgsz=32)
file = YOLO(isolated_model).export(format="executorch", imgsz=32)
assert Path(file).exists(), f"ExecuTorch export failed, directory not found: {file}"
# Check that .pte file exists in the exported directory
pte_file = Path(file) / "model.pte"
@ -376,7 +376,7 @@ def test_export_executorch():
@pytest.mark.slow
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0")
@pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows")
@pytest.mark.parametrize("task", TASKS)
@pytest.mark.parametrize("task", sorted(TASKS))
def test_export_executorch_matrix(task):
"""Test YOLO export to ExecuTorch format for various task types."""
skip_rpi_semantic(task)
@ -399,10 +399,10 @@ def test_export_executorch_matrix(task):
reason="Axelera export is only supported on Linux and is not supported on ARM64 Docker",
)
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Test disabled due to OOM (Out of Memory) issues on Raspberry Pi 5 16GB")
def test_export_axelera():
def test_export_axelera(isolated_model):
"""Test YOLO export to Axelera format."""
# For faster testing, use a smaller calibration dataset (32 image size crashes axelera export, so 64 is used)
file = YOLO(MODEL).export(format="axelera", imgsz=64, data="coco8.yaml")
file = YOLO(isolated_model).export(format="axelera", imgsz=64, data="coco8.yaml")
assert Path(file).exists(), f"Axelera export failed, directory not found: {file}"
# Note: Inference testing skipped as it requires Axelera hardware
shutil.rmtree(file, ignore_errors=True) # cleanup
@ -413,9 +413,9 @@ def test_export_axelera():
@pytest.mark.skipif(
not checks.IS_PYTHON_MINIMUM_3_12, reason="Requires Python>=3.12 for CI validation due to torch upgrades"
)
def test_export_deepx():
def test_export_deepx(isolated_model):
"""Test YOLO export to DeepX format."""
file = YOLO(MODEL).export(format="deepx", imgsz=32)
file = YOLO(isolated_model).export(format="deepx", imgsz=32)
assert Path(file).exists(), f"DeepX export failed, directory not found: {file}"
# Note: Inference testing skipped as it requires DeepX hardware
shutil.rmtree(file, ignore_errors=True) # cleanup
@ -425,7 +425,7 @@ def test_export_deepx():
not (WINDOWS or (LINUX and ARM64)) or sys.version_info < (3, 11),
reason="onnxruntime-qnn ships prebuilt wheels only for Windows (x64/ARM64) and Linux ARM64 on Python>=3.11",
)
def test_export_qnn():
def test_export_qnn(isolated_model):
"""Test YOLO export to Qualcomm QNN format via the ONNX Runtime QNN Execution Provider."""
import importlib.util
@ -441,7 +441,7 @@ def test_export_qnn():
).exists()
if not has_qnn:
pytest.skip("onnxruntime-qnn / QNN Execution Provider not available")
file = YOLO(MODEL).export(format="qnn", imgsz=32)
file = YOLO(isolated_model).export(format="qnn", imgsz=32)
assert next(Path(file).rglob("*_qnn.onnx"), None), f"QNN export failed, no context binary found in: {file}"
# Note: on-device inference is not exercised here as it requires Qualcomm Snapdragon hardware
shutil.rmtree(file, ignore_errors=True) # cleanup

View File

@ -8,7 +8,7 @@ from pathlib import Path
import pytest
from tests import MODEL, SOURCE
from tests import SOURCE
from ultralytics import YOLO, download
from ultralytics.utils import ASSETS_URL, DATASETS_DIR, SETTINGS
from ultralytics.utils.checks import check_requirements
@ -71,7 +71,7 @@ def test_mlflow_keep_run_active():
@pytest.mark.skipif(not check_requirements("tritonclient", install=False), reason="tritonclient[all] not installed")
def test_triton(tmp_path):
def test_triton(tmp_path, isolated_model):
"""Test NVIDIA Triton Server functionalities with YOLO model."""
check_requirements("tritonclient[all]")
from tritonclient.http import InferenceServerClient
@ -82,7 +82,7 @@ def test_triton(tmp_path):
triton_model = triton_repo / model_name # Triton model path
# Export model to ONNX
f = YOLO(MODEL).export(format="onnx", dynamic=True)
f = YOLO(isolated_model).export(format="onnx", dynamic=True)
# Prepare Triton repo
(triton_model / "1").mkdir(parents=True, exist_ok=True)

View File

@ -202,7 +202,7 @@ def test_youtube():
"""Test YOLO model on a YouTube video stream, handling potential network-related errors."""
model = YOLO(MODEL)
try:
model.predict("https://youtu.be/G17sBkb38XQ", imgsz=96, save=True)
model.predict("https://youtu.be/G17sBkb38XQ", imgsz=32, save=True)
# Handle internet connection errors and 'urllib.error.HTTPError: HTTP Error 429: Too Many Requests'
except (urllib.error.HTTPError, ConnectionError) as e:
LOGGER.error(f"YouTube Test Error: {e}")
@ -280,15 +280,15 @@ def test_all_model_yamls():
for m in (ROOT / "cfg" / "models").rglob("*.yaml"):
if "rtdetr" in m.name:
if TORCH_1_11:
_ = RTDETR(m.name)(SOURCE, imgsz=640) # must be 640
_ = RTDETR(m.name)(SOURCE, imgsz=160)
else:
YOLO(m.name)
@pytest.mark.skipif(WINDOWS, reason="Windows slow CI export bug https://github.com/ultralytics/ultralytics/pull/16003")
def test_workflow():
def test_workflow(isolated_model):
"""Test the complete workflow including training, validation, prediction, and exporting."""
model = YOLO(MODEL)
model = YOLO(isolated_model)
model.train(data="coco8.yaml", epochs=1, imgsz=32, optimizer="SGD")
model.val(imgsz=32)
model.predict(SOURCE, imgsz=32)
@ -310,7 +310,7 @@ def test_predict_callback_and_setup():
dataset = load_inference_source(source=SOURCE)
bs = dataset.bs # access predictor properties
results = model.predict(dataset, stream=True, imgsz=160) # source already setup
results = model.predict(dataset, stream=True, imgsz=32) # source already setup
for r, im0, bs in results:
print("test_callback", im0.shape)
print("test_callback", bs)
@ -350,7 +350,7 @@ def test_results(model: str, tmp_path):
def test_labels_and_crops():
"""Test output from prediction args for saving YOLO detection labels and crops."""
imgs = [SOURCE, ASSETS / "zidane.jpg"]
results = YOLO(WEIGHTS_DIR / "yolo26n.pt")(imgs, imgsz=320, save_txt=True, save_crop=True)
results = YOLO(WEIGHTS_DIR / "yolo26n.pt")(imgs, imgsz=160, save_txt=True, save_crop=True)
save_path = Path(results[0].save_dir)
for r in results:
im_name = Path(r.path).stem
@ -730,6 +730,7 @@ def test_classify_transforms_train(image, auto_augment, erasing, force_color_jit
@pytest.mark.slow
@pytest.mark.skipif(IS_RASPBERRYPI or IS_JETSON, reason="Edge devices not intended for tuning")
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
def test_model_tune():
"""Tune YOLO model for performance improvement."""
@ -741,6 +742,7 @@ def test_model_tune():
@pytest.mark.slow
@pytest.mark.skipif(IS_RASPBERRYPI or IS_JETSON, reason="Edge devices not intended for tuning")
@pytest.mark.skipif(not ONLINE or not checks.IS_PYTHON_MINIMUM_3_10, reason="environment is offline")
@pytest.mark.skipif(not checks.check_requirements("ray", install=False), reason="ray[tune] not installed")
def test_model_tune_ray():
@ -767,6 +769,7 @@ def test_model_embeddings():
assert len(model_segment.embed(source=batch, imgsz=32)) == len(batch)
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Edge devices not intended for CLIP-based models")
@pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="YOLOWorld with CLIP is not supported in Python 3.12")
@pytest.mark.skipif(
checks.IS_PYTHON_3_8 and LINUX and ARM64,
@ -803,6 +806,7 @@ def test_yolo_world():
)
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Edge devices not intended for heavy CLIP-based models")
@pytest.mark.skipif(not TORCH_1_13, reason="YOLOE with CLIP requires torch>=1.13")
@pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="YOLOE with CLIP is not supported in Python 3.12")
@pytest.mark.skipif(

View File

@ -178,10 +178,8 @@ def process_video(solution, video_path: str, needs_frame_count: bool = False):
)
def test_solution(name, solution_class, needs_frame_count, video_key, kwargs_update, tmp_path, solution_assets):
"""Test individual Ultralytics solution with video processing and parameter validation."""
# Get video path from persistent cache (no copying needed, read-only access)
video_path = str(solution_assets(video_key)) if video_key else None
# Update kwargs to use cached paths for parking manager
kwargs = {}
for key, value in kwargs_update.items():
if key.startswith("temp_"):
@ -269,7 +267,7 @@ def test_plot_with_no_masks():
assert results.plot_im is not None, "Instance segmentation plot returned None"
def test_streamlit_handle_video_upload_creates_file():
def test_streamlit_handle_video_upload_creates_file(tmp_path):
"""Test Streamlit video upload logic saves file correctly."""
import io
@ -277,17 +275,18 @@ def test_streamlit_handle_video_upload_creates_file():
fake_file.read = fake_file.getvalue
if fake_file is not None:
g = io.BytesIO(fake_file.read())
with open("ultralytics.mp4", "wb") as out:
with open(tmp_path / "ultralytics.mp4", "wb") as out:
out.write(g.read())
output_path = "ultralytics.mp4"
output_path = str(tmp_path / "ultralytics.mp4")
else:
output_path = None
assert output_path == "ultralytics.mp4", f"Expected output_path 'ultralytics.mp4', got {output_path}"
assert os.path.exists("ultralytics.mp4"), "ultralytics.mp4 file not created"
with open("ultralytics.mp4", "rb") as f:
assert output_path == str(tmp_path / "ultralytics.mp4"), (
f"Expected output_path '{tmp_path / 'ultralytics.mp4'}', got {output_path}"
)
assert os.path.exists(tmp_path / "ultralytics.mp4"), "ultralytics.mp4 file not created"
with open(tmp_path / "ultralytics.mp4", "rb") as f:
content = f.read()
assert content == b"fake video content", f"File content mismatch: {content}"
os.remove("ultralytics.mp4")
@pytest.mark.skipif(not TORCH_2_4, reason=f"VisualAISearch requires torch>=2.4 (found torch=={TORCH_VERSION})")

View File

@ -3,6 +3,7 @@
from __future__ import annotations
import ast
import os
import shutil
import subprocess
import sys
@ -453,7 +454,11 @@ def get_save_dir(args: SimpleNamespace, name: str | None = None) -> Path:
project = args.project or ""
if not Path(project).is_absolute():
project = (ROOT.parent / "tests/tmp/runs" if TESTS_RUNNING else RUNS_DIR) / args.task / project
base = ROOT.parent / "tests/tmp/runs" if TESTS_RUNNING else RUNS_DIR
worker = os.environ.get("PYTEST_XDIST_WORKER")
if worker and TESTS_RUNNING: # isolate parallel pytest-xdist workers
base = base / worker
project = base / args.task / project
name = name or args.name or f"{args.mode}"
save_dir = increment_path(Path(project) / name, exist_ok=args.exist_ok if RANK in {-1, 0} else True)