From b97bfe2997efa968cc2f26d9b7140adb69835dbe Mon Sep 17 00:00:00 2001 From: Jing Qiu <61612323+Laughing-q@users.noreply.github.com> Date: Tue, 26 May 2026 03:07:18 +0800 Subject: [PATCH] Enable parallel CI tests via pytest-xdist and fix export/train race conditions (#24565) Co-authored-by: UltralyticsAssistant Co-authored-by: Glenn Jocher --- .github/workflows/ci.yml | 60 ++++++++++++++++++++------ .gitignore | 3 ++ pyproject.toml | 1 + tests/__init__.py | 14 +++++- tests/cache_test_assets.py | 85 +++++++++++++++++++++++++++++++++++++ tests/conftest.py | 75 ++++++++++++++++---------------- tests/test_cli.py | 21 +++++---- tests/test_cuda.py | 12 +++--- tests/test_exports.py | 78 +++++++++++++++++----------------- tests/test_integrations.py | 6 +-- tests/test_python.py | 16 ++++--- tests/test_solutions.py | 17 ++++---- ultralytics/cfg/__init__.py | 7 ++- 13 files changed, 266 insertions(+), 129 deletions(-) create mode 100644 tests/cache_test_assets.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbca93d3..fb4ed526 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,14 +145,27 @@ jobs: if [ "${{ matrix.torch }}" != "latest" ]; then torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}" fi - uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match + uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match - name: Check environment run: | yolo checks uv pip list + - name: Configure Ultralytics cache paths + shell: bash # for Windows compatibility + run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets" + - name: Cache test assets + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.ultralytics_cache + key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }} + restore-keys: | + ultralytics-test-assets-${{ runner.os }}- + - name: Pre-cache test assets + shell: bash # for Windows compatibility + run: python tests/cache_test_assets.py - name: Pytest tests shell: bash # for Windows compatibility - run: pytest --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx + run: pytest -n auto --dist=loadfile --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx - name: Upload Coverage Reports to CodeCov if: github.repository == 'ultralytics/ultralytics' # && matrix.os == 'ubuntu-latest' && matrix.python == '3.12' uses: codecov/codecov-action@v6 @@ -263,16 +276,30 @@ jobs: if [ "${{ matrix.torch }}" != "latest" ]; then torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}" fi - uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch faster-coco-eval mlflow pytest-cov --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match + uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch faster-coco-eval mlflow pytest-cov pytest-xdist --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match - name: Check environment run: | yolo checks uv pip list + - name: Configure Ultralytics cache paths + shell: bash # for Windows compatibility + run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets" + - name: Cache test assets + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.ultralytics_cache + key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }} + restore-keys: | + ultralytics-test-assets-${{ runner.os }}- + - name: Pre-cache test assets + shell: bash # for Windows compatibility + run: python tests/cache_test_assets.py - name: Pytest tests uses: ultralytics/actions/retry@main with: shell: bash # for Windows compatibility - run: pytest --slow --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx + # -n 2 limits parallelism for resource-heavy export tests on shared runners + run: pytest -vv -s -n 2 --dist=loadfile --slow --cov=ultralytics/ --cov-report=xml tests/ --deselect tests/test_exports.py::test_export_imx --deselect tests/test_exports.py::test_export_mnn --deselect tests/test_exports.py::test_export_ncnn --deselect tests/test_exports.py::test_export_executorch retries: 1 # Retry once after initial attempt (2 total runs) timeout_minutes: 180 retry_delay_seconds: 60 @@ -330,13 +357,24 @@ jobs: - uses: astral-sh/setup-uv@v7 - name: Install requirements run: | - uv pip install -e ".[export]" pytest mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match + uv pip install -e ".[export]" pytest pytest-xdist mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match - name: Check environment run: | yolo checks uv pip list + - name: Configure Ultralytics cache paths + run: yolo settings weights_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/weights" datasets_dir="${GITHUB_WORKSPACE}/.ultralytics_cache/datasets" + - name: Cache test assets + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.ultralytics_cache + key: ultralytics-test-assets-${{ runner.os }}-${{ hashFiles('tests/cache_test_assets.py', 'tests/__init__.py') }} + restore-keys: | + ultralytics-test-assets-${{ runner.os }}- + - name: Pre-cache test assets + run: python tests/cache_test_assets.py - name: Pytest tests - run: pytest --slow tests/ --deselect tests/test_exports.py::test_export_imx + run: pytest -n auto --dist=loadfile --slow tests/ --deselect tests/test_exports.py::test_export_imx - name: Activate Virtual Environment for Benchmarks run: | python3.11 -m venv env-benchmarks @@ -447,14 +485,8 @@ jobs: run: uv pip install pytest - name: Check environment run: conda list - - name: Test CLI - run: | - yolo predict model=yolo26n.pt imgsz=320 - yolo train model=yolo26n.pt data=coco8.yaml epochs=1 imgsz=32 - yolo val model=yolo26n.pt data=coco8.yaml imgsz=32 - yolo export model=yolo26n.pt format=torchscript imgsz=160 - yolo benchmark model=yolo26n.pt data='coco8.yaml' imgsz=640 format=onnx - yolo solutions + - name: Test benchmark CLI + run: yolo benchmark model=yolo26n.pt data='coco8.yaml' imgsz=640 format=onnx - name: Test Python # Note this step must use the updated default bash environment, not a Python environment run: | diff --git a/.gitignore b/.gitignore index 94755fdf..a1d17dc9 100644 --- a/.gitignore +++ b/.gitignore @@ -203,3 +203,6 @@ calibration_*.npy # Training logs args.yaml results.csv + +# CI test asset cache +.ultralytics_cache/ diff --git a/pyproject.toml b/pyproject.toml index 198cbc6f..360a6698 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ dev = [ "ipython", "pytest", "pytest-cov", + "pytest-xdist>=3.0.0", "coverage[toml]", "zensical>=0.0.15; python_version >= '3.10'", "mkdocs-ultralytics-plugin>=0.2.4", # for meta descriptions and images, dates and authors diff --git a/tests/__init__.py b/tests/__init__.py index 5abb4fb0..d765c085 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -10,14 +10,24 @@ SOURCE = ASSETS / "bus.jpg" SOURCES_LIST = [ASSETS / "bus.jpg", ASSETS, ASSETS / "*", ASSETS / "**/*.jpg"] CUDA_IS_AVAILABLE = checks.cuda_is_available() CUDA_DEVICE_COUNT = checks.cuda_device_count() -TASK_MODEL_DATA = [(task, WEIGHTS_DIR / TASK2MODEL[task], TASK2DATA[task]) for task in TASKS] -MODELS = frozenset([*list(TASK2MODEL.values()), "yolo11n-grayscale.pt"]) +TASK_MODEL_DATA = sorted([(task, WEIGHTS_DIR / TASK2MODEL[task], TASK2DATA[task]) for task in TASKS]) +MODELS = sorted([*list(TASK2MODEL.values()), "yolo11n-grayscale.pt"]) +SOLUTION_ASSETS = { + "demo_video": "solutions_ci_demo.mp4", + "crop_video": "decelera_landscape_min.mov", + "pose_video": "solution_ci_pose_demo.mp4", + "parking_video": "solution_ci_parking_demo.mp4", + "vertical_video": "solution_vertical_demo.mp4", + "parking_areas": "solution_ci_parking_areas.json", + "parking_model": "solutions_ci_parking_model.pt", +} __all__ = ( "CFG", "CUDA_DEVICE_COUNT", "CUDA_IS_AVAILABLE", "MODEL", + "SOLUTION_ASSETS", "SOURCE", "SOURCES_LIST", ) diff --git a/tests/cache_test_assets.py b/tests/cache_test_assets.py new file mode 100644 index 00000000..503e37d6 --- /dev/null +++ b/tests/cache_test_assets.py @@ -0,0 +1,85 @@ +# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license +"""Pre-download shared test assets to avoid race conditions under pytest-xdist. + +Run this script once before `pytest -n auto` to ensure all model weights, +datasets, and solution assets are already cached locally. Each xdist worker +can then reuse existing files instead of competing to download the same remote resources. +""" + +import shutil +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tests import MODEL, SOLUTION_ASSETS +from ultralytics.cfg import TASK2CALIBRATIONDATA, TASK2DATA, TASK2MODEL +from ultralytics.data.utils import check_cls_dataset, check_det_dataset +from ultralytics.utils import ASSETS_URL, LOGGER, WEIGHTS_DIR +from ultralytics.utils.downloads import attempt_download_asset, safe_download + +WEIGHTS = [ + *TASK2MODEL.values(), + "yolo11n-grayscale.pt", + "rtdetr-l.pt", + "FastSAM-s.pt", + "mobile_sam.pt", + "sam2.1_b.pt", + "yolov8s-world.pt", + "yolov8s-worldv2.pt", + "yoloe-11s-seg.pt", + "yoloe-11s-seg-pf.pt", +] + +DATASETS = [ + *TASK2DATA.values(), + *TASK2CALIBRATIONDATA.values(), + "coco8-grayscale.yaml", + "coco8-multispectral.yaml", + "coco12-formats.yaml", +] + + +def cache_weights() -> None: + """Download all model weights used by tests.""" + LOGGER.info("[cache] Downloading model weights ...") + for w in WEIGHTS: + attempt_download_asset(WEIGHTS_DIR / w) + if not MODEL.exists(): + MODEL.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(WEIGHTS_DIR / "yolo26n.pt", MODEL) + LOGGER.info("[cache] Weights done.") + + +def cache_datasets() -> None: + """Download / extract all datasets used by tests.""" + LOGGER.info("[cache] Downloading datasets ...") + for ds in DATASETS: + if ds.startswith("imagenet"): + check_cls_dataset(ds) + else: + check_det_dataset(ds, autodownload=True) + LOGGER.info("[cache] Datasets done.") + + +def cache_solution_assets() -> None: + """Download solution test assets (videos, parking json, etc.).""" + LOGGER.info("[cache] Downloading solution assets ...") + cache_dir = WEIGHTS_DIR / "solution_assets" + cache_dir.mkdir(parents=True, exist_ok=True) + for asset in SOLUTION_ASSETS.values(): + dst = cache_dir / asset + if not dst.exists(): + safe_download(url=f"{ASSETS_URL}/{asset}", dir=cache_dir) + LOGGER.info("[cache] Solution assets done.") + + +def main() -> None: + """Main function to orchestrate caching of all test assets.""" + cache_weights() + cache_datasets() + cache_solution_assets() + LOGGER.info("[cache] All test assets are ready.") + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index d53768af..afcb955b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,46 +3,25 @@ import shutil from pathlib import Path +import numpy.testing # noqa: F401 # Pre-import before any test can corrupt numpy via in-place upgrade import pytest @pytest.fixture(scope="session") def solution_assets(): - """Session-scoped fixture to cache solution test assets. - - Lazily downloads solution assets into a persistent directory (WEIGHTS_DIR/solution_assets) and returns a callable - that resolves asset names to cached paths. - """ + """Return cached solution asset paths by name.""" + from tests import SOLUTION_ASSETS from ultralytics.utils import ASSETS_URL, WEIGHTS_DIR from ultralytics.utils.downloads import safe_download - # Use persistent directory alongside weights cache_dir = WEIGHTS_DIR / "solution_assets" cache_dir.mkdir(parents=True, exist_ok=True) - # Define all assets needed for solution tests - assets = { - # Videos - "demo_video": "solutions_ci_demo.mp4", - "crop_video": "decelera_landscape_min.mov", - "pose_video": "solution_ci_pose_demo.mp4", - "parking_video": "solution_ci_parking_demo.mp4", - "vertical_video": "solution_vertical_demo.mp4", - # Parking manager files - "parking_areas": "solution_ci_parking_areas.json", - "parking_model": "solutions_ci_parking_model.pt", - } - - asset_paths = {} - def get_asset(name): - """Return the cached path for a named solution asset, downloading it on first use.""" - if name not in asset_paths: - asset_path = cache_dir / assets[name] - if not asset_path.exists(): - safe_download(url=f"{ASSETS_URL}/{asset_path.name}", dir=cache_dir) - asset_paths[name] = asset_path - return asset_paths[name] + asset_path = cache_dir / SOLUTION_ASSETS[name] + if not asset_path.exists(): + safe_download(url=f"{ASSETS_URL}/{asset_path.name}", dir=cache_dir) + return asset_path return get_asset @@ -78,17 +57,38 @@ def pytest_sessionstart(session): init_seeds() -def pytest_terminal_summary(terminalreporter, exitstatus, config): +@pytest.fixture +def isolated_model(tmp_path): + """Provide an isolated copy of the test model to prevent export file races under pytest-xdist. + + When multiple xdist workers run export tests simultaneously, they derive output filenames from the model path (e.g., + model.onnx, model.torchscript). Using the same MODEL path causes workers to overwrite each other's + intermediate/export files. This fixture copies the shared model to a per-test temporary directory so each test + exports to a unique path. + """ + from tests import MODEL + + if not Path(MODEL).exists(): + from ultralytics.utils.downloads import attempt_download_asset + + MODEL.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(attempt_download_asset("yolo26n.pt"), MODEL) + + dst = tmp_path / "model.pt" + shutil.copy(MODEL, dst) + return str(dst) + + +def pytest_sessionfinish(session, exitstatus): """Cleanup operations after pytest session. - This function is automatically called by pytest at the end of the entire test session. It removes certain files and - directories used during testing. - - Args: - terminalreporter: The terminal reporter object used for terminal output. - exitstatus (int): The exit status of the test run. - config: The pytest config object. + Runs only on the pytest controller (or serial run), skipping xdist workers to avoid race conditions where one worker + deletes shared assets while another is still reading them. """ + # Skip on xdist workers - only the controller should clean up shared resources + if hasattr(session.config, "workerinput"): + return + from ultralytics.utils import WEIGHTS_DIR # Remove files @@ -97,6 +97,5 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): Path(file).unlink(missing_ok=True) # Remove directories - models = [path for x in {"*.mlpackage", "*_openvino_model"} for path in WEIGHTS_DIR.rglob(x)] - for directory in [WEIGHTS_DIR / "solution_assets", WEIGHTS_DIR / "path with spaces", *models]: + for directory in [path for x in {"*.mlpackage", "*_openvino_model"} for path in WEIGHTS_DIR.rglob(x)]: shutil.rmtree(directory, ignore_errors=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4b5b6279..d78f46c7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -36,35 +36,34 @@ def test_train(task: str, model: str, data: str) -> None: def test_val(task: str, model: str, data: str) -> None: """Test YOLO validation process for specified task, model, and data using a shell command.""" for end2end in {False, True}: - run( - f"yolo val {task} model={model} data={data} imgsz=32 save_txt save_json visualize end2end={end2end} max_det=100 agnostic_nms" - ) + run(f"yolo val {task} model={model} data={data} imgsz=32 end2end={end2end} max_det=100 agnostic_nms") @pytest.mark.parametrize("task,model,data", TASK_MODEL_DATA) def test_predict(task: str, model: str, data: str) -> None: """Test YOLO prediction on provided sample assets for specified task and model.""" for end2end in {False, True}: - run( - f"yolo {task} predict model={model} source={ASSETS} imgsz=32 save save_crop save_txt visualize end2end={end2end} max_det=100" - ) + run(f"yolo {task} predict model={model} source={ASSETS} imgsz=32 save end2end={end2end} max_det=100") @pytest.mark.parametrize("model", MODELS) -def test_export(model: str) -> None: +def test_export(model: str, tmp_path: Path) -> None: # use tmp_path to prevent the race condition with test_exports.py """Test exporting a YOLO model to TorchScript format.""" for end2end in {False, True}: - run(f"yolo export model={model} format=torchscript imgsz=32 end2end={end2end} max_det=100") + run( + f"yolo export model={model} format=torchscript imgsz=32 end2end={end2end} max_det=100 project={tmp_path} name=export" + ) @pytest.mark.skipif(not TORCH_1_11, reason="RTDETR requires torch>=1.11") def test_rtdetr(task: str = "detect", model: Path = WEIGHTS_DIR / "rtdetr-l.pt", data: str = "coco8.yaml") -> None: """Test the RTDETR functionality within Ultralytics for detection tasks using specified model and data.""" # Add comma, spaces, fraction=0.25 args to test single-image training - run(f"yolo predict {task} model={model} source={ASSETS / 'bus.jpg'} imgsz=160 save save_crop save_txt") + run(f"yolo predict {task} model={model} source={ASSETS / 'bus.jpg'} imgsz=160 save") run(f"yolo train {task} model={model} data={data} --imgsz= 160 epochs =1, cache = disk fraction=0.25") +@pytest.mark.skipif(IS_RASPBERRYPI, reason="Edge devices not intended for heavy FastSAM tests") @pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="MobileSAM with CLIP is not supported in Python 3.12") @pytest.mark.skipif( checks.IS_PYTHON_3_8 and LINUX and ARM64, @@ -77,7 +76,7 @@ def test_fastsam( source = ASSETS / "bus.jpg" run(f"yolo segment val {task} model={model} data={data} imgsz=32") - run(f"yolo segment predict model={model} source={source} imgsz=32 save save_crop save_txt") + run(f"yolo segment predict model={model} source={source} imgsz=32 save") from ultralytics import FastSAM from ultralytics.models.sam import Predictor @@ -87,7 +86,7 @@ def test_fastsam( # Run inference on an image for s in (source, Image.open(source)): - everything_results = sam_model(s, device="cpu", retina_masks=True, imgsz=320, conf=0.4, iou=0.9) + everything_results = sam_model(s, device="cpu", retina_masks=True, imgsz=160, conf=0.4, iou=0.9) # Remove small regions _new_masks, _ = Predictor.remove_small_regions(everything_results[0].masks.data, min_area=20) diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 9ba0418e..f181c4cd 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -52,7 +52,7 @@ def test_amp(): [ # generate all combinations except for exclusion cases (task, dynamic, int8, half, batch, simplify, nms) for task, dynamic, int8, half, batch, simplify, nms in product( - TASKS, [True, False], [False], [False], [1, 2], [True, False], [True, False] + sorted(TASKS), [True, False], [False], [False], [1, 2], [True, False], [True, False] ) if not ( (int8 and half) or (task == "classify" and nms) or (task == "obb" and nms and (not TORCH_1_13 or IS_JETSON)) @@ -85,7 +85,7 @@ def test_export_onnx_matrix(task, dynamic, int8, half, batch, simplify, nms): (task, dynamic, int8, half, batch) # Note: tests reduced below pending compute availability expansion as GPU CI runner utilization is high # for task, dynamic, int8, half, batch in product(TASKS, [True, False], [True, False], [True, False], [1, 2]) - for task, dynamic, int8, half, batch in product(TASKS, [True], [True], [False], [2]) + for task, dynamic, int8, half, batch in product(sorted(TASKS), [True], [True], [False], [2]) if not (int8 and half) # exclude cases where both int8 and half are True ], ) @@ -168,19 +168,19 @@ def test_autobatch(): """Check optimal batch size for YOLO model training using autobatch utility.""" from ultralytics.utils.autobatch import check_train_batch_size - check_train_batch_size(YOLO(MODEL).model.to(f"cuda:{DEVICES[0]}"), imgsz=128, amp=True) + check_train_batch_size(YOLO(MODEL).model.to(f"cuda:{DEVICES[0]}"), imgsz=64, amp=True) @pytest.mark.slow @pytest.mark.skipif(not DEVICES, reason="No CUDA devices available") -def test_utils_benchmarks(): +def test_utils_benchmarks(isolated_model): """Profile YOLO models for performance benchmarks.""" from ultralytics.utils.benchmarks import ProfileModels # Pre-export a dynamic engine model to use dynamic inference - YOLO(MODEL).export(format="engine", imgsz=32, dynamic=True, batch=1, device=DEVICES[0]) + YOLO(isolated_model).export(format="engine", imgsz=32, dynamic=True, batch=1, device=DEVICES[0]) ProfileModels( - [MODEL], + [isolated_model], imgsz=32, half=False, min_time=1, diff --git a/tests/test_exports.py b/tests/test_exports.py index 91c54509..da492682 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -13,7 +13,7 @@ from pathlib import Path import pytest import torch -from tests import MODEL, SOURCE +from tests import SOURCE from ultralytics import YOLO from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS from ultralytics.utils import ARM64, IS_DOCKER, IS_RASPBERRYPI, LINUX, MACOS, MACOS_VERSION, WINDOWS, checks @@ -37,16 +37,16 @@ def skip_rpi_semantic(task): @pytest.mark.parametrize("end2end", [False, True]) -def test_export_torchscript(end2end): +def test_export_torchscript(end2end, isolated_model): """Test YOLO model export to TorchScript format for compatibility and correctness.""" - file = YOLO(MODEL).export(format="torchscript", optimize=False, imgsz=32, end2end=end2end) + file = YOLO(isolated_model).export(format="torchscript", optimize=False, imgsz=32, end2end=end2end) YOLO(file)(SOURCE, imgsz=32) # exported model inference @pytest.mark.parametrize("end2end", [False, True]) -def test_export_onnx(end2end): +def test_export_onnx(end2end, isolated_model): """Test YOLO model export to ONNX format with dynamic axes.""" - file = YOLO(MODEL).export(format="onnx", dynamic=True, imgsz=32, end2end=end2end) + file = YOLO(isolated_model).export(format="onnx", dynamic=True, imgsz=32, end2end=end2end) YOLO(file)(SOURCE, imgsz=32) # exported model inference @@ -86,9 +86,9 @@ def test_torch2onnx_serializes_concurrent_exports(monkeypatch, tmp_path): @pytest.mark.skipif(not TORCH_2_1, reason="OpenVINO requires torch>=2.1") @pytest.mark.parametrize("end2end", [False, True]) -def test_export_openvino(end2end): +def test_export_openvino(end2end, isolated_model): """Test YOLO export to OpenVINO format for model inference compatibility.""" - file = YOLO(MODEL).export(format="openvino", imgsz=32, end2end=end2end) + file = YOLO(isolated_model).export(format="openvino", imgsz=32, end2end=end2end) if WINDOWS: # Ensure a unique export path per test to prevent OpenVINO file writes file = Path(file) @@ -103,7 +103,7 @@ def test_export_openvino(end2end): [ # generate all combinations except for exclusion cases (task, dynamic, int8, half, batch, nms, end2end) for task, dynamic, int8, half, batch, nms, end2end in product( - TASKS, [True, False], [True, False], [True, False], [1, 2], [True, False], [True] + sorted(TASKS), [True, False], [True, False], [True, False], [1, 2], [True, False], [True] ) if not ((int8 and half) or (task == "classify" and nms) or (end2end and nms)) ], @@ -137,7 +137,7 @@ def test_export_openvino_matrix(task, dynamic, int8, half, batch, nms, end2end): [ # generate all combinations except for exclusion cases (task, dynamic, int8, half, batch, simplify, nms, end2end) for task, dynamic, int8, half, batch, simplify, nms, end2end in product( - TASKS, [True, False], [False], [False], [1, 2], [True, False], [True, False], [True, False] + sorted(TASKS), [True, False], [False], [False], [1, 2], [True, False], [True, False], [True, False] ) if not ((int8 and half) or (task == "classify" and nms) or (nms and not TORCH_1_13) or (end2end and nms)) ], @@ -166,7 +166,7 @@ def test_export_onnx_matrix(task, dynamic, int8, half, batch, simplify, nms, end [ # generate all combinations except for exclusion cases (task, dynamic, int8, half, batch, nms, end2end) for task, dynamic, int8, half, batch, nms, end2end in product( - TASKS, [False, True], [False], [False, True], [1, 2], [True, False], [True, False] + sorted(TASKS), [False, True], [False], [False, True], [1, 2], [True, False], [True, False] ) if not ((task == "classify" and nms) or (end2end and nms)) ], @@ -193,7 +193,7 @@ def test_export_torchscript_matrix(task, dynamic, int8, half, batch, nms, end2en [ # generate all combinations except for exclusion cases (task, dynamic, int8, half, nms, batch, end2end) for task, dynamic, int8, half, nms, batch, end2end in product( - TASKS, [True, False], [True, False], [True, False], [True, False], [1], [True, False] + sorted(TASKS), [True, False], [True, False], [True, False], [True, False], [1], [True, False] ) if not (int8 and half) and not (task != "detect" and nms) @@ -232,7 +232,7 @@ def test_export_coreml_matrix(task, dynamic, int8, half, nms, batch, end2end): [ # generate all combinations except for exclusion cases (task, dynamic, int8, half, batch, nms, end2end) for task, dynamic, int8, half, batch, nms, end2end in product( - TASKS, [False], [True, False], [True, False], [1], [True, False], [True, False] + sorted(TASKS), [False], [True, False], [True, False], [1], [True, False], [True, False] ) if not ( (int8 and half) @@ -257,14 +257,14 @@ def test_export_tflite_matrix(task, dynamic, int8, half, batch, nms, end2end): @pytest.mark.skipif(WINDOWS, reason="CoreML not supported on Windows") # RuntimeError: BlobWriter not loaded @pytest.mark.skipif(LINUX and ARM64, reason="CoreML not supported on aarch64 Linux") @pytest.mark.skipif(checks.IS_PYTHON_3_13, reason="CoreML not supported in Python 3.13") -def test_export_coreml(): +def test_export_coreml(isolated_model): """Test YOLO export to CoreML format and check for errors.""" # Capture stdout and stderr stdout, stderr = io.StringIO(), io.StringIO() with redirect_stdout(stdout), redirect_stderr(stderr): - YOLO(MODEL).export(format="coreml", nms=True, imgsz=32) + YOLO(isolated_model).export(format="coreml", nms=True, imgsz=32) if MACOS: - file = YOLO(MODEL).export(format="coreml", imgsz=32) + file = YOLO(isolated_model).export(format="coreml", imgsz=32) YOLO(file)(SOURCE, imgsz=32) # model prediction only supported on macOS for nms=False models # Check captured output for errors @@ -276,33 +276,32 @@ def test_export_coreml(): @pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10, reason="TFLite export requires Python>=3.10") @pytest.mark.skipif(not TORCH_1_13, reason="TFLite export requires torch>=1.13") @pytest.mark.skipif(not LINUX, reason="Test disabled as TF suffers from install conflicts on Windows and macOS") -def test_export_tflite(): +def test_export_tflite(isolated_model): """Test YOLO export to TFLite format under specific OS and Python version conditions.""" - model = YOLO(MODEL) + model = YOLO(isolated_model) file = model.export(format="tflite", imgsz=32) YOLO(file)(SOURCE, imgsz=32) @pytest.mark.skipif(True, reason="Test disabled") @pytest.mark.skipif(not LINUX, reason="TF suffers from install conflicts on Windows and macOS") -def test_export_pb(): +def test_export_pb(isolated_model): """Test YOLO export to TensorFlow's Protobuf (*.pb) format.""" - model = YOLO(MODEL) + model = YOLO(isolated_model) file = model.export(format="pb", imgsz=32) YOLO(file)(SOURCE, imgsz=32) @pytest.mark.skipif(True, reason="Test disabled as Paddle protobuf and ONNX protobuf requirements conflict.") -def test_export_paddle(): +def test_export_paddle(isolated_model): """Test YOLO export to Paddle format, noting protobuf conflicts with ONNX.""" - YOLO(MODEL).export(format="paddle", imgsz=32) + YOLO(isolated_model).export(format="paddle", imgsz=32) -@pytest.mark.slow @pytest.mark.skipif(not TORCH_1_10, reason="MNN export requires torch>=1.10") -def test_export_mnn(): +def test_export_mnn(isolated_model): """Test YOLO export to MNN format (WARNING: MNN test must precede NCNN test or CI error on Windows).""" - file = YOLO(MODEL).export(format="mnn", imgsz=32) + file = YOLO(isolated_model).export(format="mnn", imgsz=32) YOLO(file)(SOURCE, imgsz=32) # exported model inference @@ -312,7 +311,9 @@ def test_export_mnn(): "task, int8, half, batch, end2end", [ # generate all combinations except for exclusion cases (task, int8, half, batch, end2end) - for task, int8, half, batch, end2end in product(TASKS, [True, False], [True, False], [1, 2], [True, False]) + for task, int8, half, batch, end2end in product( + sorted(TASKS), [True, False], [True, False], [1, 2], [True, False] + ) if not (int8 and half) ], ) @@ -324,17 +325,16 @@ def test_export_mnn_matrix(task, int8, half, batch, end2end): Path(file).unlink() # cleanup -@pytest.mark.slow @pytest.mark.skipif(not TORCH_2_0, reason="NCNN inference causes segfault on PyTorch<2.0") -def test_export_ncnn(): +def test_export_ncnn(isolated_model): """Test YOLO export to NCNN format.""" - file = YOLO(MODEL).export(format="ncnn", imgsz=32) + file = YOLO(isolated_model).export(format="ncnn", imgsz=32) YOLO(file)(SOURCE, imgsz=32) # exported model inference @pytest.mark.slow @pytest.mark.skipif(not TORCH_2_0, reason="NCNN inference causes segfault on PyTorch<2.0") -@pytest.mark.parametrize("task, half, batch", list(product(TASKS, [True, False], [1]))) +@pytest.mark.parametrize("task, half, batch", list(product(sorted(TASKS), [True, False], [1]))) def test_export_ncnn_matrix(task, half, batch): """Test YOLO export to NCNN format considering various export configurations.""" skip_rpi_semantic(task) @@ -349,7 +349,7 @@ def test_export_ncnn_matrix(task, half, batch): @pytest.mark.skipif( IS_RASPBERRYPI, reason="Test disabled as IMX export suffers from OOM (Out of Memory) on Raspberry Pi 5 16GB" ) -def test_export_imx(): +def test_export_imx(isolated_model): """Test YOLO export to IMX format.""" model = YOLO("yolo11n.pt") # IMX export only supports YOLO11 file = model.export(format="imx", imgsz=32) @@ -359,9 +359,9 @@ def test_export_imx(): # @pytest.mark.skipif(True, reason="Disabled for debugging ruamel.yaml installation required by executorch") @pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0") @pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows") -def test_export_executorch(): +def test_export_executorch(isolated_model): """Test YOLO model export to ExecuTorch format.""" - file = YOLO(MODEL).export(format="executorch", imgsz=32) + file = YOLO(isolated_model).export(format="executorch", imgsz=32) assert Path(file).exists(), f"ExecuTorch export failed, directory not found: {file}" # Check that .pte file exists in the exported directory pte_file = Path(file) / "model.pte" @@ -376,7 +376,7 @@ def test_export_executorch(): @pytest.mark.slow @pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0") @pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows") -@pytest.mark.parametrize("task", TASKS) +@pytest.mark.parametrize("task", sorted(TASKS)) def test_export_executorch_matrix(task): """Test YOLO export to ExecuTorch format for various task types.""" skip_rpi_semantic(task) @@ -399,10 +399,10 @@ def test_export_executorch_matrix(task): reason="Axelera export is only supported on Linux and is not supported on ARM64 Docker", ) @pytest.mark.skipif(IS_RASPBERRYPI, reason="Test disabled due to OOM (Out of Memory) issues on Raspberry Pi 5 16GB") -def test_export_axelera(): +def test_export_axelera(isolated_model): """Test YOLO export to Axelera format.""" # For faster testing, use a smaller calibration dataset (32 image size crashes axelera export, so 64 is used) - file = YOLO(MODEL).export(format="axelera", imgsz=64, data="coco8.yaml") + file = YOLO(isolated_model).export(format="axelera", imgsz=64, data="coco8.yaml") assert Path(file).exists(), f"Axelera export failed, directory not found: {file}" # Note: Inference testing skipped as it requires Axelera hardware shutil.rmtree(file, ignore_errors=True) # cleanup @@ -413,9 +413,9 @@ def test_export_axelera(): @pytest.mark.skipif( not checks.IS_PYTHON_MINIMUM_3_12, reason="Requires Python>=3.12 for CI validation due to torch upgrades" ) -def test_export_deepx(): +def test_export_deepx(isolated_model): """Test YOLO export to DeepX format.""" - file = YOLO(MODEL).export(format="deepx", imgsz=32) + file = YOLO(isolated_model).export(format="deepx", imgsz=32) assert Path(file).exists(), f"DeepX export failed, directory not found: {file}" # Note: Inference testing skipped as it requires DeepX hardware shutil.rmtree(file, ignore_errors=True) # cleanup @@ -425,7 +425,7 @@ def test_export_deepx(): not (WINDOWS or (LINUX and ARM64)) or sys.version_info < (3, 11), reason="onnxruntime-qnn ships prebuilt wheels only for Windows (x64/ARM64) and Linux ARM64 on Python>=3.11", ) -def test_export_qnn(): +def test_export_qnn(isolated_model): """Test YOLO export to Qualcomm QNN format via the ONNX Runtime QNN Execution Provider.""" import importlib.util @@ -441,7 +441,7 @@ def test_export_qnn(): ).exists() if not has_qnn: pytest.skip("onnxruntime-qnn / QNN Execution Provider not available") - file = YOLO(MODEL).export(format="qnn", imgsz=32) + file = YOLO(isolated_model).export(format="qnn", imgsz=32) assert next(Path(file).rglob("*_qnn.onnx"), None), f"QNN export failed, no context binary found in: {file}" # Note: on-device inference is not exercised here as it requires Qualcomm Snapdragon hardware shutil.rmtree(file, ignore_errors=True) # cleanup diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 7a2400af..bef43c66 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -8,7 +8,7 @@ from pathlib import Path import pytest -from tests import MODEL, SOURCE +from tests import SOURCE from ultralytics import YOLO, download from ultralytics.utils import ASSETS_URL, DATASETS_DIR, SETTINGS from ultralytics.utils.checks import check_requirements @@ -71,7 +71,7 @@ def test_mlflow_keep_run_active(): @pytest.mark.skipif(not check_requirements("tritonclient", install=False), reason="tritonclient[all] not installed") -def test_triton(tmp_path): +def test_triton(tmp_path, isolated_model): """Test NVIDIA Triton Server functionalities with YOLO model.""" check_requirements("tritonclient[all]") from tritonclient.http import InferenceServerClient @@ -82,7 +82,7 @@ def test_triton(tmp_path): triton_model = triton_repo / model_name # Triton model path # Export model to ONNX - f = YOLO(MODEL).export(format="onnx", dynamic=True) + f = YOLO(isolated_model).export(format="onnx", dynamic=True) # Prepare Triton repo (triton_model / "1").mkdir(parents=True, exist_ok=True) diff --git a/tests/test_python.py b/tests/test_python.py index afde67c4..dc4bc486 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -202,7 +202,7 @@ def test_youtube(): """Test YOLO model on a YouTube video stream, handling potential network-related errors.""" model = YOLO(MODEL) try: - model.predict("https://youtu.be/G17sBkb38XQ", imgsz=96, save=True) + model.predict("https://youtu.be/G17sBkb38XQ", imgsz=32, save=True) # Handle internet connection errors and 'urllib.error.HTTPError: HTTP Error 429: Too Many Requests' except (urllib.error.HTTPError, ConnectionError) as e: LOGGER.error(f"YouTube Test Error: {e}") @@ -280,15 +280,15 @@ def test_all_model_yamls(): for m in (ROOT / "cfg" / "models").rglob("*.yaml"): if "rtdetr" in m.name: if TORCH_1_11: - _ = RTDETR(m.name)(SOURCE, imgsz=640) # must be 640 + _ = RTDETR(m.name)(SOURCE, imgsz=160) else: YOLO(m.name) @pytest.mark.skipif(WINDOWS, reason="Windows slow CI export bug https://github.com/ultralytics/ultralytics/pull/16003") -def test_workflow(): +def test_workflow(isolated_model): """Test the complete workflow including training, validation, prediction, and exporting.""" - model = YOLO(MODEL) + model = YOLO(isolated_model) model.train(data="coco8.yaml", epochs=1, imgsz=32, optimizer="SGD") model.val(imgsz=32) model.predict(SOURCE, imgsz=32) @@ -310,7 +310,7 @@ def test_predict_callback_and_setup(): dataset = load_inference_source(source=SOURCE) bs = dataset.bs # access predictor properties - results = model.predict(dataset, stream=True, imgsz=160) # source already setup + results = model.predict(dataset, stream=True, imgsz=32) # source already setup for r, im0, bs in results: print("test_callback", im0.shape) print("test_callback", bs) @@ -350,7 +350,7 @@ def test_results(model: str, tmp_path): def test_labels_and_crops(): """Test output from prediction args for saving YOLO detection labels and crops.""" imgs = [SOURCE, ASSETS / "zidane.jpg"] - results = YOLO(WEIGHTS_DIR / "yolo26n.pt")(imgs, imgsz=320, save_txt=True, save_crop=True) + results = YOLO(WEIGHTS_DIR / "yolo26n.pt")(imgs, imgsz=160, save_txt=True, save_crop=True) save_path = Path(results[0].save_dir) for r in results: im_name = Path(r.path).stem @@ -730,6 +730,7 @@ def test_classify_transforms_train(image, auto_augment, erasing, force_color_jit @pytest.mark.slow +@pytest.mark.skipif(IS_RASPBERRYPI or IS_JETSON, reason="Edge devices not intended for tuning") @pytest.mark.skipif(not ONLINE, reason="environment is offline") def test_model_tune(): """Tune YOLO model for performance improvement.""" @@ -741,6 +742,7 @@ def test_model_tune(): @pytest.mark.slow +@pytest.mark.skipif(IS_RASPBERRYPI or IS_JETSON, reason="Edge devices not intended for tuning") @pytest.mark.skipif(not ONLINE or not checks.IS_PYTHON_MINIMUM_3_10, reason="environment is offline") @pytest.mark.skipif(not checks.check_requirements("ray", install=False), reason="ray[tune] not installed") def test_model_tune_ray(): @@ -767,6 +769,7 @@ def test_model_embeddings(): assert len(model_segment.embed(source=batch, imgsz=32)) == len(batch) +@pytest.mark.skipif(IS_RASPBERRYPI, reason="Edge devices not intended for CLIP-based models") @pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="YOLOWorld with CLIP is not supported in Python 3.12") @pytest.mark.skipif( checks.IS_PYTHON_3_8 and LINUX and ARM64, @@ -803,6 +806,7 @@ def test_yolo_world(): ) +@pytest.mark.skipif(IS_RASPBERRYPI, reason="Edge devices not intended for heavy CLIP-based models") @pytest.mark.skipif(not TORCH_1_13, reason="YOLOE with CLIP requires torch>=1.13") @pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="YOLOE with CLIP is not supported in Python 3.12") @pytest.mark.skipif( diff --git a/tests/test_solutions.py b/tests/test_solutions.py index e09aaea3..07287088 100644 --- a/tests/test_solutions.py +++ b/tests/test_solutions.py @@ -178,10 +178,8 @@ def process_video(solution, video_path: str, needs_frame_count: bool = False): ) def test_solution(name, solution_class, needs_frame_count, video_key, kwargs_update, tmp_path, solution_assets): """Test individual Ultralytics solution with video processing and parameter validation.""" - # Get video path from persistent cache (no copying needed, read-only access) video_path = str(solution_assets(video_key)) if video_key else None - # Update kwargs to use cached paths for parking manager kwargs = {} for key, value in kwargs_update.items(): if key.startswith("temp_"): @@ -269,7 +267,7 @@ def test_plot_with_no_masks(): assert results.plot_im is not None, "Instance segmentation plot returned None" -def test_streamlit_handle_video_upload_creates_file(): +def test_streamlit_handle_video_upload_creates_file(tmp_path): """Test Streamlit video upload logic saves file correctly.""" import io @@ -277,17 +275,18 @@ def test_streamlit_handle_video_upload_creates_file(): fake_file.read = fake_file.getvalue if fake_file is not None: g = io.BytesIO(fake_file.read()) - with open("ultralytics.mp4", "wb") as out: + with open(tmp_path / "ultralytics.mp4", "wb") as out: out.write(g.read()) - output_path = "ultralytics.mp4" + output_path = str(tmp_path / "ultralytics.mp4") else: output_path = None - assert output_path == "ultralytics.mp4", f"Expected output_path 'ultralytics.mp4', got {output_path}" - assert os.path.exists("ultralytics.mp4"), "ultralytics.mp4 file not created" - with open("ultralytics.mp4", "rb") as f: + assert output_path == str(tmp_path / "ultralytics.mp4"), ( + f"Expected output_path '{tmp_path / 'ultralytics.mp4'}', got {output_path}" + ) + assert os.path.exists(tmp_path / "ultralytics.mp4"), "ultralytics.mp4 file not created" + with open(tmp_path / "ultralytics.mp4", "rb") as f: content = f.read() assert content == b"fake video content", f"File content mismatch: {content}" - os.remove("ultralytics.mp4") @pytest.mark.skipif(not TORCH_2_4, reason=f"VisualAISearch requires torch>=2.4 (found torch=={TORCH_VERSION})") diff --git a/ultralytics/cfg/__init__.py b/ultralytics/cfg/__init__.py index c4eddd29..08a8fdee 100644 --- a/ultralytics/cfg/__init__.py +++ b/ultralytics/cfg/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import os import shutil import subprocess import sys @@ -453,7 +454,11 @@ def get_save_dir(args: SimpleNamespace, name: str | None = None) -> Path: project = args.project or "" if not Path(project).is_absolute(): - project = (ROOT.parent / "tests/tmp/runs" if TESTS_RUNNING else RUNS_DIR) / args.task / project + base = ROOT.parent / "tests/tmp/runs" if TESTS_RUNNING else RUNS_DIR + worker = os.environ.get("PYTEST_XDIST_WORKER") + if worker and TESTS_RUNNING: # isolate parallel pytest-xdist workers + base = base / worker + project = base / args.task / project name = name or args.name or f"{args.mode}" save_dir = increment_path(Path(project) / name, exist_ok=args.exist_ok if RANK in {-1, 0} else True)