diff --git a/docs/en/datasets/classify/caltech101.md b/docs/en/datasets/classify/caltech101.md index af0158a6..550e1f4e 100644 --- a/docs/en/datasets/classify/caltech101.md +++ b/docs/en/datasets/classify/caltech101.md @@ -8,6 +8,21 @@ keywords: Caltech-101, dataset, object recognition, machine learning, computer v The [Caltech-101](https://data.caltech.edu/records/mzrjq-6wc02) dataset is a widely used dataset for object recognition tasks, containing around 9,000 images from 101 object categories. The categories were chosen to reflect a variety of real-world objects, and the images themselves were carefully selected and annotated to provide a challenging benchmark for object recognition algorithms. +

+
+ +
+ Watch: How to Train Image Classification Model using Caltech-256 Dataset with Ultralytics HUB +

+ +!!! note "Automatic Data Splitting" + + The Caltech-101 dataset, as provided, does not come with pre-defined train/validation splits. However, when you use the training commands provided in the usage examples below, the Ultralytics framework will automatically split the dataset for you. The default split used is 80% for the training set and 20% for the validation set. + ## Key Features - The Caltech-101 dataset comprises around 9,000 color images divided into 101 categories. diff --git a/docs/en/datasets/classify/caltech256.md b/docs/en/datasets/classify/caltech256.md index b3dac996..a8ed3c81 100644 --- a/docs/en/datasets/classify/caltech256.md +++ b/docs/en/datasets/classify/caltech256.md @@ -19,6 +19,10 @@ The [Caltech-256](https://data.caltech.edu/records/nyy15-4j048) dataset is an ex Watch: How to Train Image Classification Model using Caltech-256 Dataset with Ultralytics HUB

+!!! note "Automatic Data Splitting" + + The Caltech-256 dataset, as provided, does not come with pre-defined train/validation splits. However, when you use the training commands provided in the usage examples below, the Ultralytics framework will automatically split the dataset for you. The default split used is 80% for the training set and 20% for the validation set. + ## Key Features - The Caltech-256 dataset comprises around 30,000 color images divided into 257 categories. diff --git a/docs/en/datasets/detect/xview.md b/docs/en/datasets/detect/xview.md index 9aae550b..df80952a 100644 --- a/docs/en/datasets/detect/xview.md +++ b/docs/en/datasets/detect/xview.md @@ -15,6 +15,15 @@ The [xView](http://xviewdataset.org/) dataset is one of the largest publicly ava xView builds on the success of challenges like [Common Objects in Context (COCO)](../detect/coco.md) and aims to leverage computer vision to analyze the growing amount of available imagery from space in order to understand the visual world in new ways and address a range of important applications. +!!! warning "Manual Download Required" + + The xView dataset is **not** automatically downloaded by Ultralytics scripts. You **must** manually download the dataset first from the official source: + + - **Source:** DIUx xView 2018 Challenge by U.S. National Geospatial-Intelligence Agency (NGA) + - **URL:** [https://challenge.xviewdataset.org](https://challenge.xviewdataset.org) + + **Important:** After downloading the necessary files (e.g., `train_images.tif`, `val_images.tif`, `xView_train.geojson`), you need to extract them and place them into the correct directory structure, typically expected under a `datasets/xView/` folder, **before** running the training commands provided below. Ensure the dataset is properly set up as per the challenge instructions. + ## Key Features - xView contains over 1 million object instances across 60 classes. diff --git a/docs/en/reference/data/split.md b/docs/en/reference/data/split.md new file mode 100644 index 00000000..c60bfb7f --- /dev/null +++ b/docs/en/reference/data/split.md @@ -0,0 +1,20 @@ +--- +description: Learn how to split datasets into train, validation, and test subsets using Ultralytics utilities for efficient data preparation. +keywords: dataset splitting, autosplit dataset, training dataset preparation, validation set creation, Ultralytics data tools +--- + +# Reference for `ultralytics/data/split.py` + +!!! note + + This file is available at [https://github.com/ultralytics/ultralytics/blob/main/ultralytics/data/split.py](https://github.com/ultralytics/ultralytics/blob/main/ultralytics/data/split.py). If you spot a problem please help fix it by [contributing](https://docs.ultralytics.com/help/contributing/) a [Pull Request](https://github.com/ultralytics/ultralytics/edit/main/ultralytics/data/split.py) 🛠️. Thank you 🙏! + +
+ +## ::: ultralytics.data.split.split_classify_dataset + +



+ +## ::: ultralytics.data.split.autosplit + +

diff --git a/docs/en/reference/data/utils.md b/docs/en/reference/data/utils.md index a9436ca9..30afd728 100644 --- a/docs/en/reference/data/utils.md +++ b/docs/en/reference/data/utils.md @@ -71,10 +71,6 @@ keywords: Ultralytics, dataset utils, data handling, image verification, Python,



-## ::: ultralytics.data.utils.autosplit - -



- ## ::: ultralytics.data.utils.load_dataset_cache_file



diff --git a/docs/en/usage/simple-utilities.md b/docs/en/usage/simple-utilities.md index f11a39fd..75a8fc31 100644 --- a/docs/en/usage/simple-utilities.md +++ b/docs/en/usage/simple-utilities.md @@ -206,7 +206,7 @@ autosplit( ) ``` -See the [Reference page](../reference/data/utils.md#ultralytics.data.utils.autosplit) for additional details on this function. +See the [Reference page](../reference/data/split.md#ultralytics.data.split.autosplit) for additional details on this function. ### Segment-polygon to Binary Mask diff --git a/tests/test_python.py b/tests/test_python.py index 569b083d..2be12e05 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -310,7 +310,8 @@ def test_labels_and_crops(): @pytest.mark.skipif(not ONLINE, reason="environment is offline") def test_data_utils(): """Test utility functions in ultralytics/data/utils.py, including dataset stats and auto-splitting.""" - from ultralytics.data.utils import HUBDatasetStats, autosplit + from ultralytics.data.split import autosplit + from ultralytics.data.utils import HUBDatasetStats from ultralytics.utils.downloads import zip_directory # from ultralytics.utils.files import WorkingDirectory diff --git a/ultralytics/cfg/__init__.py b/ultralytics/cfg/__init__.py index a0f41b08..5dd48b00 100644 --- a/ultralytics/cfg/__init__.py +++ b/ultralytics/cfg/__init__.py @@ -570,9 +570,7 @@ def handle_yolo_hub(args: List[str]) -> None: or 'logout'. For 'login', an optional second argument can be the API key. Examples: - ```bash - yolo login YOUR_API_KEY - ``` + $ yolo login YOUR_API_KEY Notes: - The function imports the 'hub' module from ultralytics to perform login and logout operations. diff --git a/ultralytics/data/base.py b/ultralytics/data/base.py index 56975b29..f4632e93 100644 --- a/ultralytics/data/base.py +++ b/ultralytics/data/base.py @@ -415,19 +415,17 @@ class BaseDataset(Dataset): """ Users can customize their own format here. - Note: + Examples: Ensure output is a dictionary with the following keys: - ```python - dict( - im_file=im_file, - shape=shape, # format: (height, width) - cls=cls, - bboxes=bboxes, # xywh - segments=segments, # xy - keypoints=keypoints, # xy - normalized=True, # or False - bbox_format="xyxy", # or xywh, ltwh - ) - ``` + >>> dict( + ... im_file=im_file, + ... shape=shape, # format: (height, width) + ... cls=cls, + ... bboxes=bboxes, # xywh + ... segments=segments, # xy + ... keypoints=keypoints, # xy + ... normalized=True, # or False + ... bbox_format="xyxy", # or xywh, ltwh + ... ) """ raise NotImplementedError diff --git a/ultralytics/data/split.py b/ultralytics/data/split.py new file mode 100644 index 00000000..66271d4f --- /dev/null +++ b/ultralytics/data/split.py @@ -0,0 +1,123 @@ +import random +import shutil +from pathlib import Path + +from ultralytics.data.utils import IMG_FORMATS, img2label_paths +from ultralytics.utils import DATASETS_DIR, LOGGER, TQDM + + +def split_classify_dataset(source_dir, train_ratio=0.8): + """ + Split dataset into train and val directories in a new directory. + + Creates a new directory '{source_dir}_split' with train/val subdirectories, preserving the original class + structure with an 80/20 split by default. + + Directory structure: + Before: + caltech/ + ├── class1/ + │ ├── img1.jpg + │ ├── img2.jpg + │ └── ... + ├── class2/ + │ ├── img1.jpg + │ └── ... + └── ... + + After: + caltech_split/ + ├── train/ + │ ├── class1/ + │ │ ├── img1.jpg + │ │ └── ... + │ ├── class2/ + │ │ ├── img1.jpg + │ │ └── ... + │ └── ... + └── val/ + ├── class1/ + │ ├── img2.jpg + │ └── ... + ├── class2/ + │ └── ... + └── ... + + Args: + source_dir (str | Path): Path to Caltech dataset root directory. + train_ratio (float): Ratio for train split, between 0 and 1. + + Examples: + >>> # Split dataset with default 80/20 ratio + >>> split_classify_dataset("path/to/caltech") + >>> # Split with custom ratio + >>> split_classify_dataset("path/to/caltech", 0.75) + """ + source_path = Path(source_dir) + split_path = Path(f"{source_path}_split") + train_path, val_path = split_path / "train", split_path / "val" + + # Create directory structure + split_path.mkdir(exist_ok=True) + train_path.mkdir(exist_ok=True) + val_path.mkdir(exist_ok=True) + + # Process class directories + class_dirs = [d for d in source_path.iterdir() if d.is_dir()] + total_images = sum(len(list(d.glob("*.*"))) for d in class_dirs) + stats = f"{len(class_dirs)} classes, {total_images} images" + LOGGER.info(f"Splitting {source_path} ({stats}) into {train_ratio:.0%} train, {1 - train_ratio:.0%} val...") + + for class_dir in class_dirs: + # Create class directories + (train_path / class_dir.name).mkdir(exist_ok=True) + (val_path / class_dir.name).mkdir(exist_ok=True) + + # Split and copy files + image_files = list(class_dir.glob("*.*")) + random.shuffle(image_files) + split_idx = int(len(image_files) * train_ratio) + + for img in image_files[:split_idx]: + shutil.copy2(img, train_path / class_dir.name / img.name) + + for img in image_files[split_idx:]: + shutil.copy2(img, val_path / class_dir.name / img.name) + + LOGGER.info(f"Split complete in {split_path} ✅") + return split_path + + +def autosplit(path=DATASETS_DIR / "coco8/images", weights=(0.9, 0.1, 0.0), annotated_only=False): + """ + Automatically split a dataset into train/val/test splits and save the resulting splits into autosplit_*.txt files. + + Args: + path (Path, optional): Path to images directory. + weights (list | tuple, optional): Train, validation, and test split fractions. + annotated_only (bool, optional): If True, only images with an associated txt file are used. + + Examples: + >>> from ultralytics.data.split import autosplit + >>> autosplit() + """ + path = Path(path) # images dir + files = sorted(x for x in path.rglob("*.*") if x.suffix[1:].lower() in IMG_FORMATS) # image files only + n = len(files) # number of files + random.seed(0) # for reproducibility + indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split + + txt = ["autosplit_train.txt", "autosplit_val.txt", "autosplit_test.txt"] # 3 txt files + for x in txt: + if (path.parent / x).exists(): + (path.parent / x).unlink() # remove existing + + LOGGER.info(f"Autosplitting images from {path}" + ", using *.txt labeled images only" * annotated_only) + for i, img in TQDM(zip(indices, files), total=n): + if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label + with open(path.parent / txt[i], "a", encoding="utf-8") as f: + f.write(f"./{img.relative_to(path.parent).as_posix()}" + "\n") # add image to txt file + + +if __name__ == "__main__": + split_classify_dataset("../datasets/caltech101") diff --git a/ultralytics/data/utils.py b/ultralytics/data/utils.py index ebe51c81..c20a010a 100644 --- a/ultralytics/data/utils.py +++ b/ultralytics/data/utils.py @@ -506,6 +506,17 @@ def check_cls_dataset(dataset, split=""): s = f"Dataset download success ✅ ({time.time() - t:.1f}s), saved to {colorstr('bold', data_dir)}\n" LOGGER.info(s) train_set = data_dir / "train" + if not train_set.is_dir(): + LOGGER.warning(f"WARNING ⚠️ Dataset 'split=train' not found at {train_set}") + image_files = list(data_dir.rglob("*.jpg")) + list(data_dir.rglob("*.png")) + if image_files: + from ultralytics.data.split import split_classify_dataset + + LOGGER.info(f"Found {len(image_files)} images in subdirectories. Attempting to split...") + data_dir = split_classify_dataset(data_dir, train_ratio=0.8) + train_set = data_dir / "train" + else: + LOGGER.error(f"No images found in {data_dir} or its subdirectories.") val_set = ( data_dir / "val" if (data_dir / "val").exists() @@ -737,37 +748,6 @@ def compress_one_image(f, f_new=None, max_dim=1920, quality=50): cv2.imwrite(str(f_new or f), im) -def autosplit(path=DATASETS_DIR / "coco8/images", weights=(0.9, 0.1, 0.0), annotated_only=False): - """ - Automatically split a dataset into train/val/test splits and save the resulting splits into autosplit_*.txt files. - - Args: - path (Path, optional): Path to images directory. - weights (list | tuple, optional): Train, validation, and test split fractions. - annotated_only (bool, optional): If True, only images with an associated txt file are used. - - Examples: - >>> from ultralytics.data.utils import autosplit - >>> autosplit() - """ - path = Path(path) # images dir - files = sorted(x for x in path.rglob("*.*") if x.suffix[1:].lower() in IMG_FORMATS) # image files only - n = len(files) # number of files - random.seed(0) # for reproducibility - indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split - - txt = ["autosplit_train.txt", "autosplit_val.txt", "autosplit_test.txt"] # 3 txt files - for x in txt: - if (path.parent / x).exists(): - (path.parent / x).unlink() # remove existing - - LOGGER.info(f"Autosplitting images from {path}" + ", using *.txt labeled images only" * annotated_only) - for i, img in TQDM(zip(indices, files), total=n): - if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label - with open(path.parent / txt[i], "a", encoding="utf-8") as f: - f.write(f"./{img.relative_to(path.parent).as_posix()}" + "\n") # add image to txt file - - def load_dataset_cache_file(path): """Load an Ultralytics *.cache dictionary from path.""" import gc