diff --git a/docs/en/guides/hyperparameter-tuning.md b/docs/en/guides/hyperparameter-tuning.md index 317645ce..4e18d3e1 100644 --- a/docs/en/guides/hyperparameter-tuning.md +++ b/docs/en/guides/hyperparameter-tuning.md @@ -137,6 +137,31 @@ Here's how to define a search space and use the `model.tune()` method to utilize ) ``` +## Resuming An Interrupted Hyperparameter Tuning Session + +You can resume an interrupted hyperparameter tuning session by passing `resume=True`. You can optionally pass the directory `name` used under `runs/{task}` to resume. Otherwise, it would resume the last interrupted session. You also need to provide all the previous training arguments including `data`, `epochs`, `iterations` and `space`. + +!!! example "Using `resume=True` with `model.tune()`" + + ```python + from ultralytics import YOLO + + # Define a YOLO model + model = YOLO("yolo11n.pt") + + # Define search space + search_space = { + "lr0": (1e-5, 1e-1), + "degrees": (0.0, 45.0), + } + + # Resume previous run + results = model.tune(data="coco8.yaml", epochs=50, iterations=300, space=search_space, resume=True) + + # Resume tuning run with name 'tune_exp' + results = model.tune(data="coco8.yaml", epochs=50, iterations=300, space=search_space, name="tune_exp", resume=True) + ``` + ## Results After you've successfully completed the hyperparameter tuning process, you will obtain several files and directories that encapsulate the results of the tuning. The following describes each: diff --git a/docs/en/integrations/ray-tune.md b/docs/en/integrations/ray-tune.md index c7782587..4abb53f8 100644 --- a/docs/en/integrations/ray-tune.md +++ b/docs/en/integrations/ray-tune.md @@ -124,6 +124,25 @@ In this example, we demonstrate how to use a custom search space for hyperparame In the code snippet above, we create a YOLO model with the "yolo11n.pt" pretrained weights. Then, we call the `tune()` method, specifying the dataset configuration with "coco8.yaml". We provide a custom search space for the initial learning rate `lr0` using a dictionary with the key "lr0" and the value `tune.uniform(1e-5, 1e-1)`. Finally, we pass additional training arguments, such as the number of epochs directly to the tune method as `epochs=50`. +## Resuming An Interrupted Hyperparameter Tuning Session With Ray Tune + +You can resume an interrupted Ray Tune session by passing `resume=True`. You can optionally pass the directory `name` used by Ray Tune under `runs/{task}` to resume. Otherwise, it would resume the last interrupted session. You don't need to provide the `iterations` and `space` again, but you need to provide the rest of the training arguments again including `data` and `epochs`. + +!!! example "Using `resume=True` with `model.tune()`" + + ```python + from ultralytics import YOLO + + # Define a YOLO model + model = YOLO("yolo11n.pt") + + # Resume previous run + results = model.tune(use_ray=True, data="coco8.yaml", epochs=50, resume=True) + + # Resume Ray Tune run with name 'tune_exp_2' + results = model.tune(use_ray=True, data="coco8.yaml", epochs=50, name="tune_exp_2", resume=True) + ``` + ## Processing Ray Tune Results After running a hyperparameter tuning experiment with Ray Tune, you might want to perform various analyses on the obtained results. This guide will take you through common workflows for processing and analyzing these results. diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index c5db5600..43eef9d5 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license -__version__ = "8.3.102" +__version__ = "8.3.103" import os diff --git a/ultralytics/engine/tuner.py b/ultralytics/engine/tuner.py index d9615a12..77982dfa 100644 --- a/ultralytics/engine/tuner.py +++ b/ultralytics/engine/tuner.py @@ -93,8 +93,9 @@ class Tuner: "copy_paste": (0.0, 1.0), # segment copy-paste (probability) } self.args = get_cfg(overrides=args) + self.args.exist_ok = self.args.resume # resume w/ same tune_dir self.tune_dir = get_save_dir(self.args, name=self.args.name or "tune") - self.args.name = None # reset to not affect training directory + self.args.name, self.args.exist_ok, self.args.resume = (None, False, False) # reset to not affect training self.tune_csv = self.tune_dir / "tune_results.csv" self.callbacks = _callbacks or callbacks.get_default_callbacks() self.prefix = colorstr("Tuner: ") @@ -173,7 +174,12 @@ class Tuner: t0 = time.time() best_save_dir, best_metrics = None, None (self.tune_dir / "weights").mkdir(parents=True, exist_ok=True) - for i in range(iterations): + start = 0 + if self.tune_csv.exists(): + x = np.loadtxt(self.tune_csv, ndmin=2, delimiter=",", skiprows=1) + start = x.shape[0] + LOGGER.info(f"{self.prefix}Resuming tuning run {self.tune_dir} from iteration {start + 1}...") + for i in range(start, iterations): # Mutate hyperparameters mutated_hyp = self._mutate() LOGGER.info(f"{self.prefix}Starting iteration {i + 1}/{iterations} with hyperparameters: {mutated_hyp}") diff --git a/ultralytics/utils/tuner.py b/ultralytics/utils/tuner.py index 39069aab..08cf158f 100644 --- a/ultralytics/utils/tuner.py +++ b/ultralytics/utils/tuner.py @@ -1,7 +1,7 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license from ultralytics.cfg import TASK2DATA, TASK2METRIC, get_cfg, get_save_dir -from ultralytics.utils import DEFAULT_CFG, DEFAULT_CFG_DICT, LOGGER, NUM_THREADS, checks +from ultralytics.utils import DEFAULT_CFG, DEFAULT_CFG_DICT, LOGGER, NUM_THREADS, checks, colorstr def run_ray_tune( @@ -95,7 +95,7 @@ def run_ray_tune( return results.results_dict # Get search space - if not space: + if not space and not train_args.get("resume"): space = default_space LOGGER.warning("WARNING ⚠️ search space not provided, using default search space.") @@ -123,15 +123,23 @@ def run_ray_tune( # Create the Ray Tune hyperparameter search tuner tune_dir = get_save_dir( - get_cfg(DEFAULT_CFG, train_args), name=train_args.pop("name", "tune") + get_cfg( + DEFAULT_CFG, + {**train_args, **{"exist_ok": train_args.pop("resume", False)}}, # resume w/ same tune_dir + ), + name=train_args.pop("name", "tune"), # runs/{task}/{tune_dir} ).resolve() # must be absolute dir tune_dir.mkdir(parents=True, exist_ok=True) - tuner = tune.Tuner( - trainable_with_resources, - param_space=space, - tune_config=tune.TuneConfig(scheduler=asha_scheduler, num_samples=max_samples), - run_config=RunConfig(callbacks=tuner_callbacks, storage_path=tune_dir), - ) + if tune.Tuner.can_restore(tune_dir): + LOGGER.info(f"{colorstr('Tuner: ')} Resuming tuning run {tune_dir}...") + tuner = tune.Tuner.restore(str(tune_dir), trainable=trainable_with_resources, resume_errored=True) + else: + tuner = tune.Tuner( + trainable_with_resources, + param_space=space, + tune_config=tune.TuneConfig(scheduler=asha_scheduler, num_samples=max_samples), + run_config=RunConfig(callbacks=tuner_callbacks, storage_path=tune_dir.parent, name=tune_dir.name), + ) # Run the hyperparameter search tuner.fit()