ultralytics 8.3.103 New RayTune resume=True functionality (#20037)
Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> Co-authored-by: UltralyticsAssistant <web@ultralytics.com> Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
parent
59975c2523
commit
fbbe463d3c
@ -137,6 +137,31 @@ Here's how to define a search space and use the `model.tune()` method to utilize
|
||||
)
|
||||
```
|
||||
|
||||
## Resuming An Interrupted Hyperparameter Tuning Session
|
||||
|
||||
You can resume an interrupted hyperparameter tuning session by passing `resume=True`. You can optionally pass the directory `name` used under `runs/{task}` to resume. Otherwise, it would resume the last interrupted session. You also need to provide all the previous training arguments including `data`, `epochs`, `iterations` and `space`.
|
||||
|
||||
!!! example "Using `resume=True` with `model.tune()`"
|
||||
|
||||
```python
|
||||
from ultralytics import YOLO
|
||||
|
||||
# Define a YOLO model
|
||||
model = YOLO("yolo11n.pt")
|
||||
|
||||
# Define search space
|
||||
search_space = {
|
||||
"lr0": (1e-5, 1e-1),
|
||||
"degrees": (0.0, 45.0),
|
||||
}
|
||||
|
||||
# Resume previous run
|
||||
results = model.tune(data="coco8.yaml", epochs=50, iterations=300, space=search_space, resume=True)
|
||||
|
||||
# Resume tuning run with name 'tune_exp'
|
||||
results = model.tune(data="coco8.yaml", epochs=50, iterations=300, space=search_space, name="tune_exp", resume=True)
|
||||
```
|
||||
|
||||
## Results
|
||||
|
||||
After you've successfully completed the hyperparameter tuning process, you will obtain several files and directories that encapsulate the results of the tuning. The following describes each:
|
||||
|
||||
@ -124,6 +124,25 @@ In this example, we demonstrate how to use a custom search space for hyperparame
|
||||
|
||||
In the code snippet above, we create a YOLO model with the "yolo11n.pt" pretrained weights. Then, we call the `tune()` method, specifying the dataset configuration with "coco8.yaml". We provide a custom search space for the initial learning rate `lr0` using a dictionary with the key "lr0" and the value `tune.uniform(1e-5, 1e-1)`. Finally, we pass additional training arguments, such as the number of epochs directly to the tune method as `epochs=50`.
|
||||
|
||||
## Resuming An Interrupted Hyperparameter Tuning Session With Ray Tune
|
||||
|
||||
You can resume an interrupted Ray Tune session by passing `resume=True`. You can optionally pass the directory `name` used by Ray Tune under `runs/{task}` to resume. Otherwise, it would resume the last interrupted session. You don't need to provide the `iterations` and `space` again, but you need to provide the rest of the training arguments again including `data` and `epochs`.
|
||||
|
||||
!!! example "Using `resume=True` with `model.tune()`"
|
||||
|
||||
```python
|
||||
from ultralytics import YOLO
|
||||
|
||||
# Define a YOLO model
|
||||
model = YOLO("yolo11n.pt")
|
||||
|
||||
# Resume previous run
|
||||
results = model.tune(use_ray=True, data="coco8.yaml", epochs=50, resume=True)
|
||||
|
||||
# Resume Ray Tune run with name 'tune_exp_2'
|
||||
results = model.tune(use_ray=True, data="coco8.yaml", epochs=50, name="tune_exp_2", resume=True)
|
||||
```
|
||||
|
||||
## Processing Ray Tune Results
|
||||
|
||||
After running a hyperparameter tuning experiment with Ray Tune, you might want to perform various analyses on the obtained results. This guide will take you through common workflows for processing and analyzing these results.
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
__version__ = "8.3.102"
|
||||
__version__ = "8.3.103"
|
||||
|
||||
import os
|
||||
|
||||
|
||||
@ -93,8 +93,9 @@ class Tuner:
|
||||
"copy_paste": (0.0, 1.0), # segment copy-paste (probability)
|
||||
}
|
||||
self.args = get_cfg(overrides=args)
|
||||
self.args.exist_ok = self.args.resume # resume w/ same tune_dir
|
||||
self.tune_dir = get_save_dir(self.args, name=self.args.name or "tune")
|
||||
self.args.name = None # reset to not affect training directory
|
||||
self.args.name, self.args.exist_ok, self.args.resume = (None, False, False) # reset to not affect training
|
||||
self.tune_csv = self.tune_dir / "tune_results.csv"
|
||||
self.callbacks = _callbacks or callbacks.get_default_callbacks()
|
||||
self.prefix = colorstr("Tuner: ")
|
||||
@ -173,7 +174,12 @@ class Tuner:
|
||||
t0 = time.time()
|
||||
best_save_dir, best_metrics = None, None
|
||||
(self.tune_dir / "weights").mkdir(parents=True, exist_ok=True)
|
||||
for i in range(iterations):
|
||||
start = 0
|
||||
if self.tune_csv.exists():
|
||||
x = np.loadtxt(self.tune_csv, ndmin=2, delimiter=",", skiprows=1)
|
||||
start = x.shape[0]
|
||||
LOGGER.info(f"{self.prefix}Resuming tuning run {self.tune_dir} from iteration {start + 1}...")
|
||||
for i in range(start, iterations):
|
||||
# Mutate hyperparameters
|
||||
mutated_hyp = self._mutate()
|
||||
LOGGER.info(f"{self.prefix}Starting iteration {i + 1}/{iterations} with hyperparameters: {mutated_hyp}")
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
from ultralytics.cfg import TASK2DATA, TASK2METRIC, get_cfg, get_save_dir
|
||||
from ultralytics.utils import DEFAULT_CFG, DEFAULT_CFG_DICT, LOGGER, NUM_THREADS, checks
|
||||
from ultralytics.utils import DEFAULT_CFG, DEFAULT_CFG_DICT, LOGGER, NUM_THREADS, checks, colorstr
|
||||
|
||||
|
||||
def run_ray_tune(
|
||||
@ -95,7 +95,7 @@ def run_ray_tune(
|
||||
return results.results_dict
|
||||
|
||||
# Get search space
|
||||
if not space:
|
||||
if not space and not train_args.get("resume"):
|
||||
space = default_space
|
||||
LOGGER.warning("WARNING ⚠️ search space not provided, using default search space.")
|
||||
|
||||
@ -123,15 +123,23 @@ def run_ray_tune(
|
||||
|
||||
# Create the Ray Tune hyperparameter search tuner
|
||||
tune_dir = get_save_dir(
|
||||
get_cfg(DEFAULT_CFG, train_args), name=train_args.pop("name", "tune")
|
||||
get_cfg(
|
||||
DEFAULT_CFG,
|
||||
{**train_args, **{"exist_ok": train_args.pop("resume", False)}}, # resume w/ same tune_dir
|
||||
),
|
||||
name=train_args.pop("name", "tune"), # runs/{task}/{tune_dir}
|
||||
).resolve() # must be absolute dir
|
||||
tune_dir.mkdir(parents=True, exist_ok=True)
|
||||
tuner = tune.Tuner(
|
||||
trainable_with_resources,
|
||||
param_space=space,
|
||||
tune_config=tune.TuneConfig(scheduler=asha_scheduler, num_samples=max_samples),
|
||||
run_config=RunConfig(callbacks=tuner_callbacks, storage_path=tune_dir),
|
||||
)
|
||||
if tune.Tuner.can_restore(tune_dir):
|
||||
LOGGER.info(f"{colorstr('Tuner: ')} Resuming tuning run {tune_dir}...")
|
||||
tuner = tune.Tuner.restore(str(tune_dir), trainable=trainable_with_resources, resume_errored=True)
|
||||
else:
|
||||
tuner = tune.Tuner(
|
||||
trainable_with_resources,
|
||||
param_space=space,
|
||||
tune_config=tune.TuneConfig(scheduler=asha_scheduler, num_samples=max_samples),
|
||||
run_config=RunConfig(callbacks=tuner_callbacks, storage_path=tune_dir.parent, name=tune_dir.name),
|
||||
)
|
||||
|
||||
# Run the hyperparameter search
|
||||
tuner.fit()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user