Resume WandB run when training resumes (#24110)
Co-authored-by: UltralyticsAssistant <web@ultralytics.com> Co-authored-by: Onuralp SEZER <onuralp@ultralytics.com> Co-authored-by: Jing Qiu <61612323+Laughing-q@users.noreply.github.com>
This commit is contained in:
parent
fa1db1f467
commit
fde498ad30
@ -129,13 +129,19 @@ def on_pretrain_routine_start(trainer):
|
||||
"""Initialize and start wandb project if module is present."""
|
||||
if not wb.run:
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
name = str(trainer.args.name).replace("/", "-").replace(" ", "_")
|
||||
latest_run = Path(trainer.save_dir) / "wandb" / "latest-run"
|
||||
resuming = trainer.args.resume and latest_run.exists()
|
||||
wb.init(
|
||||
project=str(trainer.args.project).replace("/", "-") if trainer.args.project else "Ultralytics",
|
||||
name=name,
|
||||
config=vars(trainer.args),
|
||||
id=f"{name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}", # add unique id
|
||||
id=latest_run.resolve().name.split("-", 2)[2]
|
||||
if resuming
|
||||
else f"{name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
|
||||
resume="allow" if resuming else None,
|
||||
dir=str(trainer.save_dir),
|
||||
)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user