ultralytics 8.4.19 Platform training pass model ID fix (#23761)

Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
This commit is contained in:
Glenn Jocher 2026-02-28 13:56:07 +01:00 committed by GitHub
parent 33f79093be
commit f080cbbe46
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 47 additions and 36 deletions

View File

@ -1,6 +1,6 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
__version__ = "8.4.18"
__version__ = "8.4.19"
import importlib
import os

View File

@ -180,7 +180,7 @@ def _send_async(event, data, project, name, model_id=None):
_executor.submit(_send, event, data, project, name, model_id)
def _upload_model(model_path, project, name, progress=False, retry=1):
def _upload_model(model_path, project, name, progress=False, retry=1, model_id=None):
"""Upload model checkpoint to Platform via signed URL."""
from ultralytics.utils.uploads import safe_upload
@ -192,9 +192,12 @@ def _upload_model(model_path, project, name, progress=False, retry=1):
# Get signed upload URL from Platform (server sanitizes filename for storage safety)
@Retry(times=3, delay=2)
def get_signed_url():
payload = {"project": project, "name": name, "filename": model_path.name}
if model_id:
payload["modelId"] = model_id # Direct lookup avoids slug mismatch from auto-increment
r = requests.post(
f"{PLATFORM_API_URL}/models/upload",
json={"project": project, "name": name, "filename": model_path.name},
json=payload,
headers={"Authorization": f"Bearer {_api_key}"},
timeout=30,
)
@ -213,9 +216,9 @@ def _upload_model(model_path, project, name, progress=False, retry=1):
return None
def _upload_model_async(model_path, project, name):
def _upload_model_async(model_path, project, name, model_id=None):
"""Upload model asynchronously using bounded thread pool."""
_executor.submit(_upload_model, model_path, project, name)
_executor.submit(_upload_model, model_path, project, name, model_id=model_id)
def _get_environment_info():
@ -282,13 +285,12 @@ def on_pretrain_routine_start(trainer):
if RANK not in {-1, 0} or not trainer.args.project:
return
# Per-trainer state to isolate concurrent training runs
trainer._platform_model_id = None
trainer._platform_last_upload = time()
project, name = _get_project_name(trainer)
url = f"{PLATFORM_URL}/{project}/{name}"
LOGGER.info(f"{PREFIX}Streaming to {url}")
LOGGER.info(f"{PREFIX}Streaming training metrics to Platform")
# Single dict for all platform callback state (like trainer.hub_session for HUB callbacks)
ctx = {"model_id": None, "last_upload": time(), "cancelled": False, "console_logger": None, "system_logger": None}
trainer.platform = ctx
# Create callback to send console output to Platform
def send_console_output(content, line_count, chunk_id):
@ -298,12 +300,12 @@ def on_pretrain_routine_start(trainer):
{"chunkId": chunk_id, "content": content, "lineCount": line_count},
project,
name,
getattr(trainer, "_platform_model_id", None),
ctx["model_id"],
)
# Start console capture with batching (5 lines or 5 seconds)
trainer._platform_console_logger = ConsoleLogger(batch_size=5, flush_interval=5.0, on_flush=send_console_output)
trainer._platform_console_logger.start_capture()
ctx["console_logger"] = ConsoleLogger(batch_size=5, flush_interval=5.0, on_flush=send_console_output)
ctx["console_logger"].start_capture()
# Collect environment info (W&B-style metadata)
environment = _get_environment_info()
@ -326,25 +328,32 @@ def on_pretrain_routine_start(trainer):
retry=4,
)
if response and response.get("modelId"):
trainer._platform_model_id = response["modelId"]
ctx["model_id"] = response["modelId"]
# Server returns actual slug (may differ from requested name due to auto-increment, e.g. "train" → "train-2")
if response.get("modelSlug"):
ctx["model_slug"] = response["modelSlug"]
url = f"{PLATFORM_URL}/{project}/{ctx['model_slug']}"
LOGGER.info(f"{PREFIX}View model at {url}")
# Check for immediate cancellation (cancelled before training started)
# Note: trainer.stop is set in on_pretrain_routine_end (after _setup_train resets it)
if response.get("cancelled"):
trainer._platform_cancelled = True
ctx["cancelled"] = True
else:
LOGGER.warning(f"{PREFIX}Failed to register training session - metrics may not sync to Platform")
def on_pretrain_routine_end(trainer):
"""Apply pre-start cancellation after _setup_train resets trainer.stop."""
if getattr(trainer, "_platform_cancelled", False):
ctx = getattr(trainer, "platform", None)
if ctx and ctx["cancelled"]:
LOGGER.info(f"{PREFIX}Training cancelled from Platform before starting ✅")
trainer.stop = True
def on_fit_epoch_end(trainer):
"""Log training and system metrics at epoch end."""
if RANK not in {-1, 0} or not trainer.args.project:
ctx = getattr(trainer, "platform", None)
if not ctx or RANK not in {-1, 0} or not trainer.args.project:
return
project, name = _get_project_name(trainer)
@ -366,12 +375,12 @@ def on_fit_epoch_end(trainer):
except Exception:
pass
# Get system metrics (cache SystemLogger on trainer for efficiency)
# Get system metrics (cache SystemLogger in platform context for efficiency)
system = {}
try:
if not hasattr(trainer, "_platform_system_logger"):
trainer._platform_system_logger = SystemLogger()
system = trainer._platform_system_logger.get_metrics(rates=True)
if not ctx["system_logger"]:
ctx["system_logger"] = SystemLogger()
system = ctx["system_logger"].get_metrics(rates=True)
except Exception:
pass
@ -387,22 +396,23 @@ def on_fit_epoch_end(trainer):
def _send_and_check_cancel():
"""Send epoch_end and check response for cancellation (runs in background thread)."""
response = _send("epoch_end", payload, project, name, getattr(trainer, "_platform_model_id", None), retry=1)
response = _send("epoch_end", payload, project, name, ctx["model_id"], retry=1)
if response and response.get("cancelled"):
LOGGER.info(f"{PREFIX}Training cancelled from Platform ✅")
trainer.stop = True
trainer._platform_cancelled = True
ctx["cancelled"] = True
_executor.submit(_send_and_check_cancel)
def on_model_save(trainer):
"""Upload model checkpoint (rate limited to every 15 min)."""
if RANK not in {-1, 0} or not trainer.args.project:
ctx = getattr(trainer, "platform", None)
if not ctx or RANK not in {-1, 0} or not trainer.args.project:
return
# Rate limit to every 15 minutes (900 seconds)
if time() - getattr(trainer, "_platform_last_upload", 0) < 900:
if time() - ctx["last_upload"] < 900:
return
model_path = trainer.best if trainer.best and Path(trainer.best).exists() else trainer.last
@ -410,31 +420,32 @@ def on_model_save(trainer):
return
project, name = _get_project_name(trainer)
_upload_model_async(model_path, project, name)
trainer._platform_last_upload = time()
_upload_model_async(model_path, project, name, model_id=ctx["model_id"])
ctx["last_upload"] = time()
def on_train_end(trainer):
"""Log final results, upload best model, and send validation plot data."""
if RANK not in {-1, 0} or not trainer.args.project:
ctx = getattr(trainer, "platform", None)
if not ctx or RANK not in {-1, 0} or not trainer.args.project:
return
project, name = _get_project_name(trainer)
if getattr(trainer, "_platform_cancelled", False):
if ctx["cancelled"]:
LOGGER.info(f"{PREFIX}Uploading partial results for cancelled training")
# Stop console capture
if hasattr(trainer, "_platform_console_logger") and trainer._platform_console_logger:
trainer._platform_console_logger.stop_capture()
trainer._platform_console_logger = None
if ctx["console_logger"]:
ctx["console_logger"].stop_capture()
ctx["console_logger"] = None
# Upload best model (blocking with progress bar to ensure it completes)
gcs_path = None
model_size = None
if trainer.best and Path(trainer.best).exists():
model_size = Path(trainer.best).stat().st_size
gcs_path = _upload_model(trainer.best, project, name, progress=True, retry=3)
gcs_path = _upload_model(trainer.best, project, name, progress=True, retry=3, model_id=ctx["model_id"])
if not gcs_path:
LOGGER.warning(f"{PREFIX}Model will not be available for download on Platform (upload failed)")
@ -467,10 +478,10 @@ def on_train_end(trainer):
},
project,
name,
getattr(trainer, "_platform_model_id", None),
ctx["model_id"],
retry=4, # Critical, more retries
)
url = f"{PLATFORM_URL}/{project}/{name}"
url = f"{PLATFORM_URL}/{project}/{ctx.get('model_slug', name)}"
LOGGER.info(f"{PREFIX}View results at {url}")