From f080cbbe46fe5df76a2dc2d719e594db69d482de Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sat, 28 Feb 2026 13:56:07 +0100 Subject: [PATCH] `ultralytics 8.4.19` Platform training pass model ID fix (#23761) Co-authored-by: UltralyticsAssistant --- ultralytics/__init__.py | 2 +- ultralytics/utils/callbacks/platform.py | 81 ++++++++++++++----------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index 5031be34..f7bac265 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license -__version__ = "8.4.18" +__version__ = "8.4.19" import importlib import os diff --git a/ultralytics/utils/callbacks/platform.py b/ultralytics/utils/callbacks/platform.py index 620560cf..db96bb6e 100644 --- a/ultralytics/utils/callbacks/platform.py +++ b/ultralytics/utils/callbacks/platform.py @@ -180,7 +180,7 @@ def _send_async(event, data, project, name, model_id=None): _executor.submit(_send, event, data, project, name, model_id) -def _upload_model(model_path, project, name, progress=False, retry=1): +def _upload_model(model_path, project, name, progress=False, retry=1, model_id=None): """Upload model checkpoint to Platform via signed URL.""" from ultralytics.utils.uploads import safe_upload @@ -192,9 +192,12 @@ def _upload_model(model_path, project, name, progress=False, retry=1): # Get signed upload URL from Platform (server sanitizes filename for storage safety) @Retry(times=3, delay=2) def get_signed_url(): + payload = {"project": project, "name": name, "filename": model_path.name} + if model_id: + payload["modelId"] = model_id # Direct lookup avoids slug mismatch from auto-increment r = requests.post( f"{PLATFORM_API_URL}/models/upload", - json={"project": project, "name": name, "filename": model_path.name}, + json=payload, headers={"Authorization": f"Bearer {_api_key}"}, timeout=30, ) @@ -213,9 +216,9 @@ def _upload_model(model_path, project, name, progress=False, retry=1): return None -def _upload_model_async(model_path, project, name): +def _upload_model_async(model_path, project, name, model_id=None): """Upload model asynchronously using bounded thread pool.""" - _executor.submit(_upload_model, model_path, project, name) + _executor.submit(_upload_model, model_path, project, name, model_id=model_id) def _get_environment_info(): @@ -282,13 +285,12 @@ def on_pretrain_routine_start(trainer): if RANK not in {-1, 0} or not trainer.args.project: return - # Per-trainer state to isolate concurrent training runs - trainer._platform_model_id = None - trainer._platform_last_upload = time() - project, name = _get_project_name(trainer) - url = f"{PLATFORM_URL}/{project}/{name}" - LOGGER.info(f"{PREFIX}Streaming to {url}") + LOGGER.info(f"{PREFIX}Streaming training metrics to Platform") + + # Single dict for all platform callback state (like trainer.hub_session for HUB callbacks) + ctx = {"model_id": None, "last_upload": time(), "cancelled": False, "console_logger": None, "system_logger": None} + trainer.platform = ctx # Create callback to send console output to Platform def send_console_output(content, line_count, chunk_id): @@ -298,12 +300,12 @@ def on_pretrain_routine_start(trainer): {"chunkId": chunk_id, "content": content, "lineCount": line_count}, project, name, - getattr(trainer, "_platform_model_id", None), + ctx["model_id"], ) # Start console capture with batching (5 lines or 5 seconds) - trainer._platform_console_logger = ConsoleLogger(batch_size=5, flush_interval=5.0, on_flush=send_console_output) - trainer._platform_console_logger.start_capture() + ctx["console_logger"] = ConsoleLogger(batch_size=5, flush_interval=5.0, on_flush=send_console_output) + ctx["console_logger"].start_capture() # Collect environment info (W&B-style metadata) environment = _get_environment_info() @@ -326,25 +328,32 @@ def on_pretrain_routine_start(trainer): retry=4, ) if response and response.get("modelId"): - trainer._platform_model_id = response["modelId"] + ctx["model_id"] = response["modelId"] + # Server returns actual slug (may differ from requested name due to auto-increment, e.g. "train" → "train-2") + if response.get("modelSlug"): + ctx["model_slug"] = response["modelSlug"] + url = f"{PLATFORM_URL}/{project}/{ctx['model_slug']}" + LOGGER.info(f"{PREFIX}View model at {url}") # Check for immediate cancellation (cancelled before training started) # Note: trainer.stop is set in on_pretrain_routine_end (after _setup_train resets it) if response.get("cancelled"): - trainer._platform_cancelled = True + ctx["cancelled"] = True else: LOGGER.warning(f"{PREFIX}Failed to register training session - metrics may not sync to Platform") def on_pretrain_routine_end(trainer): """Apply pre-start cancellation after _setup_train resets trainer.stop.""" - if getattr(trainer, "_platform_cancelled", False): + ctx = getattr(trainer, "platform", None) + if ctx and ctx["cancelled"]: LOGGER.info(f"{PREFIX}Training cancelled from Platform before starting ✅") trainer.stop = True def on_fit_epoch_end(trainer): """Log training and system metrics at epoch end.""" - if RANK not in {-1, 0} or not trainer.args.project: + ctx = getattr(trainer, "platform", None) + if not ctx or RANK not in {-1, 0} or not trainer.args.project: return project, name = _get_project_name(trainer) @@ -366,12 +375,12 @@ def on_fit_epoch_end(trainer): except Exception: pass - # Get system metrics (cache SystemLogger on trainer for efficiency) + # Get system metrics (cache SystemLogger in platform context for efficiency) system = {} try: - if not hasattr(trainer, "_platform_system_logger"): - trainer._platform_system_logger = SystemLogger() - system = trainer._platform_system_logger.get_metrics(rates=True) + if not ctx["system_logger"]: + ctx["system_logger"] = SystemLogger() + system = ctx["system_logger"].get_metrics(rates=True) except Exception: pass @@ -387,22 +396,23 @@ def on_fit_epoch_end(trainer): def _send_and_check_cancel(): """Send epoch_end and check response for cancellation (runs in background thread).""" - response = _send("epoch_end", payload, project, name, getattr(trainer, "_platform_model_id", None), retry=1) + response = _send("epoch_end", payload, project, name, ctx["model_id"], retry=1) if response and response.get("cancelled"): LOGGER.info(f"{PREFIX}Training cancelled from Platform ✅") trainer.stop = True - trainer._platform_cancelled = True + ctx["cancelled"] = True _executor.submit(_send_and_check_cancel) def on_model_save(trainer): """Upload model checkpoint (rate limited to every 15 min).""" - if RANK not in {-1, 0} or not trainer.args.project: + ctx = getattr(trainer, "platform", None) + if not ctx or RANK not in {-1, 0} or not trainer.args.project: return # Rate limit to every 15 minutes (900 seconds) - if time() - getattr(trainer, "_platform_last_upload", 0) < 900: + if time() - ctx["last_upload"] < 900: return model_path = trainer.best if trainer.best and Path(trainer.best).exists() else trainer.last @@ -410,31 +420,32 @@ def on_model_save(trainer): return project, name = _get_project_name(trainer) - _upload_model_async(model_path, project, name) - trainer._platform_last_upload = time() + _upload_model_async(model_path, project, name, model_id=ctx["model_id"]) + ctx["last_upload"] = time() def on_train_end(trainer): """Log final results, upload best model, and send validation plot data.""" - if RANK not in {-1, 0} or not trainer.args.project: + ctx = getattr(trainer, "platform", None) + if not ctx or RANK not in {-1, 0} or not trainer.args.project: return project, name = _get_project_name(trainer) - if getattr(trainer, "_platform_cancelled", False): + if ctx["cancelled"]: LOGGER.info(f"{PREFIX}Uploading partial results for cancelled training") # Stop console capture - if hasattr(trainer, "_platform_console_logger") and trainer._platform_console_logger: - trainer._platform_console_logger.stop_capture() - trainer._platform_console_logger = None + if ctx["console_logger"]: + ctx["console_logger"].stop_capture() + ctx["console_logger"] = None # Upload best model (blocking with progress bar to ensure it completes) gcs_path = None model_size = None if trainer.best and Path(trainer.best).exists(): model_size = Path(trainer.best).stat().st_size - gcs_path = _upload_model(trainer.best, project, name, progress=True, retry=3) + gcs_path = _upload_model(trainer.best, project, name, progress=True, retry=3, model_id=ctx["model_id"]) if not gcs_path: LOGGER.warning(f"{PREFIX}Model will not be available for download on Platform (upload failed)") @@ -467,10 +478,10 @@ def on_train_end(trainer): }, project, name, - getattr(trainer, "_platform_model_id", None), + ctx["model_id"], retry=4, # Critical, more retries ) - url = f"{PLATFORM_URL}/{project}/{name}" + url = f"{PLATFORM_URL}/{project}/{ctx.get('model_slug', name)}" LOGGER.info(f"{PREFIX}View results at {url}")