Fix plots logging (#23105)
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
This commit is contained in:
parent
433ea7f7c1
commit
932c069b02
@ -11,6 +11,10 @@ keywords: platform callbacks, training callbacks, console logging, YOLO11 traini
|
||||
|
||||
<br>
|
||||
|
||||
## ::: ultralytics.utils.callbacks.platform._interp_plot
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.callbacks.platform._send
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
@ -364,7 +364,10 @@ class BaseValidator:
|
||||
return []
|
||||
|
||||
def on_plot(self, name, data=None):
|
||||
"""Register plots for visualization."""
|
||||
"""Register plots for visualization, deduplicating by type."""
|
||||
plot_type = data.get("type") if data else None
|
||||
if plot_type and any((v.get("data") or {}).get("type") == plot_type for v in self.plots.values()):
|
||||
return # Skip duplicate plot types
|
||||
self.plots[Path(name)] = {"data": data, "timestamp": time.time()}
|
||||
|
||||
def plot_val_samples(self, batch, ni):
|
||||
|
||||
@ -8,8 +8,9 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from time import time
|
||||
|
||||
from ultralytics.utils import ENVIRONMENT, GIT, LOGGER, PYTHON_VERSION, RANK, SETTINGS, TESTS_RUNNING
|
||||
from ultralytics.utils import ENVIRONMENT, GIT, LOGGER, PYTHON_VERSION, RANK, SETTINGS, TESTS_RUNNING, colorstr
|
||||
|
||||
PREFIX = colorstr("Platform: ")
|
||||
_last_upload = 0 # Rate limit model uploads
|
||||
_console_logger = None # Global console logger instance
|
||||
_system_logger = None # Cached system logger instance
|
||||
@ -31,6 +32,34 @@ except (AssertionError, ImportError):
|
||||
_api_key = None
|
||||
|
||||
|
||||
def _interp_plot(plot, n=101):
|
||||
"""Interpolate plot curve data from 1000 to n points to reduce storage size."""
|
||||
import numpy as np
|
||||
|
||||
if not plot.get("x") or not plot.get("y"):
|
||||
return plot # No interpolation needed (e.g., confusion_matrix)
|
||||
|
||||
x, y = np.array(plot["x"]), np.array(plot["y"])
|
||||
if len(x) <= n:
|
||||
return plot # Already small enough
|
||||
|
||||
# New x values (101 points gives clean 0.01 increments: 0, 0.01, 0.02, ..., 1.0)
|
||||
x_new = np.linspace(x[0], x[-1], n)
|
||||
|
||||
# Interpolate y values (handle both 1D and 2D arrays)
|
||||
if y.ndim == 1:
|
||||
y_new = np.interp(x_new, x, y)
|
||||
else:
|
||||
y_new = np.array([np.interp(x_new, x, yi) for yi in y])
|
||||
|
||||
# Also interpolate ap if present (for PR curves)
|
||||
result = {**plot, "x": x_new.tolist(), "y": y_new.tolist()}
|
||||
if "ap" in plot:
|
||||
result["ap"] = plot["ap"] # Keep AP values as-is (per-class scalars)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _send(event, data, project, name):
|
||||
"""Send event to Platform endpoint."""
|
||||
try:
|
||||
@ -75,7 +104,8 @@ def _upload_model(model_path, project, name):
|
||||
timeout=600, # 10 min timeout for large models
|
||||
).raise_for_status()
|
||||
|
||||
LOGGER.info(f"Platform: Model uploaded to '{project}'")
|
||||
# url = f"https://alpha.ultralytics.com/{project}/{name}"
|
||||
# LOGGER.info(f"{PREFIX}Model uploaded to {url}")
|
||||
return data.get("gcsPath")
|
||||
|
||||
except Exception as e:
|
||||
@ -150,7 +180,8 @@ def on_pretrain_routine_start(trainer):
|
||||
_last_upload = time()
|
||||
|
||||
project, name = str(trainer.args.project), str(trainer.args.name or "train")
|
||||
LOGGER.info(f"Platform: Streaming to project '{project}' as '{name}'")
|
||||
url = f"https://alpha.ultralytics.com/{project}/{name}"
|
||||
LOGGER.info(f"{PREFIX}Streaming to {url}")
|
||||
|
||||
# Create callback to send console output to Platform
|
||||
def send_console_output(content, line_count, chunk_id):
|
||||
@ -272,11 +303,15 @@ def on_train_end(trainer):
|
||||
model_size = Path(trainer.best).stat().st_size
|
||||
model_path = _upload_model(trainer.best, project, name)
|
||||
|
||||
# Collect plots from trainer and validator
|
||||
plots = [info["data"] for info in getattr(trainer, "plots", {}).values() if info.get("data")]
|
||||
plots += [
|
||||
info["data"] for info in getattr(getattr(trainer, "validator", None), "plots", {}).values() if info.get("data")
|
||||
]
|
||||
# Collect plots from trainer and validator, deduplicating by type
|
||||
plots_by_type = {}
|
||||
for info in getattr(trainer, "plots", {}).values():
|
||||
if info.get("data") and info["data"].get("type"):
|
||||
plots_by_type[info["data"]["type"]] = info["data"]
|
||||
for info in getattr(getattr(trainer, "validator", None), "plots", {}).values():
|
||||
if info.get("data") and info["data"].get("type"):
|
||||
plots_by_type.setdefault(info["data"]["type"], info["data"]) # Don't overwrite trainer plots
|
||||
plots = [_interp_plot(p) for p in plots_by_type.values()] # Interpolate curves to reduce size
|
||||
|
||||
# Get class names
|
||||
names = getattr(getattr(trainer, "validator", None), "names", None) or (trainer.data or {}).get("names")
|
||||
@ -298,7 +333,8 @@ def on_train_end(trainer):
|
||||
project,
|
||||
name,
|
||||
)
|
||||
LOGGER.info(f"Platform: Training complete, results uploaded to '{project}' ({len(plots)} plots)")
|
||||
url = f"https://alpha.ultralytics.com/{project}/{name}"
|
||||
LOGGER.info(f"{PREFIX}View results at {url}")
|
||||
|
||||
|
||||
callbacks = (
|
||||
|
||||
@ -330,14 +330,19 @@ class SystemLogger:
|
||||
|
||||
def _init_nvidia(self):
|
||||
"""Initialize NVIDIA GPU monitoring with pynvml."""
|
||||
if MACOS:
|
||||
return False
|
||||
|
||||
try:
|
||||
assert not MACOS
|
||||
check_requirements("nvidia-ml-py>=12.0.0")
|
||||
self.pynvml = __import__("pynvml")
|
||||
self.pynvml.nvmlInit()
|
||||
return True
|
||||
except Exception as e:
|
||||
LOGGER.warning(f"SystemLogger NVML init failed: {e}")
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
LOGGER.warning(f"SystemLogger NVML init failed: {e}")
|
||||
return False
|
||||
|
||||
def get_metrics(self, rates=False):
|
||||
|
||||
@ -568,7 +568,6 @@ class ConfusionMatrix(DataExportMixin):
|
||||
fig.savefig(plot_fname, dpi=250)
|
||||
plt.close(fig)
|
||||
if on_plot:
|
||||
# Pass confusion matrix data for interactive plotting (raw counts only, normalization done on frontend)
|
||||
on_plot(plot_fname, {"type": "confusion_matrix", "matrix": self.matrix.tolist()})
|
||||
|
||||
def print(self):
|
||||
@ -663,7 +662,8 @@ def plot_pr_curve(
|
||||
plt.close(fig)
|
||||
if on_plot:
|
||||
# Pass PR curve data for interactive plotting (class names stored at model level)
|
||||
on_plot(save_dir, {"type": "pr_curve", "x": px.tolist(), "y": py.tolist(), "ap": ap.tolist()})
|
||||
# Transpose py to match other curves: y[class][point] format
|
||||
on_plot(save_dir, {"type": "pr_curve", "x": px.tolist(), "y": py.T.tolist(), "ap": ap.tolist()})
|
||||
|
||||
|
||||
@plt_settings()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user