From 1e397c0dea02ac82c8bef1a76a12e6c30e9be57f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 22 Aug 2025 13:49:55 +0200 Subject: [PATCH] `ultralytics 8.3.184` New SystemLogger class (#21764) Signed-off-by: Glenn Jocher Co-authored-by: UltralyticsAssistant --- docs/en/reference/utils/logger.md | 4 + ultralytics/__init__.py | 2 +- ultralytics/utils/callbacks/platform.py | 16 +- ultralytics/utils/logger.py | 239 +++++++++++++++++++++++- 4 files changed, 243 insertions(+), 18 deletions(-) diff --git a/docs/en/reference/utils/logger.md b/docs/en/reference/utils/logger.md index 75b4fa32..68d02b59 100644 --- a/docs/en/reference/utils/logger.md +++ b/docs/en/reference/utils/logger.md @@ -13,4 +13,8 @@ keywords: ConsoleLogger, console capture, log streaming, API logging, file loggi ## ::: ultralytics.utils.logger.ConsoleLogger +



+ +## ::: ultralytics.utils.logger.SystemLogger +

diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index 9a4b9a8d..fc990dea 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license -__version__ = "8.3.183" +__version__ = "8.3.184" import os diff --git a/ultralytics/utils/callbacks/platform.py b/ultralytics/utils/callbacks/platform.py index 12ea0f66..e7ca1913 100644 --- a/ultralytics/utils/callbacks/platform.py +++ b/ultralytics/utils/callbacks/platform.py @@ -1,15 +1,15 @@ # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license from ultralytics.utils import RANK, SETTINGS -from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger +from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger, SystemLogger def on_pretrain_routine_start(trainer): """Initialize and start console logging immediately at the very beginning.""" if RANK in {-1, 0}: - # Create and start logger immediately before any other output - trainer.platform_logger = ConsoleLogger(DEFAULT_LOG_PATH) - trainer.platform_logger.start_capture() + trainer.system_logger = SystemLogger() + trainer.console_logger = ConsoleLogger(DEFAULT_LOG_PATH) + trainer.console_logger.start_capture() def on_pretrain_routine_end(trainer): @@ -18,8 +18,10 @@ def on_pretrain_routine_end(trainer): def on_fit_epoch_end(trainer): - """Handle end of training epoch event.""" - pass + """Handle end of training epoch event and collect system metrics.""" + if RANK in {-1, 0} and hasattr(trainer, "system_logger"): + system_metrics = trainer.system_logger.get_metrics() + print(system_metrics) # for debug def on_model_save(trainer): @@ -29,7 +31,7 @@ def on_model_save(trainer): def on_train_end(trainer): """Stop console capture and finalize logs.""" - if logger := getattr(trainer, "platform_logger", None): + if logger := getattr(trainer, "console_logger", None): logger.stop_capture() diff --git a/ultralytics/utils/logger.py b/ultralytics/utils/logger.py index 845305a1..764e8cb5 100644 --- a/ultralytics/utils/logger.py +++ b/ultralytics/utils/logger.py @@ -2,15 +2,18 @@ import logging import queue +import shutil import sys import threading +import time from datetime import datetime from pathlib import Path -from time import time +import psutil import requests -from ultralytics.utils import RANK +from ultralytics.utils import MACOS, RANK +from ultralytics.utils.checks import check_requirements # Initialize default log file DEFAULT_LOG_PATH = Path("train.log") @@ -19,10 +22,46 @@ if RANK in {-1, 0} and DEFAULT_LOG_PATH.exists(): class ConsoleLogger: - """Console output capture with API/file streaming and deduplication.""" + """ + Console output capture with API/file streaming and deduplication. + + Captures stdout/stderr output and streams it to either an API endpoint or local file, with intelligent + deduplication to reduce noise from repetitive console output. + + Attributes: + destination (str | Path): Target destination for streaming (URL or Path object). + is_api (bool): Whether destination is an API endpoint (True) or local file (False). + original_stdout: Reference to original sys.stdout for restoration. + original_stderr: Reference to original sys.stderr for restoration. + log_queue (queue.Queue): Thread-safe queue for buffering log messages. + active (bool): Whether console capture is currently active. + worker_thread (threading.Thread): Background thread for processing log queue. + last_line (str): Last processed line for deduplication. + last_time (float): Timestamp of last processed line. + last_progress_line (str): Last progress bar line for progress deduplication. + last_was_progress (bool): Whether the last line was a progress bar. + + Examples: + Basic file logging: + >>> logger = ConsoleLogger("training.log") + >>> logger.start_capture() + >>> print("This will be logged") + >>> logger.stop_capture() + + API streaming: + >>> logger = ConsoleLogger("https://api.example.com/logs") + >>> logger.start_capture() + >>> # All output streams to API + >>> logger.stop_capture() + """ def __init__(self, destination): - """Initialize with API endpoint or local file path.""" + """ + Initialize with API endpoint or local file path. + + Args: + destination (str | Path): API endpoint URL (http/https) or local file path for streaming output. + """ self.destination = destination self.is_api = isinstance(destination, str) and destination.startswith(("http://", "https://")) if not self.is_api: @@ -42,7 +81,7 @@ class ConsoleLogger: self.last_was_progress = False # Track if last line was a progress bar def start_capture(self): - """Start capturing console output.""" + """Start capturing console output and redirect stdout/stderr to custom capture objects.""" if self.active: return @@ -61,7 +100,7 @@ class ConsoleLogger: self.worker_thread.start() def stop_capture(self): - """Stop capturing console output.""" + """Stop capturing console output and restore original stdout/stderr.""" if not self.active: return @@ -71,11 +110,11 @@ class ConsoleLogger: self.log_queue.put(None) def _queue_log(self, text): - """Queue console text with deduplication.""" + """Queue console text with deduplication and timestamp processing.""" if not self.active: return - current_time = time() + current_time = time.time() # Handle carriage returns and process lines if "\r" in text: @@ -137,7 +176,7 @@ class ConsoleLogger: return False def _stream_worker(self): - """Background worker for streaming logs.""" + """Background worker for streaming logs to destination.""" while self.active: try: log_text = self.log_queue.get(timeout=1) @@ -148,7 +187,7 @@ class ConsoleLogger: continue def _write_log(self, text): - """Write log to destination.""" + """Write log to API endpoint or local file destination.""" try: if self.is_api: payload = {"timestamp": datetime.now().isoformat(), "message": text.strip()} @@ -187,3 +226,183 @@ class ConsoleLogger: def emit(self, record): self.callback(self.format(record) + "\n") + + +class SystemLogger: + """ + Log dynamic system metrics for training monitoring. + + Captures real-time system metrics including CPU, RAM, disk I/O, network I/O, and NVIDIA GPU statistics for + training performance monitoring and analysis. + + Attributes: + pynvml: NVIDIA pynvml module instance if successfully imported, None otherwise. + nvidia_initialized (bool): Whether NVIDIA GPU monitoring is available and initialized. + process (psutil.Process): Current psutil.Process instance for process-specific metrics. + net_start: Initial network I/O counters for calculating cumulative usage. + disk_start: Initial disk I/O counters for calculating cumulative usage. + + Examples: + Basic usage: + >>> logger = SystemLogger() + >>> metrics = logger.get_metrics() + >>> print(f"CPU: {metrics['cpu']}%, RAM: {metrics['ram']}%") + >>> if metrics["gpus"]: + ... gpu0 = metrics["gpus"]["0"] + ... print(f"GPU0: {gpu0['usage']}% usage, {gpu0['temp']}°C") + + Training loop integration: + >>> system_logger = SystemLogger() + >>> for epoch in range(epochs): + ... # Training code here + ... metrics = system_logger.get_metrics() + ... # Log to database/file + """ + + def __init__(self): + """Initialize the system logger.""" + self.pynvml = None + self.nvidia_initialized = self._init_nvidia() + self.process = psutil.Process() + self.net_start = psutil.net_io_counters() + self.disk_start = psutil.disk_io_counters() + + def _init_nvidia(self): + """Initialize NVIDIA GPU monitoring with pynvml.""" + try: + assert not MACOS + check_requirements("pynvml>=12.0.0") + self.pynvml = __import__("pynvml") + self.pynvml.nvmlInit() + return True + except Exception: + return False + + def get_metrics(self): + """ + Get current system metrics. + + Collects comprehensive system metrics including CPU usage, RAM usage, disk I/O statistics, + network I/O statistics, and GPU metrics (if available). Example output: + + ```json + { + 'cpu': 45.2, + 'ram': 78.9, + 'disk': {'read_mb': 156.7, 'write_mb': 89.3, 'used_gb': 256.8}, + 'network': {'recv_mb': 157.2, 'sent_mb': 89.1}, + 'gpus': { + 0: {'usage': 95.6, 'memory': 85.4, 'temp': 72, 'power': 285}, + 1: {'usage': 94.1, 'memory': 82.7, 'temp': 70, 'power': 278} + } + } + ``` + + - cpu (float): CPU usage percentage (0-100%) + - ram (float): RAM usage percentage (0-100%) + - disk (dict): + - read_mb (float): Cumulative disk read in MB since initialization + - write_mb (float): Cumulative disk write in MB since initialization + - used_gb (float): Total disk space used in GB + - network (dict): + - recv_mb (float): Cumulative network received in MB since initialization + - sent_mb (float): Cumulative network sent in MB since initialization + - gpus (dict): GPU metrics by device index (e.g., 0, 1) containing: + - usage (int): GPU utilization percentage (0-100%) + - memory (float): CUDA memory usage percentage (0-100%) + - temp (int): GPU temperature in degrees Celsius + - power (int): GPU power consumption in watts + + Returns: + (dict): System metrics containing 'cpu', 'ram', 'disk', 'network', 'gpus' with respective usage data. + """ + net = psutil.net_io_counters() + disk = psutil.disk_io_counters() + memory = psutil.virtual_memory() + disk_usage = shutil.disk_usage("/") + + metrics = { + "cpu": round(psutil.cpu_percent(), 3), + "ram": round(memory.percent, 3), + "disk": { + "read_mb": round((disk.read_bytes - self.disk_start.read_bytes) / (1 << 20), 3), + "write_mb": round((disk.write_bytes - self.disk_start.write_bytes) / (1 << 20), 3), + "used_gb": round(disk_usage.used / (1 << 30), 3), + }, + "network": { + "recv_mb": round((net.bytes_recv - self.net_start.bytes_recv) / (1 << 20), 3), + "sent_mb": round((net.bytes_sent - self.net_start.bytes_sent) / (1 << 20), 3), + }, + "gpus": {}, + } + + # Add GPU metrics (NVIDIA only) + if self.nvidia_initialized: + metrics["gpus"].update(self._get_nvidia_metrics()) + + return metrics + + def _get_nvidia_metrics(self): + """Get NVIDIA GPU metrics including utilization, memory, temperature, and power.""" + gpus = {} + if not self.nvidia_initialized or not self.pynvml: + return gpus + try: + device_count = self.pynvml.nvmlDeviceGetCount() + for i in range(device_count): + handle = self.pynvml.nvmlDeviceGetHandleByIndex(i) + util = self.pynvml.nvmlDeviceGetUtilizationRates(handle) + memory = self.pynvml.nvmlDeviceGetMemoryInfo(handle) + temp = self.pynvml.nvmlDeviceGetTemperature(handle, self.pynvml.NVML_TEMPERATURE_GPU) + power = self.pynvml.nvmlDeviceGetPowerUsage(handle) // 1000 + + gpus[str(i)] = { + "usage": round(util.gpu, 3), + "memory": round((memory.used / memory.total) * 100, 3), + "temp": temp, + "power": power, + } + except Exception: + pass + return gpus + + +if __name__ == "__main__": + print("SystemLogger Real-time Metrics Monitor") + print("Press Ctrl+C to stop\n") + + logger = SystemLogger() + + try: + while True: + metrics = logger.get_metrics() + + # Clear screen (works on most terminals) + print("\033[H\033[J", end="") + + # Display system metrics + print(f"CPU: {metrics['cpu']:5.1f}%") + print(f"RAM: {metrics['ram']:5.1f}%") + print(f"Disk Read: {metrics['disk']['read_mb']:8.1f} MB") + print(f"Disk Write: {metrics['disk']['write_mb']:7.1f} MB") + print(f"Disk Used: {metrics['disk']['used_gb']:8.1f} GB") + print(f"Net Recv: {metrics['network']['recv_mb']:9.1f} MB") + print(f"Net Sent: {metrics['network']['sent_mb']:9.1f} MB") + + # Display GPU metrics if available + if metrics["gpus"]: + print("\nGPU Metrics:") + for gpu_id, gpu_data in metrics["gpus"].items(): + print( + f" GPU {gpu_id}: {gpu_data['usage']:3}% | " + f"Mem: {gpu_data['memory']:5.1f}% | " + f"Temp: {gpu_data['temp']:2}°C | " + f"Power: {gpu_data['power']:3}W" + ) + else: + print("\nGPU: No NVIDIA GPUs detected") + + time.sleep(1) + + except KeyboardInterrupt: + print("\n\nStopped monitoring.")