ultralytics 8.3.184 New SystemLogger class (#21764)

Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
This commit is contained in:
Glenn Jocher 2025-08-22 13:49:55 +02:00 committed by GitHub
parent c4bd54b1af
commit 1e397c0dea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 243 additions and 18 deletions

View File

@ -13,4 +13,8 @@ keywords: ConsoleLogger, console capture, log streaming, API logging, file loggi
## ::: ultralytics.utils.logger.ConsoleLogger
<br><br><hr><br>
## ::: ultralytics.utils.logger.SystemLogger
<br><br>

View File

@ -1,6 +1,6 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
__version__ = "8.3.183"
__version__ = "8.3.184"
import os

View File

@ -1,15 +1,15 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
from ultralytics.utils import RANK, SETTINGS
from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger
from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger, SystemLogger
def on_pretrain_routine_start(trainer):
"""Initialize and start console logging immediately at the very beginning."""
if RANK in {-1, 0}:
# Create and start logger immediately before any other output
trainer.platform_logger = ConsoleLogger(DEFAULT_LOG_PATH)
trainer.platform_logger.start_capture()
trainer.system_logger = SystemLogger()
trainer.console_logger = ConsoleLogger(DEFAULT_LOG_PATH)
trainer.console_logger.start_capture()
def on_pretrain_routine_end(trainer):
@ -18,8 +18,10 @@ def on_pretrain_routine_end(trainer):
def on_fit_epoch_end(trainer):
"""Handle end of training epoch event."""
pass
"""Handle end of training epoch event and collect system metrics."""
if RANK in {-1, 0} and hasattr(trainer, "system_logger"):
system_metrics = trainer.system_logger.get_metrics()
print(system_metrics) # for debug
def on_model_save(trainer):
@ -29,7 +31,7 @@ def on_model_save(trainer):
def on_train_end(trainer):
"""Stop console capture and finalize logs."""
if logger := getattr(trainer, "platform_logger", None):
if logger := getattr(trainer, "console_logger", None):
logger.stop_capture()

View File

@ -2,15 +2,18 @@
import logging
import queue
import shutil
import sys
import threading
import time
from datetime import datetime
from pathlib import Path
from time import time
import psutil
import requests
from ultralytics.utils import RANK
from ultralytics.utils import MACOS, RANK
from ultralytics.utils.checks import check_requirements
# Initialize default log file
DEFAULT_LOG_PATH = Path("train.log")
@ -19,10 +22,46 @@ if RANK in {-1, 0} and DEFAULT_LOG_PATH.exists():
class ConsoleLogger:
"""Console output capture with API/file streaming and deduplication."""
"""
Console output capture with API/file streaming and deduplication.
Captures stdout/stderr output and streams it to either an API endpoint or local file, with intelligent
deduplication to reduce noise from repetitive console output.
Attributes:
destination (str | Path): Target destination for streaming (URL or Path object).
is_api (bool): Whether destination is an API endpoint (True) or local file (False).
original_stdout: Reference to original sys.stdout for restoration.
original_stderr: Reference to original sys.stderr for restoration.
log_queue (queue.Queue): Thread-safe queue for buffering log messages.
active (bool): Whether console capture is currently active.
worker_thread (threading.Thread): Background thread for processing log queue.
last_line (str): Last processed line for deduplication.
last_time (float): Timestamp of last processed line.
last_progress_line (str): Last progress bar line for progress deduplication.
last_was_progress (bool): Whether the last line was a progress bar.
Examples:
Basic file logging:
>>> logger = ConsoleLogger("training.log")
>>> logger.start_capture()
>>> print("This will be logged")
>>> logger.stop_capture()
API streaming:
>>> logger = ConsoleLogger("https://api.example.com/logs")
>>> logger.start_capture()
>>> # All output streams to API
>>> logger.stop_capture()
"""
def __init__(self, destination):
"""Initialize with API endpoint or local file path."""
"""
Initialize with API endpoint or local file path.
Args:
destination (str | Path): API endpoint URL (http/https) or local file path for streaming output.
"""
self.destination = destination
self.is_api = isinstance(destination, str) and destination.startswith(("http://", "https://"))
if not self.is_api:
@ -42,7 +81,7 @@ class ConsoleLogger:
self.last_was_progress = False # Track if last line was a progress bar
def start_capture(self):
"""Start capturing console output."""
"""Start capturing console output and redirect stdout/stderr to custom capture objects."""
if self.active:
return
@ -61,7 +100,7 @@ class ConsoleLogger:
self.worker_thread.start()
def stop_capture(self):
"""Stop capturing console output."""
"""Stop capturing console output and restore original stdout/stderr."""
if not self.active:
return
@ -71,11 +110,11 @@ class ConsoleLogger:
self.log_queue.put(None)
def _queue_log(self, text):
"""Queue console text with deduplication."""
"""Queue console text with deduplication and timestamp processing."""
if not self.active:
return
current_time = time()
current_time = time.time()
# Handle carriage returns and process lines
if "\r" in text:
@ -137,7 +176,7 @@ class ConsoleLogger:
return False
def _stream_worker(self):
"""Background worker for streaming logs."""
"""Background worker for streaming logs to destination."""
while self.active:
try:
log_text = self.log_queue.get(timeout=1)
@ -148,7 +187,7 @@ class ConsoleLogger:
continue
def _write_log(self, text):
"""Write log to destination."""
"""Write log to API endpoint or local file destination."""
try:
if self.is_api:
payload = {"timestamp": datetime.now().isoformat(), "message": text.strip()}
@ -187,3 +226,183 @@ class ConsoleLogger:
def emit(self, record):
self.callback(self.format(record) + "\n")
class SystemLogger:
"""
Log dynamic system metrics for training monitoring.
Captures real-time system metrics including CPU, RAM, disk I/O, network I/O, and NVIDIA GPU statistics for
training performance monitoring and analysis.
Attributes:
pynvml: NVIDIA pynvml module instance if successfully imported, None otherwise.
nvidia_initialized (bool): Whether NVIDIA GPU monitoring is available and initialized.
process (psutil.Process): Current psutil.Process instance for process-specific metrics.
net_start: Initial network I/O counters for calculating cumulative usage.
disk_start: Initial disk I/O counters for calculating cumulative usage.
Examples:
Basic usage:
>>> logger = SystemLogger()
>>> metrics = logger.get_metrics()
>>> print(f"CPU: {metrics['cpu']}%, RAM: {metrics['ram']}%")
>>> if metrics["gpus"]:
... gpu0 = metrics["gpus"]["0"]
... print(f"GPU0: {gpu0['usage']}% usage, {gpu0['temp']}°C")
Training loop integration:
>>> system_logger = SystemLogger()
>>> for epoch in range(epochs):
... # Training code here
... metrics = system_logger.get_metrics()
... # Log to database/file
"""
def __init__(self):
"""Initialize the system logger."""
self.pynvml = None
self.nvidia_initialized = self._init_nvidia()
self.process = psutil.Process()
self.net_start = psutil.net_io_counters()
self.disk_start = psutil.disk_io_counters()
def _init_nvidia(self):
"""Initialize NVIDIA GPU monitoring with pynvml."""
try:
assert not MACOS
check_requirements("pynvml>=12.0.0")
self.pynvml = __import__("pynvml")
self.pynvml.nvmlInit()
return True
except Exception:
return False
def get_metrics(self):
"""
Get current system metrics.
Collects comprehensive system metrics including CPU usage, RAM usage, disk I/O statistics,
network I/O statistics, and GPU metrics (if available). Example output:
```json
{
'cpu': 45.2,
'ram': 78.9,
'disk': {'read_mb': 156.7, 'write_mb': 89.3, 'used_gb': 256.8},
'network': {'recv_mb': 157.2, 'sent_mb': 89.1},
'gpus': {
0: {'usage': 95.6, 'memory': 85.4, 'temp': 72, 'power': 285},
1: {'usage': 94.1, 'memory': 82.7, 'temp': 70, 'power': 278}
}
}
```
- cpu (float): CPU usage percentage (0-100%)
- ram (float): RAM usage percentage (0-100%)
- disk (dict):
- read_mb (float): Cumulative disk read in MB since initialization
- write_mb (float): Cumulative disk write in MB since initialization
- used_gb (float): Total disk space used in GB
- network (dict):
- recv_mb (float): Cumulative network received in MB since initialization
- sent_mb (float): Cumulative network sent in MB since initialization
- gpus (dict): GPU metrics by device index (e.g., 0, 1) containing:
- usage (int): GPU utilization percentage (0-100%)
- memory (float): CUDA memory usage percentage (0-100%)
- temp (int): GPU temperature in degrees Celsius
- power (int): GPU power consumption in watts
Returns:
(dict): System metrics containing 'cpu', 'ram', 'disk', 'network', 'gpus' with respective usage data.
"""
net = psutil.net_io_counters()
disk = psutil.disk_io_counters()
memory = psutil.virtual_memory()
disk_usage = shutil.disk_usage("/")
metrics = {
"cpu": round(psutil.cpu_percent(), 3),
"ram": round(memory.percent, 3),
"disk": {
"read_mb": round((disk.read_bytes - self.disk_start.read_bytes) / (1 << 20), 3),
"write_mb": round((disk.write_bytes - self.disk_start.write_bytes) / (1 << 20), 3),
"used_gb": round(disk_usage.used / (1 << 30), 3),
},
"network": {
"recv_mb": round((net.bytes_recv - self.net_start.bytes_recv) / (1 << 20), 3),
"sent_mb": round((net.bytes_sent - self.net_start.bytes_sent) / (1 << 20), 3),
},
"gpus": {},
}
# Add GPU metrics (NVIDIA only)
if self.nvidia_initialized:
metrics["gpus"].update(self._get_nvidia_metrics())
return metrics
def _get_nvidia_metrics(self):
"""Get NVIDIA GPU metrics including utilization, memory, temperature, and power."""
gpus = {}
if not self.nvidia_initialized or not self.pynvml:
return gpus
try:
device_count = self.pynvml.nvmlDeviceGetCount()
for i in range(device_count):
handle = self.pynvml.nvmlDeviceGetHandleByIndex(i)
util = self.pynvml.nvmlDeviceGetUtilizationRates(handle)
memory = self.pynvml.nvmlDeviceGetMemoryInfo(handle)
temp = self.pynvml.nvmlDeviceGetTemperature(handle, self.pynvml.NVML_TEMPERATURE_GPU)
power = self.pynvml.nvmlDeviceGetPowerUsage(handle) // 1000
gpus[str(i)] = {
"usage": round(util.gpu, 3),
"memory": round((memory.used / memory.total) * 100, 3),
"temp": temp,
"power": power,
}
except Exception:
pass
return gpus
if __name__ == "__main__":
print("SystemLogger Real-time Metrics Monitor")
print("Press Ctrl+C to stop\n")
logger = SystemLogger()
try:
while True:
metrics = logger.get_metrics()
# Clear screen (works on most terminals)
print("\033[H\033[J", end="")
# Display system metrics
print(f"CPU: {metrics['cpu']:5.1f}%")
print(f"RAM: {metrics['ram']:5.1f}%")
print(f"Disk Read: {metrics['disk']['read_mb']:8.1f} MB")
print(f"Disk Write: {metrics['disk']['write_mb']:7.1f} MB")
print(f"Disk Used: {metrics['disk']['used_gb']:8.1f} GB")
print(f"Net Recv: {metrics['network']['recv_mb']:9.1f} MB")
print(f"Net Sent: {metrics['network']['sent_mb']:9.1f} MB")
# Display GPU metrics if available
if metrics["gpus"]:
print("\nGPU Metrics:")
for gpu_id, gpu_data in metrics["gpus"].items():
print(
f" GPU {gpu_id}: {gpu_data['usage']:3}% | "
f"Mem: {gpu_data['memory']:5.1f}% | "
f"Temp: {gpu_data['temp']:2}°C | "
f"Power: {gpu_data['power']:3}W"
)
else:
print("\nGPU: No NVIDIA GPUs detected")
time.sleep(1)
except KeyboardInterrupt:
print("\n\nStopped monitoring.")