ultralytics 8.3.184 New SystemLogger class (#21764)
Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
This commit is contained in:
parent
c4bd54b1af
commit
1e397c0dea
@ -13,4 +13,8 @@ keywords: ConsoleLogger, console capture, log streaming, API logging, file loggi
|
||||
|
||||
## ::: ultralytics.utils.logger.ConsoleLogger
|
||||
|
||||
<br><br><hr><br>
|
||||
|
||||
## ::: ultralytics.utils.logger.SystemLogger
|
||||
|
||||
<br><br>
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
__version__ = "8.3.183"
|
||||
__version__ = "8.3.184"
|
||||
|
||||
import os
|
||||
|
||||
|
||||
@ -1,15 +1,15 @@
|
||||
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
from ultralytics.utils import RANK, SETTINGS
|
||||
from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger
|
||||
from ultralytics.utils.logger import DEFAULT_LOG_PATH, ConsoleLogger, SystemLogger
|
||||
|
||||
|
||||
def on_pretrain_routine_start(trainer):
|
||||
"""Initialize and start console logging immediately at the very beginning."""
|
||||
if RANK in {-1, 0}:
|
||||
# Create and start logger immediately before any other output
|
||||
trainer.platform_logger = ConsoleLogger(DEFAULT_LOG_PATH)
|
||||
trainer.platform_logger.start_capture()
|
||||
trainer.system_logger = SystemLogger()
|
||||
trainer.console_logger = ConsoleLogger(DEFAULT_LOG_PATH)
|
||||
trainer.console_logger.start_capture()
|
||||
|
||||
|
||||
def on_pretrain_routine_end(trainer):
|
||||
@ -18,8 +18,10 @@ def on_pretrain_routine_end(trainer):
|
||||
|
||||
|
||||
def on_fit_epoch_end(trainer):
|
||||
"""Handle end of training epoch event."""
|
||||
pass
|
||||
"""Handle end of training epoch event and collect system metrics."""
|
||||
if RANK in {-1, 0} and hasattr(trainer, "system_logger"):
|
||||
system_metrics = trainer.system_logger.get_metrics()
|
||||
print(system_metrics) # for debug
|
||||
|
||||
|
||||
def on_model_save(trainer):
|
||||
@ -29,7 +31,7 @@ def on_model_save(trainer):
|
||||
|
||||
def on_train_end(trainer):
|
||||
"""Stop console capture and finalize logs."""
|
||||
if logger := getattr(trainer, "platform_logger", None):
|
||||
if logger := getattr(trainer, "console_logger", None):
|
||||
logger.stop_capture()
|
||||
|
||||
|
||||
|
||||
@ -2,15 +2,18 @@
|
||||
|
||||
import logging
|
||||
import queue
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from time import time
|
||||
|
||||
import psutil
|
||||
import requests
|
||||
|
||||
from ultralytics.utils import RANK
|
||||
from ultralytics.utils import MACOS, RANK
|
||||
from ultralytics.utils.checks import check_requirements
|
||||
|
||||
# Initialize default log file
|
||||
DEFAULT_LOG_PATH = Path("train.log")
|
||||
@ -19,10 +22,46 @@ if RANK in {-1, 0} and DEFAULT_LOG_PATH.exists():
|
||||
|
||||
|
||||
class ConsoleLogger:
|
||||
"""Console output capture with API/file streaming and deduplication."""
|
||||
"""
|
||||
Console output capture with API/file streaming and deduplication.
|
||||
|
||||
Captures stdout/stderr output and streams it to either an API endpoint or local file, with intelligent
|
||||
deduplication to reduce noise from repetitive console output.
|
||||
|
||||
Attributes:
|
||||
destination (str | Path): Target destination for streaming (URL or Path object).
|
||||
is_api (bool): Whether destination is an API endpoint (True) or local file (False).
|
||||
original_stdout: Reference to original sys.stdout for restoration.
|
||||
original_stderr: Reference to original sys.stderr for restoration.
|
||||
log_queue (queue.Queue): Thread-safe queue for buffering log messages.
|
||||
active (bool): Whether console capture is currently active.
|
||||
worker_thread (threading.Thread): Background thread for processing log queue.
|
||||
last_line (str): Last processed line for deduplication.
|
||||
last_time (float): Timestamp of last processed line.
|
||||
last_progress_line (str): Last progress bar line for progress deduplication.
|
||||
last_was_progress (bool): Whether the last line was a progress bar.
|
||||
|
||||
Examples:
|
||||
Basic file logging:
|
||||
>>> logger = ConsoleLogger("training.log")
|
||||
>>> logger.start_capture()
|
||||
>>> print("This will be logged")
|
||||
>>> logger.stop_capture()
|
||||
|
||||
API streaming:
|
||||
>>> logger = ConsoleLogger("https://api.example.com/logs")
|
||||
>>> logger.start_capture()
|
||||
>>> # All output streams to API
|
||||
>>> logger.stop_capture()
|
||||
"""
|
||||
|
||||
def __init__(self, destination):
|
||||
"""Initialize with API endpoint or local file path."""
|
||||
"""
|
||||
Initialize with API endpoint or local file path.
|
||||
|
||||
Args:
|
||||
destination (str | Path): API endpoint URL (http/https) or local file path for streaming output.
|
||||
"""
|
||||
self.destination = destination
|
||||
self.is_api = isinstance(destination, str) and destination.startswith(("http://", "https://"))
|
||||
if not self.is_api:
|
||||
@ -42,7 +81,7 @@ class ConsoleLogger:
|
||||
self.last_was_progress = False # Track if last line was a progress bar
|
||||
|
||||
def start_capture(self):
|
||||
"""Start capturing console output."""
|
||||
"""Start capturing console output and redirect stdout/stderr to custom capture objects."""
|
||||
if self.active:
|
||||
return
|
||||
|
||||
@ -61,7 +100,7 @@ class ConsoleLogger:
|
||||
self.worker_thread.start()
|
||||
|
||||
def stop_capture(self):
|
||||
"""Stop capturing console output."""
|
||||
"""Stop capturing console output and restore original stdout/stderr."""
|
||||
if not self.active:
|
||||
return
|
||||
|
||||
@ -71,11 +110,11 @@ class ConsoleLogger:
|
||||
self.log_queue.put(None)
|
||||
|
||||
def _queue_log(self, text):
|
||||
"""Queue console text with deduplication."""
|
||||
"""Queue console text with deduplication and timestamp processing."""
|
||||
if not self.active:
|
||||
return
|
||||
|
||||
current_time = time()
|
||||
current_time = time.time()
|
||||
|
||||
# Handle carriage returns and process lines
|
||||
if "\r" in text:
|
||||
@ -137,7 +176,7 @@ class ConsoleLogger:
|
||||
return False
|
||||
|
||||
def _stream_worker(self):
|
||||
"""Background worker for streaming logs."""
|
||||
"""Background worker for streaming logs to destination."""
|
||||
while self.active:
|
||||
try:
|
||||
log_text = self.log_queue.get(timeout=1)
|
||||
@ -148,7 +187,7 @@ class ConsoleLogger:
|
||||
continue
|
||||
|
||||
def _write_log(self, text):
|
||||
"""Write log to destination."""
|
||||
"""Write log to API endpoint or local file destination."""
|
||||
try:
|
||||
if self.is_api:
|
||||
payload = {"timestamp": datetime.now().isoformat(), "message": text.strip()}
|
||||
@ -187,3 +226,183 @@ class ConsoleLogger:
|
||||
|
||||
def emit(self, record):
|
||||
self.callback(self.format(record) + "\n")
|
||||
|
||||
|
||||
class SystemLogger:
|
||||
"""
|
||||
Log dynamic system metrics for training monitoring.
|
||||
|
||||
Captures real-time system metrics including CPU, RAM, disk I/O, network I/O, and NVIDIA GPU statistics for
|
||||
training performance monitoring and analysis.
|
||||
|
||||
Attributes:
|
||||
pynvml: NVIDIA pynvml module instance if successfully imported, None otherwise.
|
||||
nvidia_initialized (bool): Whether NVIDIA GPU monitoring is available and initialized.
|
||||
process (psutil.Process): Current psutil.Process instance for process-specific metrics.
|
||||
net_start: Initial network I/O counters for calculating cumulative usage.
|
||||
disk_start: Initial disk I/O counters for calculating cumulative usage.
|
||||
|
||||
Examples:
|
||||
Basic usage:
|
||||
>>> logger = SystemLogger()
|
||||
>>> metrics = logger.get_metrics()
|
||||
>>> print(f"CPU: {metrics['cpu']}%, RAM: {metrics['ram']}%")
|
||||
>>> if metrics["gpus"]:
|
||||
... gpu0 = metrics["gpus"]["0"]
|
||||
... print(f"GPU0: {gpu0['usage']}% usage, {gpu0['temp']}°C")
|
||||
|
||||
Training loop integration:
|
||||
>>> system_logger = SystemLogger()
|
||||
>>> for epoch in range(epochs):
|
||||
... # Training code here
|
||||
... metrics = system_logger.get_metrics()
|
||||
... # Log to database/file
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the system logger."""
|
||||
self.pynvml = None
|
||||
self.nvidia_initialized = self._init_nvidia()
|
||||
self.process = psutil.Process()
|
||||
self.net_start = psutil.net_io_counters()
|
||||
self.disk_start = psutil.disk_io_counters()
|
||||
|
||||
def _init_nvidia(self):
|
||||
"""Initialize NVIDIA GPU monitoring with pynvml."""
|
||||
try:
|
||||
assert not MACOS
|
||||
check_requirements("pynvml>=12.0.0")
|
||||
self.pynvml = __import__("pynvml")
|
||||
self.pynvml.nvmlInit()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_metrics(self):
|
||||
"""
|
||||
Get current system metrics.
|
||||
|
||||
Collects comprehensive system metrics including CPU usage, RAM usage, disk I/O statistics,
|
||||
network I/O statistics, and GPU metrics (if available). Example output:
|
||||
|
||||
```json
|
||||
{
|
||||
'cpu': 45.2,
|
||||
'ram': 78.9,
|
||||
'disk': {'read_mb': 156.7, 'write_mb': 89.3, 'used_gb': 256.8},
|
||||
'network': {'recv_mb': 157.2, 'sent_mb': 89.1},
|
||||
'gpus': {
|
||||
0: {'usage': 95.6, 'memory': 85.4, 'temp': 72, 'power': 285},
|
||||
1: {'usage': 94.1, 'memory': 82.7, 'temp': 70, 'power': 278}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- cpu (float): CPU usage percentage (0-100%)
|
||||
- ram (float): RAM usage percentage (0-100%)
|
||||
- disk (dict):
|
||||
- read_mb (float): Cumulative disk read in MB since initialization
|
||||
- write_mb (float): Cumulative disk write in MB since initialization
|
||||
- used_gb (float): Total disk space used in GB
|
||||
- network (dict):
|
||||
- recv_mb (float): Cumulative network received in MB since initialization
|
||||
- sent_mb (float): Cumulative network sent in MB since initialization
|
||||
- gpus (dict): GPU metrics by device index (e.g., 0, 1) containing:
|
||||
- usage (int): GPU utilization percentage (0-100%)
|
||||
- memory (float): CUDA memory usage percentage (0-100%)
|
||||
- temp (int): GPU temperature in degrees Celsius
|
||||
- power (int): GPU power consumption in watts
|
||||
|
||||
Returns:
|
||||
(dict): System metrics containing 'cpu', 'ram', 'disk', 'network', 'gpus' with respective usage data.
|
||||
"""
|
||||
net = psutil.net_io_counters()
|
||||
disk = psutil.disk_io_counters()
|
||||
memory = psutil.virtual_memory()
|
||||
disk_usage = shutil.disk_usage("/")
|
||||
|
||||
metrics = {
|
||||
"cpu": round(psutil.cpu_percent(), 3),
|
||||
"ram": round(memory.percent, 3),
|
||||
"disk": {
|
||||
"read_mb": round((disk.read_bytes - self.disk_start.read_bytes) / (1 << 20), 3),
|
||||
"write_mb": round((disk.write_bytes - self.disk_start.write_bytes) / (1 << 20), 3),
|
||||
"used_gb": round(disk_usage.used / (1 << 30), 3),
|
||||
},
|
||||
"network": {
|
||||
"recv_mb": round((net.bytes_recv - self.net_start.bytes_recv) / (1 << 20), 3),
|
||||
"sent_mb": round((net.bytes_sent - self.net_start.bytes_sent) / (1 << 20), 3),
|
||||
},
|
||||
"gpus": {},
|
||||
}
|
||||
|
||||
# Add GPU metrics (NVIDIA only)
|
||||
if self.nvidia_initialized:
|
||||
metrics["gpus"].update(self._get_nvidia_metrics())
|
||||
|
||||
return metrics
|
||||
|
||||
def _get_nvidia_metrics(self):
|
||||
"""Get NVIDIA GPU metrics including utilization, memory, temperature, and power."""
|
||||
gpus = {}
|
||||
if not self.nvidia_initialized or not self.pynvml:
|
||||
return gpus
|
||||
try:
|
||||
device_count = self.pynvml.nvmlDeviceGetCount()
|
||||
for i in range(device_count):
|
||||
handle = self.pynvml.nvmlDeviceGetHandleByIndex(i)
|
||||
util = self.pynvml.nvmlDeviceGetUtilizationRates(handle)
|
||||
memory = self.pynvml.nvmlDeviceGetMemoryInfo(handle)
|
||||
temp = self.pynvml.nvmlDeviceGetTemperature(handle, self.pynvml.NVML_TEMPERATURE_GPU)
|
||||
power = self.pynvml.nvmlDeviceGetPowerUsage(handle) // 1000
|
||||
|
||||
gpus[str(i)] = {
|
||||
"usage": round(util.gpu, 3),
|
||||
"memory": round((memory.used / memory.total) * 100, 3),
|
||||
"temp": temp,
|
||||
"power": power,
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
return gpus
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("SystemLogger Real-time Metrics Monitor")
|
||||
print("Press Ctrl+C to stop\n")
|
||||
|
||||
logger = SystemLogger()
|
||||
|
||||
try:
|
||||
while True:
|
||||
metrics = logger.get_metrics()
|
||||
|
||||
# Clear screen (works on most terminals)
|
||||
print("\033[H\033[J", end="")
|
||||
|
||||
# Display system metrics
|
||||
print(f"CPU: {metrics['cpu']:5.1f}%")
|
||||
print(f"RAM: {metrics['ram']:5.1f}%")
|
||||
print(f"Disk Read: {metrics['disk']['read_mb']:8.1f} MB")
|
||||
print(f"Disk Write: {metrics['disk']['write_mb']:7.1f} MB")
|
||||
print(f"Disk Used: {metrics['disk']['used_gb']:8.1f} GB")
|
||||
print(f"Net Recv: {metrics['network']['recv_mb']:9.1f} MB")
|
||||
print(f"Net Sent: {metrics['network']['sent_mb']:9.1f} MB")
|
||||
|
||||
# Display GPU metrics if available
|
||||
if metrics["gpus"]:
|
||||
print("\nGPU Metrics:")
|
||||
for gpu_id, gpu_data in metrics["gpus"].items():
|
||||
print(
|
||||
f" GPU {gpu_id}: {gpu_data['usage']:3}% | "
|
||||
f"Mem: {gpu_data['memory']:5.1f}% | "
|
||||
f"Temp: {gpu_data['temp']:2}°C | "
|
||||
f"Power: {gpu_data['power']:3}W"
|
||||
)
|
||||
else:
|
||||
print("\nGPU: No NVIDIA GPUs detected")
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nStopped monitoring.")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user