empty_cache() only if GPU utilization above 90% (#18949)

Signed-off-by: Mohammed Yasin <32206511+Y-T-G@users.noreply.github.com>
Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: UltralyticsAssistant <web@ultralytics.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
Mohammed Yasin 2025-03-11 03:29:15 +08:00 committed by GitHub
parent 71eff4dee3
commit cd442d7e13
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -452,7 +452,8 @@ class BaseTrainer:
self.scheduler.last_epoch = self.epoch # do not move
self.stop |= epoch >= self.epochs # stop if exceeded epochs
self.run_callbacks("on_fit_epoch_end")
self._clear_memory()
if self._get_memory(fraction=True) > 0.9:
self._clear_memory() # clear if memory utilization > 90%
# Early Stopping
if RANK != -1: # if DDP training
@ -485,15 +486,20 @@ class BaseTrainer:
max_num_obj=max_num_obj,
) # returns batch size
def _get_memory(self):
"""Get accelerator memory utilization in GB."""
def _get_memory(self, fraction=False):
"""Get accelerator memory utilization in GB or fraction."""
memory, total = 0, 0
if self.device.type == "mps":
memory = torch.mps.driver_allocated_memory()
if fraction:
total = torch.mps.get_mem_info()[0]
elif self.device.type == "cpu":
memory = 0
pass
else:
memory = torch.cuda.memory_reserved()
return memory / (2**30)
if fraction:
total = torch.cuda.get_device_properties(self.device).total_memory
return ((memory / total) if total > 0 else 0) if fraction else (memory / 2**30)
def _clear_memory(self):
"""Clear accelerator memory on different platforms."""