empty_cache() only if GPU utilization above 90% (#18949)
Signed-off-by: Mohammed Yasin <32206511+Y-T-G@users.noreply.github.com> Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> Co-authored-by: UltralyticsAssistant <web@ultralytics.com> Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
parent
71eff4dee3
commit
cd442d7e13
@ -452,7 +452,8 @@ class BaseTrainer:
|
||||
self.scheduler.last_epoch = self.epoch # do not move
|
||||
self.stop |= epoch >= self.epochs # stop if exceeded epochs
|
||||
self.run_callbacks("on_fit_epoch_end")
|
||||
self._clear_memory()
|
||||
if self._get_memory(fraction=True) > 0.9:
|
||||
self._clear_memory() # clear if memory utilization > 90%
|
||||
|
||||
# Early Stopping
|
||||
if RANK != -1: # if DDP training
|
||||
@ -485,15 +486,20 @@ class BaseTrainer:
|
||||
max_num_obj=max_num_obj,
|
||||
) # returns batch size
|
||||
|
||||
def _get_memory(self):
|
||||
"""Get accelerator memory utilization in GB."""
|
||||
def _get_memory(self, fraction=False):
|
||||
"""Get accelerator memory utilization in GB or fraction."""
|
||||
memory, total = 0, 0
|
||||
if self.device.type == "mps":
|
||||
memory = torch.mps.driver_allocated_memory()
|
||||
if fraction:
|
||||
total = torch.mps.get_mem_info()[0]
|
||||
elif self.device.type == "cpu":
|
||||
memory = 0
|
||||
pass
|
||||
else:
|
||||
memory = torch.cuda.memory_reserved()
|
||||
return memory / (2**30)
|
||||
if fraction:
|
||||
total = torch.cuda.get_device_properties(self.device).total_memory
|
||||
return ((memory / total) if total > 0 else 0) if fraction else (memory / 2**30)
|
||||
|
||||
def _clear_memory(self):
|
||||
"""Clear accelerator memory on different platforms."""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user