From cd442d7e131af8b09a3e36b685fa8cc79eb5bc71 Mon Sep 17 00:00:00 2001 From: Mohammed Yasin <32206511+Y-T-G@users.noreply.github.com> Date: Tue, 11 Mar 2025 03:29:15 +0800 Subject: [PATCH] `empty_cache()` only if GPU utilization above 90% (#18949) Signed-off-by: Mohammed Yasin <32206511+Y-T-G@users.noreply.github.com> Signed-off-by: Glenn Jocher Co-authored-by: UltralyticsAssistant Co-authored-by: Glenn Jocher --- ultralytics/engine/trainer.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ultralytics/engine/trainer.py b/ultralytics/engine/trainer.py index 5b6b64e2..f2a4903a 100644 --- a/ultralytics/engine/trainer.py +++ b/ultralytics/engine/trainer.py @@ -452,7 +452,8 @@ class BaseTrainer: self.scheduler.last_epoch = self.epoch # do not move self.stop |= epoch >= self.epochs # stop if exceeded epochs self.run_callbacks("on_fit_epoch_end") - self._clear_memory() + if self._get_memory(fraction=True) > 0.9: + self._clear_memory() # clear if memory utilization > 90% # Early Stopping if RANK != -1: # if DDP training @@ -485,15 +486,20 @@ class BaseTrainer: max_num_obj=max_num_obj, ) # returns batch size - def _get_memory(self): - """Get accelerator memory utilization in GB.""" + def _get_memory(self, fraction=False): + """Get accelerator memory utilization in GB or fraction.""" + memory, total = 0, 0 if self.device.type == "mps": memory = torch.mps.driver_allocated_memory() + if fraction: + total = torch.mps.get_mem_info()[0] elif self.device.type == "cpu": - memory = 0 + pass else: memory = torch.cuda.memory_reserved() - return memory / (2**30) + if fraction: + total = torch.cuda.get_device_properties(self.device).total_memory + return ((memory / total) if total > 0 else 0) if fraction else (memory / 2**30) def _clear_memory(self): """Clear accelerator memory on different platforms."""