diff --git a/ultralytics/engine/trainer.py b/ultralytics/engine/trainer.py index 5b6b64e2..f2a4903a 100644 --- a/ultralytics/engine/trainer.py +++ b/ultralytics/engine/trainer.py @@ -452,7 +452,8 @@ class BaseTrainer: self.scheduler.last_epoch = self.epoch # do not move self.stop |= epoch >= self.epochs # stop if exceeded epochs self.run_callbacks("on_fit_epoch_end") - self._clear_memory() + if self._get_memory(fraction=True) > 0.9: + self._clear_memory() # clear if memory utilization > 90% # Early Stopping if RANK != -1: # if DDP training @@ -485,15 +486,20 @@ class BaseTrainer: max_num_obj=max_num_obj, ) # returns batch size - def _get_memory(self): - """Get accelerator memory utilization in GB.""" + def _get_memory(self, fraction=False): + """Get accelerator memory utilization in GB or fraction.""" + memory, total = 0, 0 if self.device.type == "mps": memory = torch.mps.driver_allocated_memory() + if fraction: + total = torch.mps.get_mem_info()[0] elif self.device.type == "cpu": - memory = 0 + pass else: memory = torch.cuda.memory_reserved() - return memory / (2**30) + if fraction: + total = torch.cuda.get_device_properties(self.device).total_memory + return ((memory / total) if total > 0 else 0) if fraction else (memory / 2**30) def _clear_memory(self): """Clear accelerator memory on different platforms."""