diff --git a/ultralytics/engine/trainer.py b/ultralytics/engine/trainer.py index 3d8ccffe..9a4de451 100644 --- a/ultralytics/engine/trainer.py +++ b/ultralytics/engine/trainer.py @@ -341,7 +341,13 @@ class BaseTrainer: self.stride = gs # for multiscale training if self.world_size > 1: - self.model = nn.parallel.DistributedDataParallel(self.model, device_ids=[RANK], find_unused_parameters=True) + # static_graph=True permits params used >1 time per forward (e.g. flow_model in + # o2m+o2o pose loss branches) under torch.compile. + self.model = nn.parallel.DistributedDataParallel( + self.model, + device_ids=[RANK], + static_graph=bool(self.args.compile), + ) # Batch size if self.batch_size < 1 and RANK == -1: # single-GPU only, estimate best batch size diff --git a/ultralytics/nn/modules/head.py b/ultralytics/nn/modules/head.py index 54d5bbdf..b43e9fd2 100644 --- a/ultralytics/nn/modules/head.py +++ b/ultralytics/nn/modules/head.py @@ -1571,6 +1571,9 @@ class RTDETRDecoder(nn.Module): self.query_pos_head, attn_mask=attn_mask, ) + if self.training and dn_meta is None: + # Touch denoising_class_embed so DDP sees it as used when batch has zero GTs. + dec_bboxes = dec_bboxes + 0 * self.denoising_class_embed.weight.sum() x = dec_bboxes, dec_scores, enc_bboxes, enc_scores, dn_meta if self.training: return x