Use static_graph=True for compile and remove find_unused_parameters (#24413)
Co-authored-by: UltralyticsAssistant <web@ultralytics.com> Co-authored-by: Jing Qiu <61612323+Laughing-q@users.noreply.github.com>
This commit is contained in:
parent
c219cb2d5a
commit
49c0961c75
@ -341,7 +341,13 @@ class BaseTrainer:
|
||||
self.stride = gs # for multiscale training
|
||||
|
||||
if self.world_size > 1:
|
||||
self.model = nn.parallel.DistributedDataParallel(self.model, device_ids=[RANK], find_unused_parameters=True)
|
||||
# static_graph=True permits params used >1 time per forward (e.g. flow_model in
|
||||
# o2m+o2o pose loss branches) under torch.compile.
|
||||
self.model = nn.parallel.DistributedDataParallel(
|
||||
self.model,
|
||||
device_ids=[RANK],
|
||||
static_graph=bool(self.args.compile),
|
||||
)
|
||||
|
||||
# Batch size
|
||||
if self.batch_size < 1 and RANK == -1: # single-GPU only, estimate best batch size
|
||||
|
||||
@ -1571,6 +1571,9 @@ class RTDETRDecoder(nn.Module):
|
||||
self.query_pos_head,
|
||||
attn_mask=attn_mask,
|
||||
)
|
||||
if self.training and dn_meta is None:
|
||||
# Touch denoising_class_embed so DDP sees it as used when batch has zero GTs.
|
||||
dec_bboxes = dec_bboxes + 0 * self.denoising_class_embed.weight.sum()
|
||||
x = dec_bboxes, dec_scores, enc_bboxes, enc_scores, dn_meta
|
||||
if self.training:
|
||||
return x
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user