Update conv_asr.py preventing unnessory calculations (#15239)

Signed-off-by: tamilselvan0X0 <[email protected]>
Co-authored-by: Jason <[email protected]>

Author: tamilselvan0x0

nemo/collections/asr/modules/conv_asr.py

@@ -878,11 +878,11 @@ def affine_layer(
     @typecheck()
     def forward(self, encoder_output, length=None):
         pool = self._pooling(encoder_output, length)
-        embs = []
 
         for layer in self.emb_layers:
-            pool, emb = layer(pool), layer[: self.emb_id](pool)
-            embs.append(emb)
+            last_pool = pool
+            pool = layer(pool)
+        emb = layer[: self.emb_id](last_pool)
 
         pool = pool.squeeze(-1)
         if self.angular:
@@ -892,7 +892,7 @@ def forward(self, encoder_output, length=None):
 
         out = self.final(pool)
 
-        return out, embs[-1].squeeze(-1)
+        return out, emb.squeeze(-1)
 
 
 class ConvASREncoderAdapter(ConvASREncoder, adapter_mixins.AdapterModuleMixin):