diff --git a/examples/seq2seq/test_finetune_trainer.py b/examples/seq2seq/test_finetune_trainer.py
index 70cceae3c524..b8c0f4816ce1 100644
--- a/examples/seq2seq/test_finetune_trainer.py
+++ b/examples/seq2seq/test_finetune_trainer.py
@@ -4,7 +4,13 @@
 
 from transformers import BertTokenizer, EncoderDecoderModel
 from transformers.file_utils import is_datasets_available
-from transformers.testing_utils import TestCasePlus, execute_subprocess_async, get_gpu_count, slow
+from transformers.testing_utils import (
+    TestCasePlus,
+    execute_subprocess_async,
+    get_gpu_count,
+    require_torch_non_multi_gpu_but_fix_me,
+    slow,
+)
 from transformers.trainer_callback import TrainerState
 from transformers.trainer_utils import set_seed
 
@@ -46,6 +52,7 @@ def test_finetune_trainer_slow(self):
         assert "test_results.json" in contents
 
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_finetune_bert2bert(self):
         if not is_datasets_available():
             return