diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index 3bd9314e92df..98884b189caa 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -896,8 +896,10 @@ def execute_subprocess_async(cmd, env=None, stdin=None, timeout=180, quiet=False
 
     cmd_str = " ".join(cmd)
     if result.returncode > 0:
+        stderr = "\n".join(result.stderr)
         raise RuntimeError(
-            f"'{cmd_str}' failed with returncode {result.returncode} - see the `stderr:` messages from above for details."
+            f"'{cmd_str}' failed with returncode {result.returncode}\n\n"
+            f"The combined stderr from workers follows:\n{stderr}"
         )
 
     # check that the subprocess actually did run and produced some output, should the test rely on