diff --git a/src/transformers/integrations.py b/src/transformers/integrations.py
index 63f8d04080fc..c5f4442c5635 100644
--- a/src/transformers/integrations.py
+++ b/src/transformers/integrations.py
@@ -282,7 +282,9 @@ def on_train_begin(self, args, state, control, **kwargs):
                 if hasattr(model, "config") and model.config is not None:
                     model_config_json = model.config.to_json_string()
                     self.tb_writer.add_text("model_config", model_config_json)
-            self.tb_writer.add_hparams(args.to_sanitized_dict(), metric_dict={})
+            # Version of TensorBoard coming from tensorboardX does not have this method.
+            if hasattr(self.tb_writer, "add_hparams"):
+                self.tb_writer.add_hparams(args.to_sanitized_dict(), metric_dict={})
 
     def on_log(self, args, state, control, logs=None, **kwargs):
         if state.is_world_process_zero: