diff --git a/examples/reinforcement_learning/actor_critic.py b/examples/reinforcement_learning/actor_critic.py
index 3b08706302e5..01cdbfe09f5f 100644
--- a/examples/reinforcement_learning/actor_critic.py
+++ b/examples/reinforcement_learning/actor_critic.py
@@ -146,7 +146,7 @@ def should_finish_training(engine):
     )
     args = parser.parse_args()
 
-    env = gym.make("CartPole-v0")
+    env = gym.make("CartPole-v1")
     env.seed(args.seed)
     torch.manual_seed(args.seed)
 
diff --git a/examples/reinforcement_learning/reinforce.py b/examples/reinforcement_learning/reinforce.py
index ee3aade2904f..dc9aafc6f946 100644
--- a/examples/reinforcement_learning/reinforce.py
+++ b/examples/reinforcement_learning/reinforce.py
@@ -136,7 +136,7 @@ def should_finish_training(engine):
     )
     args = parser.parse_args()
 
-    env = gym.make("CartPole-v0")
+    env = gym.make("CartPole-v1")
     env.seed(args.seed)
     torch.manual_seed(args.seed)
 
diff --git a/requirements-dev.txt b/requirements-dev.txt
index f772a09d419d..c7b53b6a743e 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -31,4 +31,6 @@ nltk
 # Examples dependencies
 pandas
 gym
+# pygame is now required by gym for cartpole env
+pygame
 mkl