diff --git a/src/transport/quic/mod.rs b/src/transport/quic/mod.rs
index 90fe0ceee..d274a977c 100644
--- a/src/transport/quic/mod.rs
+++ b/src/transport/quic/mod.rs
@@ -305,7 +305,6 @@ impl Transport for QuicTransport {
     }
 
     fn reject(&mut self, connection_id: ConnectionId) -> crate::Result<()> {
-        self.canceled.insert(connection_id);
         self.pending_open
             .remove(&connection_id)
             .map_or(Err(Error::ConnectionDoesntExist(connection_id)), |_| Ok(()))
diff --git a/src/transport/tcp/mod.rs b/src/transport/tcp/mod.rs
index 14cbbf04e..35c8da597 100644
--- a/src/transport/tcp/mod.rs
+++ b/src/transport/tcp/mod.rs
@@ -373,7 +373,6 @@ impl Transport for TcpTransport {
     }
 
     fn reject(&mut self, connection_id: ConnectionId) -> crate::Result<()> {
-        self.canceled.insert(connection_id);
         self.pending_open
             .remove(&connection_id)
             .map_or(Err(Error::ConnectionDoesntExist(connection_id)), |_| Ok(()))
diff --git a/src/transport/websocket/mod.rs b/src/transport/websocket/mod.rs
index 617c2eef0..75360f31c 100644
--- a/src/transport/websocket/mod.rs
+++ b/src/transport/websocket/mod.rs
@@ -432,7 +432,6 @@ impl Transport for WebSocketTransport {
     }
 
     fn reject(&mut self, connection_id: ConnectionId) -> crate::Result<()> {
-        self.canceled.insert(connection_id);
         self.pending_open
             .remove(&connection_id)
             .map_or(Err(Error::ConnectionDoesntExist(connection_id)), |_| Ok(()))