Skip to content

kad: Fix substream opening and dialing race #222

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lexnv merged 5 commits into from
Aug 30, 2024
Merged

kad: Fix substream opening and dialing race #222

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
572cc98
kad: Make query actions robust to connection races
lexnv Aug 27, 2024
35ee1f1
find-node: Better logs on query finish
lexnv Aug 27, 2024
c4dc1ce
kad: Refactor fault tolerant substream and dial into a dedicated fn
lexnv Aug 27, 2024
1fbd48d
kad: Move Error::AlreadyConnected handling into the match statement
lexnv Aug 30, 2024
55f555e
Merge remote-tracking branch 'origin/master' into lexnv/kad-substream…
lexnv Aug 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 72 additions & 69 deletions src/protocol/libp2p/kademlia/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,50 +642,71 @@ impl Kademlia {
}
}

/// Handle next query action.
async fn on_query_action(&mut self, action: QueryAction) -> Result<(), (QueryId, PeerId)> {
match action {
QueryAction::SendMessage { query, peer, .. } => match self.service.open_substream(peer)
{
Err(_) => {
tracing::trace!(target: LOG_TARGET, ?query, ?peer, "dial peer");

match self.service.dial(&peer) {
Ok(_) => match self.pending_dials.entry(peer) {
Entry::Occupied(entry) => {
entry.into_mut().push(PeerAction::SendFindNode(query));
}
Entry::Vacant(entry) => {
entry.insert(vec![PeerAction::SendFindNode(query)]);
}
},
Err(error) => {
tracing::trace!(target: LOG_TARGET, ?query, ?peer, ?error, "failed to dial peer");
self.engine.register_response_failure(query, peer);
}
/// Open a substream with a peer or dial the peer.
fn open_substream_or_dial(
&mut self,
peer: PeerId,
action: PeerAction,
query: Option<QueryId>,
) -> Result<(), Error> {
match self.service.open_substream(peer) {
Ok(substream_id) => {
self.pending_substreams.insert(substream_id, peer);
self.peers.entry(peer).or_default().pending_actions.insert(substream_id, action);

return Ok(());
}
Err(err) => {
tracing::trace!(target: LOG_TARGET, ?query, ?peer, ?err, "Failed to open substream. Dialing peer");

match self.service.dial(&peer) {
Ok(()) => {
self.pending_dials.entry(peer).or_default().push(action);
return Ok(());
}

Ok(())
}
Ok(substream_id) => {
tracing::trace!(
target: LOG_TARGET,
?query,
?peer,
?substream_id,
"open outbound substream for peer"
);
// Already connected is a recoverable error.
Err(Error::AlreadyConnected) => (),

self.pending_substreams.insert(substream_id, peer);
self.peers
.entry(peer)
.or_default()
.pending_actions
.insert(substream_id, PeerAction::SendFindNode(query));
Err(error) => {
tracing::trace!(target: LOG_TARGET, ?query, ?peer, ?error, "Failed to dial peer");
return Err(error);
}
};

Ok(())
// Dial returned `Error::AlreadyConnected`, retry opening the substream.
match self.service.open_substream(peer) {
Ok(substream_id) => {
self.pending_substreams.insert(substream_id, peer);
self.peers
.entry(peer)
.or_default()
.pending_actions
.insert(substream_id, action);
return Ok(());
}
Err(err) => {
tracing::trace!(target: LOG_TARGET, ?query, ?peer, ?err, "Failed to open substream a second time");
return Err(err);
}
}
},
}
}
}

/// Handle next query action.
async fn on_query_action(&mut self, action: QueryAction) -> Result<(), (QueryId, PeerId)> {
match action {
QueryAction::SendMessage { query, peer, .. } => {
if self
.open_substream_or_dial(peer, PeerAction::SendFindNode(query), Some(query))
.is_err()
{
// Announce the error to the query engine.
self.engine.register_response_failure(query, peer);
}
Ok(())
}
QueryAction::FindNodeQuerySucceeded {
target,
peers,
Expand Down Expand Up @@ -720,36 +741,18 @@ impl Kademlia {
let message = KademliaMessage::put_value(record);

for peer in peers {
match self.service.open_substream(peer.peer) {
Ok(substream_id) => {
self.pending_substreams.insert(substream_id, peer.peer);
self.peers
.entry(peer.peer)
.or_default()
.pending_actions
.insert(substream_id, PeerAction::SendPutValue(message.clone()));
}
Err(_) => match self.service.dial(&peer.peer) {
Ok(_) => match self.pending_dials.entry(peer.peer) {
Entry::Occupied(entry) => {
entry
.into_mut()
.push(PeerAction::SendPutValue(message.clone()));
}
Entry::Vacant(entry) => {
entry.insert(vec![PeerAction::SendPutValue(message.clone())]);
}
},
Err(error) => {
tracing::debug!(
target: LOG_TARGET,
?peer,
?key,
?error,
"failed to dial peer",
);
}
},
if let Err(error) = self.open_substream_or_dial(
peer.peer,
PeerAction::SendPutValue(message.clone()),
None,
) {
tracing::debug!(
target: LOG_TARGET,
?peer,
?key,
?error,
"failed to put record to peer",
);
}
}

Expand Down
8 changes: 8 additions & 0 deletions src/protocol/libp2p/kademlia/query/find_node.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,14 @@ impl<T: Clone + Into<Vec<u8>>> FindNodeContext<T> {
// If we cannot make progress, return the final result.
// A query failed when we are not able to identify one single peer.
if self.is_done() {
tracing::trace!(
target: LOG_TARGET,
query = ?self.config.query,
pending = self.pending.len(),
candidates = self.candidates.len(),
"query finished"
);

return if self.responses.is_empty() {
Some(QueryAction::QueryFailed {
query: self.config.query,
Expand Down