diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index 500b500636ba..c68ba26acc0b 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -286,6 +286,7 @@ async fn handle_new_activations( "collation-builder", Box::pin(async move { let persisted_validation_data_hash = validation_data.hash(); + let parent_head_data_hash = validation_data.parent_head.hash(); let (collation, result_sender) = match (task_config.collator)(relay_parent, &validation_data).await { @@ -385,8 +386,13 @@ async fn handle_new_activations( if let Err(err) = task_sender .send( - CollatorProtocolMessage::DistributeCollation(ccr, pov, result_sender) - .into(), + CollatorProtocolMessage::DistributeCollation( + ccr, + parent_head_data_hash, + pov, + result_sender, + ) + .into(), ) .await { diff --git a/node/network/collator-protocol/src/collator_side/collation.rs b/node/network/collator-protocol/src/collator_side/collation.rs new file mode 100644 index 000000000000..36cdc7794b68 --- /dev/null +++ b/node/network/collator-protocol/src/collator_side/collation.rs @@ -0,0 +1,162 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Primitives for tracking collations-related data. + +use std::collections::{HashSet, VecDeque}; + +use futures::{future::BoxFuture, stream::FuturesUnordered}; + +use polkadot_node_network_protocol::{ + request_response::{ + incoming::OutgoingResponse, v1 as protocol_v1, vstaging as protocol_vstaging, + IncomingRequest, + }, + PeerId, +}; +use polkadot_node_primitives::PoV; +use polkadot_primitives::v2::{CandidateHash, CandidateReceipt, Hash, Id as ParaId}; + +/// The status of a collation as seen from the collator. +pub enum CollationStatus { + /// The collation was created, but we did not advertise it to any validator. + Created, + /// The collation was advertised to at least one validator. + Advertised, + /// The collation was requested by at least one validator. + Requested, +} + +impl CollationStatus { + /// Advance to the [`Self::Advertised`] status. + /// + /// This ensures that `self` isn't already [`Self::Requested`]. + pub fn advance_to_advertised(&mut self) { + if !matches!(self, Self::Requested) { + *self = Self::Advertised; + } + } + + /// Advance to the [`Self::Requested`] status. + pub fn advance_to_requested(&mut self) { + *self = Self::Requested; + } +} + +/// A collation built by the collator. +pub struct Collation { + /// Candidate receipt. + pub receipt: CandidateReceipt, + /// Parent head-data hash. + pub parent_head_data_hash: Hash, + /// Proof to verify the state transition of the parachain. + pub pov: PoV, + /// Collation status. + pub status: CollationStatus, +} + +/// Stores the state for waiting collation fetches per relay parent. +#[derive(Default)] +pub struct WaitingCollationFetches { + /// A flag indicating that we have an ongoing request. + /// This limits the number of collations being sent at any moment + /// of time to 1 for each relay parent. + /// + /// If set to `true`, any new request will be queued. + pub collation_fetch_active: bool, + /// The collation fetches waiting to be fulfilled. + pub req_queue: VecDeque, + /// All peers that are waiting or actively uploading. + /// + /// We will not accept multiple requests from the same peer, otherwise our DoS protection of + /// moving on to the next peer after `MAX_UNSHARED_UPLOAD_TIME` would be pointless. + pub waiting_peers: HashSet<(PeerId, CandidateHash)>, +} + +/// Backwards-compatible wrapper for incoming collations requests. +pub enum VersionedCollationRequest { + V1(IncomingRequest), + VStaging(IncomingRequest), +} + +impl From> for VersionedCollationRequest { + fn from(req: IncomingRequest) -> Self { + Self::V1(req) + } +} + +impl From> + for VersionedCollationRequest +{ + fn from(req: IncomingRequest) -> Self { + Self::VStaging(req) + } +} + +impl VersionedCollationRequest { + /// Returns parachain id from the request payload. + pub fn para_id(&self) -> ParaId { + match self { + VersionedCollationRequest::V1(req) => req.payload.para_id, + VersionedCollationRequest::VStaging(req) => req.payload.para_id, + } + } + + /// Returns relay parent from the request payload. + pub fn relay_parent(&self) -> Hash { + match self { + VersionedCollationRequest::V1(req) => req.payload.relay_parent, + VersionedCollationRequest::VStaging(req) => req.payload.relay_parent, + } + } + + /// Returns id of the peer the request was received from. + pub fn peer_id(&self) -> PeerId { + match self { + VersionedCollationRequest::V1(req) => req.peer, + VersionedCollationRequest::VStaging(req) => req.peer, + } + } + + /// Sends the response back to requester. + pub fn send_outgoing_response( + self, + response: OutgoingResponse, + ) -> Result<(), ()> { + match self { + VersionedCollationRequest::V1(req) => req.send_outgoing_response(response), + VersionedCollationRequest::VStaging(req) => req.send_outgoing_response(response), + } + } +} + +/// Result of the finished background send-collation task. +/// +/// Note that if the timeout was hit the request doesn't get +/// aborted, it only indicates that we should start processing +/// the next one from the queue. +pub struct CollationSendResult { + /// Candidate's relay parent. + pub relay_parent: Hash, + /// Candidate hash. + pub candidate_hash: CandidateHash, + /// Peer id. + pub peer_id: PeerId, + /// Whether the max unshared timeout was hit. + pub timed_out: bool, +} + +pub type ActiveCollationFetches = FuturesUnordered>; diff --git a/node/network/collator-protocol/src/collator_side/metrics.rs b/node/network/collator-protocol/src/collator_side/metrics.rs index 85e00406b9ba..04a9806605ab 100644 --- a/node/network/collator-protocol/src/collator_side/metrics.rs +++ b/node/network/collator-protocol/src/collator_side/metrics.rs @@ -1,4 +1,4 @@ -// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// Copyright 2022 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify @@ -20,7 +20,7 @@ use polkadot_node_subsystem_util::metrics::{self, prometheus}; pub struct Metrics(Option); impl Metrics { - pub fn on_advertisment_made(&self) { + pub fn on_advertisement_made(&self) { if let Some(metrics) = &self.0 { metrics.advertisements_made.inc(); } diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 427c92f672de..79222fde117c 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -15,25 +15,26 @@ // along with Polkadot. If not, see . use std::{ - collections::{HashMap, HashSet, VecDeque}, - pin::Pin, - time::{Duration, Instant}, + collections::{HashMap, HashSet}, + convert::TryInto, + time::Duration, }; +use bitvec::{bitvec, vec::BitVec}; use futures::{ - channel::oneshot, pin_mut, select, stream::FuturesUnordered, Future, FutureExt, StreamExt, + channel::oneshot, future::Fuse, pin_mut, select, stream::FuturesUnordered, FutureExt, StreamExt, }; use sp_core::Pair; use polkadot_node_network_protocol::{ self as net_protocol, - peer_set::PeerSet, + peer_set::{CollationVersion, PeerSet}, request_response::{ incoming::{self, OutgoingResponse}, - v1::{self as request_v1, CollationFetchingRequest, CollationFetchingResponse}, - IncomingRequest, IncomingRequestReceiver, + v1 as request_v1, vstaging as request_vstaging, IncomingRequestReceiver, }, - v1 as protocol_v1, OurView, PeerId, UnifiedReputationChange as Rep, Versioned, View, + v1 as protocol_v1, vstaging as protocol_vstaging, OurView, PeerId, + UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_primitives::{CollationSecondedSignal, PoV, Statement}; use polkadot_node_subsystem::{ @@ -41,9 +42,10 @@ use polkadot_node_subsystem::{ messages::{ CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeTxMessage, RuntimeApiMessage, }, - overseer, FromOrchestra, OverseerSignal, PerLeafSpan, + overseer, CollatorProtocolSenderTrait, FromOrchestra, OverseerSignal, PerLeafSpan, }; use polkadot_node_subsystem_util::{ + backing_implicit_view::View as ImplicitView, runtime::{get_availability_cores, get_group_rotation_info, RuntimeInfo}, TimeoutExt, }; @@ -52,20 +54,30 @@ use polkadot_primitives::v2::{ GroupIndex, Hash, Id as ParaId, SessionIndex, }; -use super::LOG_TARGET; -use crate::error::{log_error, Error, FatalError, Result}; -use fatality::Split; +use super::{ + prospective_parachains_mode, ProspectiveParachainsMode, LOG_TARGET, MAX_CANDIDATE_DEPTH, +}; +use crate::{ + error::{log_error, Error, FatalError, Result}, + modify_reputation, +}; +mod collation; mod metrics; +#[cfg(test)] +mod tests; mod validators_buffer; -use validators_buffer::{ValidatorGroupsBuffer, VALIDATORS_BUFFER_CAPACITY}; +use collation::{ + ActiveCollationFetches, Collation, CollationSendResult, CollationStatus, + VersionedCollationRequest, WaitingCollationFetches, +}; +use validators_buffer::{ + ResetInterestTimeout, ValidatorGroupsBuffer, RESET_INTEREST_TIMEOUT, VALIDATORS_BUFFER_CAPACITY, +}; pub use metrics::Metrics; -#[cfg(test)] -mod tests; - const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Peer sent unparsable request"); const COST_UNEXPECTED_MESSAGE: Rep = Rep::CostMinor("An unexpected message"); const COST_APPARENT_FLOOD: Rep = @@ -87,108 +99,112 @@ const MAX_UNSHARED_UPLOAD_TIME: Duration = Duration::from_millis(150); /// Validators are obtained from [`ValidatorGroupsBuffer::validators_to_connect`]. const RECONNECT_TIMEOUT: Duration = Duration::from_secs(12); -/// How often to check for reconnect timeout. -const RECONNECT_POLL: Duration = Duration::from_secs(1); +/// Future that when resolved indicates that we should update reserved peer-set +/// of validators we want to be connected to. +/// +/// `Pending` variant never finishes and should be used when there're no peers +/// connected. +type ReconnectTimeout = Fuse; /// Info about validators we are currently connected to. /// /// It keeps track to which validators we advertised our collation. -#[derive(Debug)] +#[derive(Debug, Default)] struct ValidatorGroup { - /// All [`ValidatorId`]'s of the current group to that we advertised our collation. - advertised_to: HashSet, + /// Validators discovery ids. Lazily initialized when first + /// distributing a collation. + validators: Vec, + + /// Bits indicating which validators have already seen the announcement + /// per candidate. + advertised_to: HashMap, } impl ValidatorGroup { - /// Create a new `ValidatorGroup` - /// - /// without any advertisements. - fn new() -> Self { - Self { advertised_to: HashSet::new() } - } - /// Returns `true` if we should advertise our collation to the given peer. fn should_advertise_to( &self, + candidate_hash: &CandidateHash, peer_ids: &HashMap>, peer: &PeerId, ) -> bool { - match peer_ids.get(peer) { - Some(discovery_ids) => !discovery_ids.iter().any(|d| self.advertised_to.contains(d)), - None => false, + let authority_ids = match peer_ids.get(peer) { + Some(authority_ids) => authority_ids, + None => return false, + }; + + for id in authority_ids { + // One peer id may correspond to different discovery ids across sessions, + // having a non-empty intersection is sufficient to assume that this peer + // belongs to this particular validator group. + let validator_index = match self.validators.iter().position(|v| v == id) { + Some(idx) => idx, + None => continue, + }; + + // Either the candidate is unseen by this validator group + // or the corresponding bit is not set. + if self + .advertised_to + .get(candidate_hash) + .map_or(true, |advertised| !advertised[validator_index]) + { + return true + } } + + false } /// Should be called after we advertised our collation to the given `peer` to keep track of it. fn advertised_to_peer( &mut self, + candidate_hash: &CandidateHash, peer_ids: &HashMap>, peer: &PeerId, ) { if let Some(authority_ids) = peer_ids.get(peer) { - authority_ids.iter().for_each(|a| { - self.advertised_to.insert(a.clone()); - }); - } - } -} - -/// The status of a collation as seen from the collator. -enum CollationStatus { - /// The collation was created, but we did not advertise it to any validator. - Created, - /// The collation was advertised to at least one validator. - Advertised, - /// The collation was requested by at least one validator. - Requested, -} - -impl CollationStatus { - /// Advance to the [`Self::Advertised`] status. - /// - /// This ensures that `self` isn't already [`Self::Requested`]. - fn advance_to_advertised(&mut self) { - if !matches!(self, Self::Requested) { - *self = Self::Advertised; + for id in authority_ids { + let validator_index = match self.validators.iter().position(|v| v == id) { + Some(idx) => idx, + None => continue, + }; + self.advertised_to + .entry(*candidate_hash) + .or_insert_with(|| bitvec![0; self.validators.len()]) + .set(validator_index, true); + } } } - - /// Advance to the [`Self::Requested`] status. - fn advance_to_requested(&mut self) { - *self = Self::Requested; - } } -/// A collation built by the collator. -struct Collation { - receipt: CandidateReceipt, - pov: PoV, - status: CollationStatus, +#[derive(Debug)] +struct PeerData { + /// Peer's view. + view: View, + /// Network protocol version. + version: CollationVersion, } -/// Stores the state for waiting collation fetches. -#[derive(Default)] -struct WaitingCollationFetches { - /// Is there currently a collation getting fetched? - collation_fetch_active: bool, - /// The collation fetches waiting to be fulfilled. - waiting: VecDeque>, - /// All peers that are waiting or actively uploading. - /// - /// We will not accept multiple requests from the same peer, otherwise our DoS protection of - /// moving on to the next peer after `MAX_UNSHARED_UPLOAD_TIME` would be pointless. - waiting_peers: HashSet, +struct PerRelayParent { + prospective_parachains_mode: ProspectiveParachainsMode, + /// Validators group responsible for backing candidates built + /// on top of this relay parent. + validator_group: ValidatorGroup, + /// Distributed collations. + collations: HashMap, } -struct CollationSendResult { - relay_parent: Hash, - peer_id: PeerId, - timed_out: bool, +impl PerRelayParent { + fn new(mode: ProspectiveParachainsMode) -> Self { + Self { + prospective_parachains_mode: mode, + validator_group: ValidatorGroup::default(), + collations: HashMap::new(), + } + } } -type ActiveCollationFetches = - FuturesUnordered + Send + 'static>>>; - struct State { /// Our network peer id. local_peer_id: PeerId, @@ -202,25 +218,34 @@ struct State { /// Track all active peers and their views /// to determine what is relevant to them. - peer_views: HashMap, + peer_data: HashMap, - /// Our own view. - view: OurView, + /// Leaves that do support asynchronous backing along with + /// implicit ancestry. Leaves from the implicit view are present in + /// `active_leaves`, the opposite doesn't hold true. + /// + /// Relay-chain blocks which don't support prospective parachains are + /// never included in the fragment trees of active leaves which do. In + /// particular, this means that if a given relay parent belongs to implicit + /// ancestry of some active leaf, then it does support prospective parachains. + implicit_view: ImplicitView, + + /// All active leaves observed by us, including both that do and do not + /// support prospective parachains. This mapping works as a replacement for + /// [`polkadot_node_network_protocol::View`] and can be dropped once the transition + /// to asynchronous backing is done. + active_leaves: HashMap, + + /// Validators and distributed collations tracked for each relay parent from + /// our view, including both leaves and implicit ancestry. + per_relay_parent: HashMap, /// Span per relay parent. span_per_relay_parent: HashMap, - /// Possessed collations. - /// - /// We will keep up to one local collation per relay-parent. - collations: HashMap, - /// The result senders per collation. collation_result_senders: HashMap>, - /// Our validator groups per active leaf. - our_validators_groups: HashMap, - /// The mapping from [`PeerId`] to [`HashSet`]. This is filled over time as we learn the [`PeerId`]'s /// by `PeerConnected` events. peer_ids: HashMap>, @@ -228,9 +253,9 @@ struct State { /// Tracks which validators we want to stay connected to. validator_groups_buf: ValidatorGroupsBuffer, - /// Timestamp of the last connection request to a non-empty list of validators, - /// `None` otherwise. - last_connected_at: Option, + /// Timeout-future that enforces collator to update the peer-set at least once + /// every [`RECONNECT_TIMEOUT`] seconds. + reconnect_timeout: ReconnectTimeout, /// Metrics. metrics: Metrics, @@ -245,6 +270,14 @@ struct State { /// /// Each future returns the relay parent of the finished collation fetch. active_collation_fetches: ActiveCollationFetches, + + /// Time limits for validators to fetch the collation once the advertisement + /// was sent. + /// + /// Given an implicit view a collation may stay in memory for significant amount + /// of time, if we don't timeout validators the node will keep attempting to connect + /// to unneeded peers. + advertisement_timeouts: FuturesUnordered, } impl State { @@ -256,28 +289,20 @@ impl State { collator_pair, metrics, collating_on: Default::default(), - peer_views: Default::default(), - view: Default::default(), + peer_data: Default::default(), + implicit_view: Default::default(), + active_leaves: Default::default(), + per_relay_parent: Default::default(), span_per_relay_parent: Default::default(), - collations: Default::default(), collation_result_senders: Default::default(), - our_validators_groups: Default::default(), peer_ids: Default::default(), validator_groups_buf: ValidatorGroupsBuffer::with_capacity(VALIDATORS_BUFFER_CAPACITY), - last_connected_at: None, + reconnect_timeout: Fuse::terminated(), waiting_collation_fetches: Default::default(), active_collation_fetches: Default::default(), + advertisement_timeouts: Default::default(), } } - - /// Get all peers which have the given relay parent in their view. - fn peers_interested_in_leaf(&self, relay_parent: &Hash) -> Vec { - self.peer_views - .iter() - .filter(|(_, v)| v.contains(relay_parent)) - .map(|(peer, _)| *peer) - .collect() - } } /// Distribute a collation. @@ -295,52 +320,77 @@ async fn distribute_collation( state: &mut State, id: ParaId, receipt: CandidateReceipt, + parent_head_data_hash: Hash, pov: PoV, result_sender: Option>, ) -> Result<()> { - let relay_parent = receipt.descriptor.relay_parent; + let candidate_relay_parent = receipt.descriptor.relay_parent; let candidate_hash = receipt.hash(); - // This collation is not in the active-leaves set. - if !state.view.contains(&relay_parent) { - gum::warn!( + let per_relay_parent = match state.per_relay_parent.get_mut(&candidate_relay_parent) { + Some(per_relay_parent) => per_relay_parent, + None => { + gum::debug!( + target: LOG_TARGET, + para_id = %id, + candidate_relay_parent = %candidate_relay_parent, + candidate_hash = ?candidate_hash, + "Candidate relay parent is out of our view", + ); + return Ok(()) + }, + }; + let relay_parent_mode = per_relay_parent.prospective_parachains_mode; + + let collations_limit = match relay_parent_mode { + ProspectiveParachainsMode::Disabled => 1, + ProspectiveParachainsMode::Enabled => MAX_CANDIDATE_DEPTH + 1, + }; + + if per_relay_parent.collations.len() >= collations_limit { + gum::debug!( target: LOG_TARGET, - ?relay_parent, - "distribute collation message parent is outside of our view", + ?candidate_relay_parent, + ?relay_parent_mode, + "The limit of {} collations per relay parent is already reached", + collations_limit, ); - return Ok(()) } // We have already seen collation for this relay parent. - if state.collations.contains_key(&relay_parent) { + if per_relay_parent.collations.contains_key(&candidate_hash) { gum::debug!( target: LOG_TARGET, - ?relay_parent, - "Already seen collation for this relay parent", + ?candidate_relay_parent, + ?candidate_hash, + "Already seen this candidate", ); return Ok(()) } // Determine which core the para collated-on is assigned to. // If it is not scheduled then ignore the message. - let (our_core, num_cores) = match determine_core(ctx.sender(), id, relay_parent).await? { - Some(core) => core, - None => { - gum::warn!( - target: LOG_TARGET, - para_id = %id, - ?relay_parent, - "looks like no core is assigned to {} at {}", id, relay_parent, - ); + let (our_core, num_cores) = + match determine_core(ctx.sender(), id, candidate_relay_parent, relay_parent_mode).await? { + Some(core) => core, + None => { + gum::warn!( + target: LOG_TARGET, + para_id = %id, + "looks like no core is assigned to {} at {}", id, candidate_relay_parent, + ); - return Ok(()) - }, - }; + return Ok(()) + }, + }; // Determine the group on that core. + // + // When prospective parachains are disabled, candidate relay parent here is + // guaranteed to be an active leaf. let GroupValidators { validators, session_index, group_index } = - determine_our_validators(ctx, runtime, our_core, num_cores, relay_parent).await?; + determine_our_validators(ctx, runtime, our_core, num_cores, candidate_relay_parent).await?; if validators.is_empty() { gum::warn!( @@ -352,13 +402,13 @@ async fn distribute_collation( return Ok(()) } - // It's important to insert new collation bits **before** + // It's important to insert new collation interests **before** // issuing a connection request. // // If a validator managed to fetch all the relevant collations // but still assigned to our core, we keep the connection alive. state.validator_groups_buf.note_collation_advertised( - relay_parent, + candidate_hash, session_index, group_index, &validators, @@ -367,7 +417,8 @@ async fn distribute_collation( gum::debug!( target: LOG_TARGET, para_id = %id, - relay_parent = %relay_parent, + candidate_relay_parent = %candidate_relay_parent, + relay_parent_mode = ?relay_parent_mode, ?candidate_hash, pov_hash = ?pov.hash(), core = ?our_core, @@ -375,23 +426,56 @@ async fn distribute_collation( "Accepted collation, connecting to validators." ); - // Update a set of connected validators if necessary. - state.last_connected_at = connect_to_validators(ctx, &state.validator_groups_buf).await; + let validators_at_relay_parent = &mut per_relay_parent.validator_group.validators; + if validators_at_relay_parent.is_empty() { + *validators_at_relay_parent = validators; + } - state.our_validators_groups.insert(relay_parent, ValidatorGroup::new()); + // Update a set of connected validators if necessary. + state.reconnect_timeout = connect_to_validators(ctx, &state.validator_groups_buf).await; if let Some(result_sender) = result_sender { state.collation_result_senders.insert(candidate_hash, result_sender); } - state - .collations - .insert(relay_parent, Collation { receipt, pov, status: CollationStatus::Created }); + per_relay_parent.collations.insert( + candidate_hash, + Collation { receipt, parent_head_data_hash, pov, status: CollationStatus::Created }, + ); + + // If prospective parachains are disabled, a leaf should be known to peer. + // Otherwise, it should be present in allowed ancestry of some leaf. + // + // It's collation-producer responsibility to verify that there exists + // a hypothetical membership in a fragment tree for candidate. + let interested = + state + .peer_data + .iter() + .filter(|(_, PeerData { view: v, .. })| match relay_parent_mode { + ProspectiveParachainsMode::Disabled => v.contains(&candidate_relay_parent), + ProspectiveParachainsMode::Enabled => v.iter().any(|block_hash| { + state + .implicit_view + .known_allowed_relay_parents_under(block_hash, Some(id)) + .unwrap_or_default() + .contains(&candidate_relay_parent) + }), + }); - let interested = state.peers_interested_in_leaf(&relay_parent); // Make sure already connected peers get collations: - for peer_id in interested { - advertise_collation(ctx, state, relay_parent, peer_id).await; + for (peer_id, peer_data) in interested { + advertise_collation( + ctx, + candidate_relay_parent, + per_relay_parent, + peer_id, + peer_data.version, + &state.peer_ids, + &mut state.advertisement_timeouts, + &state.metrics, + ) + .await; } Ok(()) @@ -403,14 +487,26 @@ async fn determine_core( sender: &mut impl overseer::SubsystemSender, para_id: ParaId, relay_parent: Hash, + relay_parent_mode: ProspectiveParachainsMode, ) -> Result> { let cores = get_availability_cores(sender, relay_parent).await?; for (idx, core) in cores.iter().enumerate() { - if let CoreState::Scheduled(occupied) = core { - if occupied.para_id == para_id { - return Ok(Some(((idx as u32).into(), cores.len()))) - } + let core_para_id = match core { + CoreState::Scheduled(scheduled) => Some(scheduled.para_id), + CoreState::Occupied(occupied) => + if relay_parent_mode.is_enabled() { + // With async backing we don't care about the core state, + // it is only needed for figuring our validators group. + Some(occupied.candidate_descriptor.para_id) + } else { + None + }, + CoreState::Free => None, + }; + + if core_para_id == Some(para_id) { + return Ok(Some(((idx as u32).into(), cores.len()))) } } @@ -467,36 +563,62 @@ async fn determine_our_validators( Ok(current_validators) } -/// Issue a `Declare` collation message to the given `peer`. -#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] -async fn declare(ctx: &mut Context, state: &mut State, peer: PeerId) { - let declare_signature_payload = protocol_v1::declare_signature_payload(&state.local_peer_id); - - if let Some(para_id) = state.collating_on { - let wire_message = protocol_v1::CollatorProtocolMessage::Declare( - state.collator_pair.public(), - para_id, - state.collator_pair.sign(&declare_signature_payload), - ); +/// Construct the declare message to be sent to validator depending on its +/// network protocol version. +fn declare_message( + state: &mut State, + version: CollationVersion, +) -> Option> { + let para_id = state.collating_on?; + Some(match version { + CollationVersion::V1 => { + let declare_signature_payload = + protocol_v1::declare_signature_payload(&state.local_peer_id); + let wire_message = protocol_v1::CollatorProtocolMessage::Declare( + state.collator_pair.public(), + para_id, + state.collator_pair.sign(&declare_signature_payload), + ); + Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)) + }, + CollationVersion::VStaging => { + let declare_signature_payload = + protocol_vstaging::declare_signature_payload(&state.local_peer_id); + let wire_message = protocol_vstaging::CollatorProtocolMessage::Declare( + state.collator_pair.public(), + para_id, + state.collator_pair.sign(&declare_signature_payload), + ); + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + wire_message, + )) + }, + }) +} - ctx.send_message(NetworkBridgeTxMessage::SendCollationMessage( - vec![peer], - Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)), - )) - .await; +/// Issue versioned `Declare` collation message to the given `peer`. +#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] +async fn declare( + ctx: &mut Context, + state: &mut State, + peer: &PeerId, + version: CollationVersion, +) { + if let Some(wire_message) = declare_message(state, version) { + ctx.send_message(NetworkBridgeTxMessage::SendCollationMessage(vec![*peer], wire_message)) + .await; } } /// Updates a set of connected validators based on their advertisement-bits /// in a validators buffer. /// -/// Returns current timestamp if the connection request was non-empty, `None` -/// otherwise. +/// Should be called again once a returned future resolves. #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn connect_to_validators( ctx: &mut Context, validator_groups_buf: &ValidatorGroupsBuffer, -) -> Option { +) -> ReconnectTimeout { let validator_ids = validator_groups_buf.validators_to_connect(); let is_disconnect = validator_ids.is_empty(); @@ -510,69 +632,105 @@ async fn connect_to_validators( }) .await; - (!is_disconnect).then_some(Instant::now()) + if is_disconnect { + gum::trace!(target: LOG_TARGET, "Disconnecting from all peers"); + // Never resolves. + Fuse::terminated() + } else { + futures_timer::Delay::new(RECONNECT_TIMEOUT).fuse() + } } /// Advertise collation to the given `peer`. /// -/// This will only advertise a collation if there exists one for the given `relay_parent` and the given `peer` is -/// set as validator for our para at the given `relay_parent`. +/// This will only advertise a collation if there exists at least one for the given +/// `relay_parent` and the given `peer` is set as validator for our para at the given `relay_parent`. +/// +/// We also make sure not to advertise the same collation multiple times to the same validator. #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn advertise_collation( ctx: &mut Context, - state: &mut State, relay_parent: Hash, - peer: PeerId, + per_relay_parent: &mut PerRelayParent, + peer: &PeerId, + protocol_version: CollationVersion, + peer_ids: &HashMap>, + advertisement_timeouts: &mut FuturesUnordered, + metrics: &Metrics, ) { - let should_advertise = state - .our_validators_groups - .get(&relay_parent) - .map(|g| g.should_advertise_to(&state.peer_ids, &peer)) - .unwrap_or(false); + for (candidate_hash, collation) in per_relay_parent.collations.iter_mut() { + // Check that peer will be able to request the collation. + if let CollationVersion::V1 = protocol_version { + if per_relay_parent.prospective_parachains_mode.is_enabled() { + gum::trace!( + target: LOG_TARGET, + ?relay_parent, + peer_id = %peer, + "Skipping advertising to validator, incorrect network protocol version", + ); + return + } + } - match (state.collations.get_mut(&relay_parent), should_advertise) { - (None, _) => { - gum::trace!( - target: LOG_TARGET, - ?relay_parent, - peer_id = %peer, - "No collation to advertise.", - ); - return - }, - (_, false) => { - gum::debug!( - target: LOG_TARGET, - ?relay_parent, - peer_id = %peer, - "Not advertising collation as we already advertised it to this validator.", - ); - return - }, - (Some(collation), true) => { + let should_advertise = + per_relay_parent + .validator_group + .should_advertise_to(candidate_hash, peer_ids, &peer); + + if !should_advertise { gum::debug!( target: LOG_TARGET, ?relay_parent, peer_id = %peer, - "Advertising collation.", + "Not advertising collation since validator is not interested", ); - collation.status.advance_to_advertised() - }, - } + continue + } - let wire_message = protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent); + gum::debug!( + target: LOG_TARGET, + ?relay_parent, + peer_id = %peer, + "Advertising collation.", + ); + collation.status.advance_to_advertised(); + + let collation_message = match protocol_version { + CollationVersion::VStaging => { + let wire_message = protocol_vstaging::CollatorProtocolMessage::AdvertiseCollation { + relay_parent, + candidate_hash: *candidate_hash, + parent_head_data_hash: collation.parent_head_data_hash, + }; + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + wire_message, + )) + }, + CollationVersion::V1 => { + let wire_message = + protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent); + Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)) + }, + }; - ctx.send_message(NetworkBridgeTxMessage::SendCollationMessage( - vec![peer.clone()], - Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)), - )) - .await; + ctx.send_message(NetworkBridgeTxMessage::SendCollationMessage( + vec![peer.clone()], + collation_message, + )) + .await; - if let Some(validators) = state.our_validators_groups.get_mut(&relay_parent) { - validators.advertised_to_peer(&state.peer_ids, &peer); - } + per_relay_parent + .validator_group + .advertised_to_peer(candidate_hash, &peer_ids, peer); + + advertisement_timeouts.push(ResetInterestTimeout::new( + *candidate_hash, + *peer, + RESET_INTEREST_TIMEOUT, + )); - state.metrics.on_advertisment_made(); + metrics.on_advertisement_made(); + } } /// The main incoming message dispatching switch. @@ -589,12 +747,13 @@ async fn process_msg( CollateOn(id) => { state.collating_on = Some(id); }, - DistributeCollation(receipt, pov, result_sender) => { + DistributeCollation(receipt, parent_head_data_hash, pov, result_sender) => { let _span1 = state .span_per_relay_parent .get(&receipt.descriptor.relay_parent) .map(|s| s.child("distributing-collation")); let _span2 = jaeger::Span::new(&pov, "distributing-collation"); + match state.collating_on { Some(id) if receipt.descriptor.para_id != id => { // If the ParaId of a collation requested to be distributed does not match @@ -608,8 +767,17 @@ async fn process_msg( }, Some(id) => { let _ = state.metrics.time_collation_distribution("distribute"); - distribute_collation(ctx, runtime, state, id, receipt, pov, result_sender) - .await?; + distribute_collation( + ctx, + runtime, + state, + id, + receipt, + parent_head_data_hash, + pov, + result_sender, + ) + .await?; }, None => { gum::warn!( @@ -647,17 +815,20 @@ async fn process_msg( /// Issue a response to a previously requested collation. async fn send_collation( state: &mut State, - request: IncomingRequest, + request: VersionedCollationRequest, receipt: CandidateReceipt, pov: PoV, ) { let (tx, rx) = oneshot::channel(); - let relay_parent = request.payload.relay_parent; - let peer_id = request.peer; + let relay_parent = request.relay_parent(); + let peer_id = request.peer_id(); + let candidate_hash = receipt.hash(); + // The response payload is the same for both versions of protocol + // and doesn't have vstaging alias for simplicity. let response = OutgoingResponse { - result: Ok(CollationFetchingResponse::Collation(receipt, pov)), + result: Ok(request_v1::CollationFetchingResponse::Collation(receipt, pov)), reputation_changes: Vec::new(), sent_feedback: Some(tx), }; @@ -671,7 +842,7 @@ async fn send_collation( let r = rx.timeout(MAX_UNSHARED_UPLOAD_TIME).await; let timed_out = r.is_none(); - CollationSendResult { relay_parent, peer_id, timed_out } + CollationSendResult { relay_parent, candidate_hash, peer_id, timed_out } } .boxed(), ); @@ -686,12 +857,16 @@ async fn handle_incoming_peer_message( runtime: &mut RuntimeInfo, state: &mut State, origin: PeerId, - msg: protocol_v1::CollatorProtocolMessage, + msg: Versioned< + protocol_v1::CollatorProtocolMessage, + protocol_vstaging::CollatorProtocolMessage, + >, ) -> Result<()> { - use protocol_v1::CollatorProtocolMessage::*; + use protocol_v1::CollatorProtocolMessage as V1; + use protocol_vstaging::CollatorProtocolMessage as VStaging; match msg { - Declare(_, _, _) => { + Versioned::V1(V1::Declare(..)) | Versioned::VStaging(VStaging::Declare(..)) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -702,24 +877,22 @@ async fn handle_incoming_peer_message( ctx.send_message(NetworkBridgeTxMessage::DisconnectPeer(origin, PeerSet::Collation)) .await; }, - AdvertiseCollation(_) => { + Versioned::V1(V1::AdvertiseCollation(_)) | + Versioned::VStaging(VStaging::AdvertiseCollation { .. }) => { gum::trace!( target: LOG_TARGET, ?origin, "AdvertiseCollation message is not expected on the collator side of the protocol", ); - ctx.send_message(NetworkBridgeTxMessage::ReportPeer( - origin.clone(), - COST_UNEXPECTED_MESSAGE, - )) - .await; + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; // If we are advertised to, this is another collator, and we should disconnect. ctx.send_message(NetworkBridgeTxMessage::DisconnectPeer(origin, PeerSet::Collation)) .await; }, - CollationSeconded(relay_parent, statement) => { + Versioned::V1(V1::CollationSeconded(relay_parent, statement)) | + Versioned::VStaging(VStaging::CollationSeconded(relay_parent, statement)) => { if !matches!(statement.unchecked_payload(), Statement::Seconded(_)) { gum::warn!( target: LOG_TARGET, @@ -764,48 +937,82 @@ async fn handle_incoming_peer_message( async fn handle_incoming_request( ctx: &mut Context, state: &mut State, - req: IncomingRequest, + req: std::result::Result, ) -> Result<()> { + let req = req?; + let relay_parent = req.relay_parent(); + let peer_id = req.peer_id(); + let para_id = req.para_id(); + let _span = state .span_per_relay_parent - .get(&req.payload.relay_parent) + .get(&relay_parent) .map(|s| s.child("request-collation")); match state.collating_on { - Some(our_para_id) if our_para_id == req.payload.para_id => { - let (receipt, pov) = - if let Some(collation) = state.collations.get_mut(&req.payload.relay_parent) { - collation.status.advance_to_requested(); - (collation.receipt.clone(), collation.pov.clone()) - } else { + Some(our_para_id) if our_para_id == para_id => { + let per_relay_parent = match state.per_relay_parent.get_mut(&relay_parent) { + Some(per_relay_parent) => per_relay_parent, + None => { + gum::debug!( + target: LOG_TARGET, + relay_parent = %relay_parent, + "received a `RequestCollation` for a relay parent out of our view", + ); + + return Ok(()) + }, + }; + let mode = per_relay_parent.prospective_parachains_mode; + + let collation = match &req { + VersionedCollationRequest::V1(_) if !mode.is_enabled() => + per_relay_parent.collations.values_mut().next(), + VersionedCollationRequest::VStaging(req) => + per_relay_parent.collations.get_mut(&req.payload.candidate_hash), + _ => { gum::warn!( target: LOG_TARGET, - relay_parent = %req.payload.relay_parent, - "received a `RequestCollation` for a relay parent we don't have collation stored.", + relay_parent = %relay_parent, + prospective_parachains_mode = ?mode, + ?peer_id, + "Collation request version is invalid", ); return Ok(()) - }; + }, + }; + let (receipt, pov) = if let Some(collation) = collation { + collation.status.advance_to_requested(); + (collation.receipt.clone(), collation.pov.clone()) + } else { + gum::warn!( + target: LOG_TARGET, + relay_parent = %relay_parent, + "received a `RequestCollation` for a relay parent we don't have collation stored.", + ); + + return Ok(()) + }; state.metrics.on_collation_sent_requested(); let _span = _span.as_ref().map(|s| s.child("sending")); - let waiting = - state.waiting_collation_fetches.entry(req.payload.relay_parent).or_default(); + let waiting = state.waiting_collation_fetches.entry(relay_parent).or_default(); + let candidate_hash = receipt.hash(); - if !waiting.waiting_peers.insert(req.peer) { + if !waiting.waiting_peers.insert((peer_id, candidate_hash)) { gum::debug!( target: LOG_TARGET, "Dropping incoming request as peer has a request in flight already." ); - ctx.send_message(NetworkBridgeTxMessage::ReportPeer(req.peer, COST_APPARENT_FLOOD)) - .await; + modify_reputation(ctx.sender(), peer_id, COST_APPARENT_FLOOD).await; return Ok(()) } if waiting.collation_fetch_active { - waiting.waiting.push_back(req); + waiting.req_queue.push_back(req); } else { waiting.collation_fetch_active = true; // Obtain a timer for sending collation @@ -816,7 +1023,7 @@ async fn handle_incoming_request( Some(our_para_id) => { gum::warn!( target: LOG_TARGET, - for_para_id = %req.payload.para_id, + for_para_id = %para_id, our_para_id = %our_para_id, "received a `CollationFetchingRequest` for unexpected para_id", ); @@ -824,7 +1031,7 @@ async fn handle_incoming_request( None => { gum::warn!( target: LOG_TARGET, - for_para_id = %req.payload.para_id, + for_para_id = %para_id, "received a `RequestCollation` while not collating on any para", ); }, @@ -832,7 +1039,8 @@ async fn handle_incoming_request( Ok(()) } -/// Our view has changed. +/// Peer's view has changed. Send advertisements for new relay parents +/// if there're any. #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] async fn handle_peer_view_change( ctx: &mut Context, @@ -840,14 +1048,54 @@ async fn handle_peer_view_change( peer_id: PeerId, view: View, ) { - let current = state.peer_views.entry(peer_id.clone()).or_default(); + let PeerData { view: current, version } = match state.peer_data.get_mut(&peer_id) { + Some(peer_data) => peer_data, + None => return, + }; let added: Vec = view.difference(&*current).cloned().collect(); *current = view; for added in added.into_iter() { - advertise_collation(ctx, state, added, peer_id.clone()).await; + let block_hashes = match state + .per_relay_parent + .get(&added) + .map(|per_relay_parent| per_relay_parent.prospective_parachains_mode) + { + Some(ProspectiveParachainsMode::Disabled) => std::slice::from_ref(&added), + Some(ProspectiveParachainsMode::Enabled) => state + .implicit_view + .known_allowed_relay_parents_under(&added, state.collating_on) + .unwrap_or_default(), + None => { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + new_leaf = ?added, + "New leaf in peer's view is unknown", + ); + continue + }, + }; + + for block_hash in block_hashes { + let per_relay_parent = match state.per_relay_parent.get_mut(block_hash) { + Some(per_relay_parent) => per_relay_parent, + None => continue, + }; + advertise_collation( + ctx, + *block_hash, + per_relay_parent, + &peer_id, + *version, + &state.peer_ids, + &mut state.advertisement_timeouts, + &state.metrics, + ) + .await; + } } } @@ -862,10 +1110,30 @@ async fn handle_network_msg( use NetworkBridgeEvent::*; match bridge_message { - PeerConnected(peer_id, observed_role, _, maybe_authority) => { + PeerConnected(peer_id, observed_role, protocol_version, maybe_authority) => { // If it is possible that a disconnected validator would attempt a reconnect // it should be handled here. gum::trace!(target: LOG_TARGET, ?peer_id, ?observed_role, "Peer connected"); + + let version = match protocol_version.try_into() { + Ok(version) => version, + Err(err) => { + // Network bridge is expected to handle this. + gum::error!( + target: LOG_TARGET, + ?peer_id, + ?observed_role, + ?err, + "Unsupported protocol version" + ); + return Ok(()) + }, + }; + state + .peer_data + .entry(peer_id) + .or_insert_with(|| PeerData { view: View::default(), version }); + if let Some(authority_ids) = maybe_authority { gum::trace!( target: LOG_TARGET, @@ -875,7 +1143,7 @@ async fn handle_network_msg( ); state.peer_ids.insert(peer_id, authority_ids); - declare(ctx, state, peer_id).await; + declare(ctx, state, &peer_id, version).await; } }, PeerViewChange(peer_id, view) => { @@ -884,17 +1152,16 @@ async fn handle_network_msg( }, PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); - state.peer_views.remove(&peer_id); + state.peer_data.remove(&peer_id); state.peer_ids.remove(&peer_id); }, OurViewChange(view) => { gum::trace!(target: LOG_TARGET, ?view, "Own view change"); - handle_our_view_change(state, view).await?; + handle_our_view_change(ctx.sender(), state, view).await?; }, - PeerMessage(remote, Versioned::V1(msg)) => { + PeerMessage(remote, msg) => { handle_incoming_peer_message(ctx, runtime, state, remote, msg).await?; }, - PeerMessage(_, Versioned::VStaging(_msg)) => {}, NewGossipTopology { .. } => { // impossible! }, @@ -904,42 +1171,99 @@ async fn handle_network_msg( } /// Handles our view changes. -async fn handle_our_view_change(state: &mut State, view: OurView) -> Result<()> { - for removed in state.view.difference(&view) { - gum::debug!(target: LOG_TARGET, relay_parent = ?removed, "Removing relay parent because our view changed."); +async fn handle_our_view_change( + sender: &mut Sender, + state: &mut State, + view: OurView, +) -> Result<()> +where + Sender: CollatorProtocolSenderTrait, +{ + let current_leaves = state.active_leaves.clone(); - if let Some(collation) = state.collations.remove(removed) { - state.collation_result_senders.remove(&collation.receipt.hash()); + let removed = current_leaves.iter().filter(|(h, _)| !view.contains(*h)); + let added = view.iter().filter(|h| !current_leaves.contains_key(h)); - match collation.status { - CollationStatus::Created => gum::warn!( - target: LOG_TARGET, - candidate_hash = ?collation.receipt.hash(), - pov_hash = ?collation.pov.hash(), - "Collation wasn't advertised to any validator.", - ), - CollationStatus::Advertised => gum::debug!( - target: LOG_TARGET, - candidate_hash = ?collation.receipt.hash(), - pov_hash = ?collation.pov.hash(), - "Collation was advertised but not requested by any validator.", - ), - CollationStatus::Requested => gum::debug!( - target: LOG_TARGET, - candidate_hash = ?collation.receipt.hash(), - pov_hash = ?collation.pov.hash(), - "Collation was requested.", - ), + for leaf in added { + let mode = prospective_parachains_mode(sender, *leaf).await?; + + if let Some(span) = view.span_per_head().get(leaf).cloned() { + let per_leaf_span = PerLeafSpan::new(span, "collator-side"); + state.span_per_relay_parent.insert(*leaf, per_leaf_span); + } + + state.active_leaves.insert(*leaf, mode); + state.per_relay_parent.insert(*leaf, PerRelayParent::new(mode)); + + if mode.is_enabled() { + state + .implicit_view + .activate_leaf(sender, *leaf) + .await + .map_err(Error::ImplicitViewFetchError)?; + + let allowed_ancestry = state + .implicit_view + .known_allowed_relay_parents_under(leaf, state.collating_on) + .unwrap_or_default(); + for block_hash in allowed_ancestry { + state + .per_relay_parent + .entry(*block_hash) + .or_insert_with(|| PerRelayParent::new(ProspectiveParachainsMode::Enabled)); } } - state.our_validators_groups.remove(removed); - state.span_per_relay_parent.remove(removed); - state.waiting_collation_fetches.remove(removed); - state.validator_groups_buf.remove_relay_parent(removed); } - state.view = view; - + for (leaf, mode) in removed { + state.active_leaves.remove(leaf); + // If the leaf is deactivated it still may stay in the view as a part + // of implicit ancestry. Only update the state after the hash is actually + // pruned from the block info storage. + let pruned = if mode.is_enabled() { + state.implicit_view.deactivate_leaf(*leaf) + } else { + vec![*leaf] + }; + + for removed in &pruned { + gum::debug!(target: LOG_TARGET, relay_parent = ?removed, "Removing relay parent because our view changed."); + + let collations = state + .per_relay_parent + .remove(removed) + .map(|per_relay_parent| per_relay_parent.collations) + .unwrap_or_default(); + for collation in collations.into_values() { + let candidate_hash = collation.receipt.hash(); + state.collation_result_senders.remove(&candidate_hash); + state.validator_groups_buf.remove_candidate(&candidate_hash); + + match collation.status { + CollationStatus::Created => gum::warn!( + target: LOG_TARGET, + candidate_hash = ?collation.receipt.hash(), + pov_hash = ?collation.pov.hash(), + "Collation wasn't advertised to any validator.", + ), + CollationStatus::Advertised => gum::debug!( + target: LOG_TARGET, + candidate_hash = ?collation.receipt.hash(), + pov_hash = ?collation.pov.hash(), + "Collation was advertised but not requested by any validator.", + ), + CollationStatus::Requested => gum::debug!( + target: LOG_TARGET, + candidate_hash = ?collation.receipt.hash(), + pov_hash = ?collation.pov.hash(), + "Collation was requested.", + ), + } + } + state.span_per_relay_parent.remove(removed); + state.waiting_collation_fetches.remove(removed); + } + } Ok(()) } @@ -949,7 +1273,8 @@ pub(crate) async fn run( mut ctx: Context, local_peer_id: PeerId, collator_pair: CollatorPair, - mut req_receiver: IncomingRequestReceiver, + mut req_v1_receiver: IncomingRequestReceiver, + mut req_v2_receiver: IncomingRequestReceiver, metrics: Metrics, ) -> std::result::Result<(), FatalError> { use OverseerSignal::*; @@ -957,12 +1282,14 @@ pub(crate) async fn run( let mut state = State::new(local_peer_id, collator_pair, metrics); let mut runtime = RuntimeInfo::new(None); - let reconnect_stream = super::tick_stream(RECONNECT_POLL); - pin_mut!(reconnect_stream); - loop { - let recv_req = req_receiver.recv(|| vec![COST_INVALID_REQUEST]).fuse(); - pin_mut!(recv_req); + let reputation_changes = || vec![COST_INVALID_REQUEST]; + let recv_req_v1 = req_v1_receiver.recv(reputation_changes).fuse(); + let recv_req_v2 = req_v2_receiver.recv(reputation_changes).fuse(); + pin_mut!(recv_req_v1); + pin_mut!(recv_req_v2); + + let mut reconnect_timeout = &mut state.reconnect_timeout; select! { msg = ctx.recv().fuse() => match msg.map_err(FatalError::SubsystemReceive)? { FromOrchestra::Communication { msg } => { @@ -975,28 +1302,30 @@ pub(crate) async fn run( FromOrchestra::Signal(BlockFinalized(..)) => {} FromOrchestra::Signal(Conclude) => return Ok(()), }, - CollationSendResult { - relay_parent, - peer_id, - timed_out, - } = state.active_collation_fetches.select_next_some() => { - if timed_out { - gum::debug!( - target: LOG_TARGET, - ?relay_parent, - ?peer_id, - "Sending collation to validator timed out, carrying on with next validator", - ); - } else { - for authority_id in state.peer_ids.get(&peer_id).into_iter().flatten() { - // Timeout not hit, this peer is no longer interested in this relay parent. - state.validator_groups_buf.reset_validator_interest(relay_parent, authority_id); + CollationSendResult { relay_parent, candidate_hash, peer_id, timed_out } = + state.active_collation_fetches.select_next_some() => { + let next = if let Some(waiting) = state.waiting_collation_fetches.get_mut(&relay_parent) { + if timed_out { + gum::debug!( + target: LOG_TARGET, + ?relay_parent, + ?peer_id, + ?candidate_hash, + "Sending collation to validator timed out, carrying on with next validator." + ); + // We try to throttle requests per relay parent to give validators + // more bandwidth, but if the collation is not received within the + // timeout, we simply start processing next request. + // The request it still alive, it should be kept in a waiting queue. + } else { + for authority_id in state.peer_ids.get(&peer_id).into_iter().flatten() { + // Timeout not hit, this peer is no longer interested in this relay parent. + state.validator_groups_buf.reset_validator_interest(candidate_hash, authority_id); + } + waiting.waiting_peers.remove(&(peer_id, candidate_hash)); } - } - let next = if let Some(waiting) = state.waiting_collation_fetches.get_mut(&relay_parent) { - waiting.waiting_peers.remove(&peer_id); - if let Some(next) = waiting.waiting.pop_front() { + if let Some(next) = waiting.req_queue.pop_front() { next } else { waiting.collation_fetch_active = false; @@ -1007,53 +1336,69 @@ pub(crate) async fn run( continue }; - if let Some(collation) = state.collations.get(&relay_parent) { + let next_collation = { + let per_relay_parent = match state.per_relay_parent.get(&relay_parent) { + Some(per_relay_parent) => per_relay_parent, + None => continue, + }; + + match (per_relay_parent.prospective_parachains_mode, &next) { + (ProspectiveParachainsMode::Disabled, VersionedCollationRequest::V1(_)) => { + per_relay_parent.collations.values().next() + }, + (ProspectiveParachainsMode::Enabled, VersionedCollationRequest::VStaging(req)) => { + per_relay_parent.collations.get(&req.payload.candidate_hash) + }, + _ => { + // Request version is checked in `handle_incoming_request`. + continue + }, + } + }; + + if let Some(collation) = next_collation { let receipt = collation.receipt.clone(); let pov = collation.pov.clone(); send_collation(&mut state, next, receipt, pov).await; } }, - _ = reconnect_stream.next() => { - let now = Instant::now(); - if state - .last_connected_at - .map_or(false, |timestamp| now - timestamp > RECONNECT_TIMEOUT) - { - // Remove all advertisements from the buffer if the timeout was hit. - // Usually, it shouldn't be necessary as leaves get deactivated, rather - // serves as a safeguard against finality lags. - state.validator_groups_buf.clear_advertisements(); - // Returns `None` if connection request is empty. - state.last_connected_at = - connect_to_validators(&mut ctx, &state.validator_groups_buf).await; - - gum::debug!( - target: LOG_TARGET, - timeout = ?RECONNECT_TIMEOUT, - "Timeout hit, sent a connection request. Disconnected from all validators = {}", - state.last_connected_at.is_none(), - ); + (candidate_hash, peer_id) = state.advertisement_timeouts.select_next_some() => { + // NOTE: it doesn't necessarily mean that a validator gets disconnected, + // it only will if there're no other advertisements we want to send. + // + // No-op if the collation was already fetched or went out of view. + for authority_id in state.peer_ids.get(&peer_id).into_iter().flatten() { + state + .validator_groups_buf + .reset_validator_interest(candidate_hash, &authority_id); } + } + _ = reconnect_timeout => { + state.reconnect_timeout = + connect_to_validators(&mut ctx, &state.validator_groups_buf).await; + + gum::trace!( + target: LOG_TARGET, + timeout = ?RECONNECT_TIMEOUT, + "Peer-set updated due to a timeout" + ); }, - in_req = recv_req => { - match in_req { - Ok(req) => { - log_error( - handle_incoming_request(&mut ctx, &mut state, req).await, - "Handling incoming request" - )?; - } - Err(error) => { - let jfyi = error.split().map_err(incoming::Error::from)?; - gum::debug!( - target: LOG_TARGET, - error = ?jfyi, - "Decoding incoming request failed" - ); - continue - } - } + in_req = recv_req_v1 => { + let request = in_req.map(VersionedCollationRequest::from); + + log_error( + handle_incoming_request(&mut ctx, &mut state, request).await, + "Handling incoming collation fetch request V1" + )?; + } + in_req = recv_req_v2 => { + let request = in_req.map(VersionedCollationRequest::from); + + log_error( + handle_incoming_request(&mut ctx, &mut state, request).await, + "Handling incoming collation fetch request VStaging" + )?; } } } diff --git a/node/network/collator-protocol/src/collator_side/tests.rs b/node/network/collator-protocol/src/collator_side/tests/mod.rs similarity index 74% rename from node/network/collator-protocol/src/collator_side/tests.rs rename to node/network/collator-protocol/src/collator_side/tests/mod.rs index c20a2d6c97a5..032d1782812e 100644 --- a/node/network/collator-protocol/src/collator_side/tests.rs +++ b/node/network/collator-protocol/src/collator_side/tests/mod.rs @@ -44,11 +44,15 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::v2::{ - AuthorityDiscoveryId, CollatorPair, GroupRotationInfo, ScheduledCore, SessionIndex, + AuthorityDiscoveryId, CollatorPair, GroupIndex, GroupRotationInfo, ScheduledCore, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, }; use polkadot_primitives_test_helpers::TestCandidateBuilder; +mod prospective_parachains; + +const API_VERSION_PROSPECTIVE_DISABLED: u32 = 2; + #[derive(Clone)] struct TestState { para_id: ParaId, @@ -184,6 +188,17 @@ impl TestState { )), ) .await; + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + relay_parent, + RuntimeApiRequest::Version(tx) + )) => { + assert_eq!(relay_parent, self.relay_parent); + tx.send(Ok(API_VERSION_PROSPECTIVE_DISABLED)).unwrap(); + } + ); } } @@ -191,7 +206,8 @@ type VirtualOverseer = test_helpers::TestSubsystemContextHandle>( @@ -212,15 +228,24 @@ fn test_harness>( let genesis_hash = Hash::repeat_byte(0xff); let req_protocol_names = ReqProtocolNames::new(&genesis_hash, None); - let (collation_req_receiver, req_cfg) = + let (collation_req_receiver, req_v1_cfg) = + IncomingRequest::get_config_receiver(&req_protocol_names); + let (collation_req_vstaging_receiver, req_vstaging_cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); let subsystem = async { - run(context, local_peer_id, collator_pair, collation_req_receiver, Default::default()) - .await - .unwrap(); + run( + context, + local_peer_id, + collator_pair, + collation_req_receiver, + collation_req_vstaging_receiver, + Default::default(), + ) + .await + .unwrap(); }; - let test_fut = test(TestHarness { virtual_overseer, req_cfg }); + let test_fut = test(TestHarness { virtual_overseer, req_v1_cfg, req_vstaging_cfg }); futures::pin_mut!(test_fut); futures::pin_mut!(subsystem); @@ -294,6 +319,17 @@ async fn setup_system(virtual_overseer: &mut VirtualOverseer, test_state: &TestS ])), ) .await; + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + relay_parent, + RuntimeApiRequest::Version(tx) + )) => { + assert_eq!(relay_parent, test_state.relay_parent); + tx.send(Ok(API_VERSION_PROSPECTIVE_DISABLED)).unwrap(); + } + ); } /// Result of [`distribute_collation`] @@ -302,29 +338,23 @@ struct DistributeCollation { pov_block: PoV, } -/// Create some PoV and distribute it. -async fn distribute_collation( +async fn distribute_collation_with_receipt( virtual_overseer: &mut VirtualOverseer, test_state: &TestState, - // whether or not we expect a connection request or not. + relay_parent: Hash, should_connect: bool, + candidate: CandidateReceipt, + pov: PoV, + parent_head_data_hash: Hash, ) -> DistributeCollation { - // Now we want to distribute a `PoVBlock` - let pov_block = PoV { block_data: BlockData(vec![42, 43, 44]) }; - - let pov_hash = pov_block.hash(); - - let candidate = TestCandidateBuilder { - para_id: test_state.para_id, - relay_parent: test_state.relay_parent, - pov_hash, - ..Default::default() - } - .build(); - overseer_send( virtual_overseer, - CollatorProtocolMessage::DistributeCollation(candidate.clone(), pov_block.clone(), None), + CollatorProtocolMessage::DistributeCollation( + candidate.clone(), + parent_head_data_hash, + pov.clone(), + None, + ), ) .await; @@ -332,10 +362,10 @@ async fn distribute_collation( assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( - relay_parent, + _relay_parent, RuntimeApiRequest::AvailabilityCores(tx) )) => { - assert_eq!(relay_parent, test_state.relay_parent); + assert_eq!(relay_parent, _relay_parent); tx.send(Ok(test_state.availability_cores.clone())).unwrap(); } ); @@ -347,7 +377,7 @@ async fn distribute_collation( relay_parent, RuntimeApiRequest::SessionIndexForChild(tx), )) => { - assert_eq!(relay_parent, test_state.relay_parent); + assert_eq!(relay_parent, relay_parent); tx.send(Ok(test_state.current_session_index())).unwrap(); }, @@ -355,17 +385,17 @@ async fn distribute_collation( relay_parent, RuntimeApiRequest::SessionInfo(index, tx), )) => { - assert_eq!(relay_parent, test_state.relay_parent); + assert_eq!(relay_parent, relay_parent); assert_eq!(index, test_state.current_session_index()); tx.send(Ok(Some(test_state.session_info.clone()))).unwrap(); }, AllMessages::RuntimeApi(RuntimeApiMessage::Request( - relay_parent, + _relay_parent, RuntimeApiRequest::ValidatorGroups(tx), )) => { - assert_eq!(relay_parent, test_state.relay_parent); + assert_eq!(_relay_parent, relay_parent); tx.send(Ok(( test_state.session_info.validator_groups.clone(), test_state.group_rotation_info.clone(), @@ -389,13 +419,48 @@ async fn distribute_collation( ); } - DistributeCollation { candidate, pov_block } + DistributeCollation { candidate, pov_block: pov } +} + +/// Create some PoV and distribute it. +async fn distribute_collation( + virtual_overseer: &mut VirtualOverseer, + test_state: &TestState, + relay_parent: Hash, + // whether or not we expect a connection request or not. + should_connect: bool, +) -> DistributeCollation { + // Now we want to distribute a `PoVBlock` + let pov_block = PoV { block_data: BlockData(vec![42, 43, 44]) }; + + let pov_hash = pov_block.hash(); + let parent_head_data_hash = Hash::zero(); + + let candidate = TestCandidateBuilder { + para_id: test_state.para_id, + relay_parent, + pov_hash, + ..Default::default() + } + .build(); + + distribute_collation_with_receipt( + virtual_overseer, + test_state, + relay_parent, + should_connect, + candidate, + pov_block, + parent_head_data_hash, + ) + .await } /// Connect a peer async fn connect_peer( virtual_overseer: &mut VirtualOverseer, peer: PeerId, + version: CollationVersion, authority_id: Option, ) { overseer_send( @@ -403,7 +468,7 @@ async fn connect_peer( CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( peer.clone(), polkadot_node_network_protocol::ObservedRole::Authority, - CollationVersion::V1.into(), + version.into(), authority_id.map(|v| HashSet::from([v])), )), ) @@ -463,30 +528,65 @@ async fn expect_declare_msg( } /// Check that the next received message is a collation advertisement message. +/// +/// Expects vstaging message if `expected_candidate_hashes` is `Some`, v1 otherwise. async fn expect_advertise_collation_msg( virtual_overseer: &mut VirtualOverseer, peer: &PeerId, expected_relay_parent: Hash, + expected_candidate_hashes: Option>, ) { - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::NetworkBridgeTx( - NetworkBridgeTxMessage::SendCollationMessage( - to, - Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)), - ) - ) => { - assert_eq!(to[0], *peer); - assert_matches!( - wire_message, - protocol_v1::CollatorProtocolMessage::AdvertiseCollation( - relay_parent, - ) => { - assert_eq!(relay_parent, expected_relay_parent); + let mut candidate_hashes: Option> = + expected_candidate_hashes.map(|hashes| hashes.into_iter().collect()); + let iter_num = candidate_hashes.as_ref().map(|hashes| hashes.len()).unwrap_or(1); + + for _ in 0..iter_num { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::SendCollationMessage( + to, + wire_message, + ) + ) => { + assert_eq!(to[0], *peer); + match (candidate_hashes.as_mut(), wire_message) { + (None, Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message))) => { + assert_matches!( + wire_message, + protocol_v1::CollatorProtocolMessage::AdvertiseCollation( + relay_parent, + ) => { + assert_eq!(relay_parent, expected_relay_parent); + } + ); + }, + ( + Some(candidate_hashes), + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + wire_message, + )), + ) => { + assert_matches!( + wire_message, + protocol_vstaging::CollatorProtocolMessage::AdvertiseCollation { + relay_parent, + candidate_hash, + .. + } => { + assert_eq!(relay_parent, expected_relay_parent); + assert!(candidate_hashes.contains(&candidate_hash)); + + // Drop the hash we've already seen. + candidate_hashes.remove(&candidate_hash); + } + ); + }, + _ => panic!("Invalid advertisement"), } - ); - } - ); + } + ); + } } /// Send a message that the given peer's view changed. @@ -513,19 +613,27 @@ fn advertise_and_send_collation() { test_harness(local_peer_id, collator_pair, |test_harness| async move { let mut virtual_overseer = test_harness.virtual_overseer; - let mut req_cfg = test_harness.req_cfg; + let mut req_v1_cfg = test_harness.req_v1_cfg; + let req_vstaging_cfg = test_harness.req_vstaging_cfg; setup_system(&mut virtual_overseer, &test_state).await; let DistributeCollation { candidate, pov_block } = - distribute_collation(&mut virtual_overseer, &test_state, true).await; + distribute_collation(&mut virtual_overseer, &test_state, test_state.relay_parent, true) + .await; for (val, peer) in test_state .current_group_validator_authority_ids() .into_iter() .zip(test_state.current_group_validator_peer_ids()) { - connect_peer(&mut virtual_overseer, peer.clone(), Some(val.clone())).await; + connect_peer( + &mut virtual_overseer, + peer.clone(), + CollationVersion::V1, + Some(val.clone()), + ) + .await; } // We declare to the connected validators that we are a collator. @@ -542,17 +650,18 @@ fn advertise_and_send_collation() { // The peer is interested in a leaf that we have a collation for; // advertise it. - expect_advertise_collation_msg(&mut virtual_overseer, &peer, test_state.relay_parent).await; + expect_advertise_collation_msg(&mut virtual_overseer, &peer, test_state.relay_parent, None) + .await; // Request a collation. let (pending_response, rx) = oneshot::channel(); - req_cfg + req_v1_cfg .inbound_queue .as_mut() .unwrap() .send(RawIncomingRequest { peer, - payload: CollationFetchingRequest { + payload: request_v1::CollationFetchingRequest { relay_parent: test_state.relay_parent, para_id: test_state.para_id, } @@ -565,13 +674,13 @@ fn advertise_and_send_collation() { { let (pending_response, rx) = oneshot::channel(); - req_cfg + req_v1_cfg .inbound_queue .as_mut() .unwrap() .send(RawIncomingRequest { peer, - payload: CollationFetchingRequest { + payload: request_v1::CollationFetchingRequest { relay_parent: test_state.relay_parent, para_id: test_state.para_id, } @@ -596,8 +705,8 @@ fn advertise_and_send_collation() { assert_matches!( rx.await, Ok(full_response) => { - let CollationFetchingResponse::Collation(receipt, pov): CollationFetchingResponse - = CollationFetchingResponse::decode( + let request_v1::CollationFetchingResponse::Collation(receipt, pov): request_v1::CollationFetchingResponse + = request_v1::CollationFetchingResponse::decode( &mut full_response.result .expect("We should have a proper answer").as_ref() ) @@ -615,13 +724,13 @@ fn advertise_and_send_collation() { // Re-request a collation. let (pending_response, rx) = oneshot::channel(); - req_cfg + req_v1_cfg .inbound_queue .as_mut() .unwrap() .send(RawIncomingRequest { peer, - payload: CollationFetchingRequest { + payload: request_v1::CollationFetchingRequest { relay_parent: old_relay_parent, para_id: test_state.para_id, } @@ -635,7 +744,8 @@ fn advertise_and_send_collation() { assert!(overseer_recv_with_timeout(&mut virtual_overseer, TIMEOUT).await.is_none()); - distribute_collation(&mut virtual_overseer, &test_state, true).await; + distribute_collation(&mut virtual_overseer, &test_state, test_state.relay_parent, true) + .await; // Send info about peer's view. overseer_send( @@ -647,8 +757,87 @@ fn advertise_and_send_collation() { ) .await; - expect_advertise_collation_msg(&mut virtual_overseer, &peer, test_state.relay_parent).await; - TestHarness { virtual_overseer, req_cfg } + expect_advertise_collation_msg(&mut virtual_overseer, &peer, test_state.relay_parent, None) + .await; + TestHarness { virtual_overseer, req_v1_cfg, req_vstaging_cfg } + }); +} + +/// Tests that collator side works with vstaging network protocol +/// before async backing is enabled. +#[test] +fn advertise_collation_vstaging_protocol() { + let test_state = TestState::default(); + let local_peer_id = test_state.local_peer_id.clone(); + let collator_pair = test_state.collator_pair.clone(); + + test_harness(local_peer_id, collator_pair, |mut test_harness| async move { + let virtual_overseer = &mut test_harness.virtual_overseer; + + setup_system(virtual_overseer, &test_state).await; + + let DistributeCollation { candidate, .. } = + distribute_collation(virtual_overseer, &test_state, test_state.relay_parent, true) + .await; + + let validators = test_state.current_group_validator_authority_ids(); + assert!(validators.len() >= 2); + let peer_ids = test_state.current_group_validator_peer_ids(); + + // Connect first peer with v1. + connect_peer( + virtual_overseer, + peer_ids[0], + CollationVersion::V1, + Some(validators[0].clone()), + ) + .await; + // The rest with vstaging. + for (val, peer) in validators.iter().zip(peer_ids.iter()).skip(1) { + connect_peer( + virtual_overseer, + peer.clone(), + CollationVersion::VStaging, + Some(val.clone()), + ) + .await; + } + + // Declare messages. + expect_declare_msg(virtual_overseer, &test_state, &peer_ids[0]).await; + for peer_id in peer_ids.iter().skip(1) { + prospective_parachains::expect_declare_msg_vstaging( + virtual_overseer, + &test_state, + &peer_id, + ) + .await; + } + + // Send info about peers view. + for peer in peer_ids.iter() { + send_peer_view_change(virtual_overseer, peer, vec![test_state.relay_parent]).await; + } + + // Versioned advertisements work. + expect_advertise_collation_msg( + virtual_overseer, + &peer_ids[0], + test_state.relay_parent, + None, + ) + .await; + for peer_id in peer_ids.iter().skip(1) { + expect_advertise_collation_msg( + virtual_overseer, + peer_id, + test_state.relay_parent, + Some(vec![candidate.hash()]), // This is `Some`, advertisement is vstaging. + ) + .await; + } + + test_harness }); } @@ -688,7 +877,13 @@ fn collators_declare_to_connected_peers() { setup_system(&mut test_harness.virtual_overseer, &test_state).await; // A validator connected to us - connect_peer(&mut test_harness.virtual_overseer, peer.clone(), Some(validator_id)).await; + connect_peer( + &mut test_harness.virtual_overseer, + peer.clone(), + CollationVersion::V1, + Some(validator_id), + ) + .await; expect_declare_msg(&mut test_harness.virtual_overseer, &test_state, &peer).await; test_harness }) @@ -712,10 +907,12 @@ fn collations_are_only_advertised_to_validators_with_correct_view() { setup_system(virtual_overseer, &test_state).await; // A validator connected to us - connect_peer(virtual_overseer, peer.clone(), Some(validator_id)).await; + connect_peer(virtual_overseer, peer.clone(), CollationVersion::V1, Some(validator_id)) + .await; // Connect the second validator - connect_peer(virtual_overseer, peer2.clone(), Some(validator_id2)).await; + connect_peer(virtual_overseer, peer2.clone(), CollationVersion::V1, Some(validator_id2)) + .await; expect_declare_msg(virtual_overseer, &test_state, &peer).await; expect_declare_msg(virtual_overseer, &test_state, &peer2).await; @@ -723,15 +920,17 @@ fn collations_are_only_advertised_to_validators_with_correct_view() { // And let it tell us that it is has the same view. send_peer_view_change(virtual_overseer, &peer2, vec![test_state.relay_parent]).await; - distribute_collation(virtual_overseer, &test_state, true).await; + distribute_collation(virtual_overseer, &test_state, test_state.relay_parent, true).await; - expect_advertise_collation_msg(virtual_overseer, &peer2, test_state.relay_parent).await; + expect_advertise_collation_msg(virtual_overseer, &peer2, test_state.relay_parent, None) + .await; // The other validator announces that it changed its view. send_peer_view_change(virtual_overseer, &peer, vec![test_state.relay_parent]).await; // After changing the view we should receive the advertisement - expect_advertise_collation_msg(virtual_overseer, &peer, test_state.relay_parent).await; + expect_advertise_collation_msg(virtual_overseer, &peer, test_state.relay_parent, None) + .await; test_harness }) } @@ -754,29 +953,32 @@ fn collate_on_two_different_relay_chain_blocks() { setup_system(virtual_overseer, &test_state).await; // A validator connected to us - connect_peer(virtual_overseer, peer.clone(), Some(validator_id)).await; + connect_peer(virtual_overseer, peer.clone(), CollationVersion::V1, Some(validator_id)) + .await; // Connect the second validator - connect_peer(virtual_overseer, peer2.clone(), Some(validator_id2)).await; + connect_peer(virtual_overseer, peer2.clone(), CollationVersion::V1, Some(validator_id2)) + .await; expect_declare_msg(virtual_overseer, &test_state, &peer).await; expect_declare_msg(virtual_overseer, &test_state, &peer2).await; - distribute_collation(virtual_overseer, &test_state, true).await; + distribute_collation(virtual_overseer, &test_state, test_state.relay_parent, true).await; let old_relay_parent = test_state.relay_parent; // Advance to a new round, while informing the subsystem that the old and the new relay parent are active. test_state.advance_to_new_round(virtual_overseer, true).await; - distribute_collation(virtual_overseer, &test_state, true).await; + distribute_collation(virtual_overseer, &test_state, test_state.relay_parent, true).await; send_peer_view_change(virtual_overseer, &peer, vec![old_relay_parent]).await; - expect_advertise_collation_msg(virtual_overseer, &peer, old_relay_parent).await; + expect_advertise_collation_msg(virtual_overseer, &peer, old_relay_parent, None).await; send_peer_view_change(virtual_overseer, &peer2, vec![test_state.relay_parent]).await; - expect_advertise_collation_msg(virtual_overseer, &peer2, test_state.relay_parent).await; + expect_advertise_collation_msg(virtual_overseer, &peer2, test_state.relay_parent, None) + .await; test_harness }) } @@ -796,17 +998,25 @@ fn validator_reconnect_does_not_advertise_a_second_time() { setup_system(virtual_overseer, &test_state).await; // A validator connected to us - connect_peer(virtual_overseer, peer.clone(), Some(validator_id.clone())).await; + connect_peer( + virtual_overseer, + peer.clone(), + CollationVersion::V1, + Some(validator_id.clone()), + ) + .await; expect_declare_msg(virtual_overseer, &test_state, &peer).await; - distribute_collation(virtual_overseer, &test_state, true).await; + distribute_collation(virtual_overseer, &test_state, test_state.relay_parent, true).await; send_peer_view_change(virtual_overseer, &peer, vec![test_state.relay_parent]).await; - expect_advertise_collation_msg(virtual_overseer, &peer, test_state.relay_parent).await; + expect_advertise_collation_msg(virtual_overseer, &peer, test_state.relay_parent, None) + .await; // Disconnect and reconnect directly disconnect_peer(virtual_overseer, peer.clone()).await; - connect_peer(virtual_overseer, peer.clone(), Some(validator_id)).await; + connect_peer(virtual_overseer, peer.clone(), CollationVersion::V1, Some(validator_id)) + .await; expect_declare_msg(virtual_overseer, &test_state, &peer).await; send_peer_view_change(virtual_overseer, &peer, vec![test_state.relay_parent]).await; @@ -832,7 +1042,8 @@ fn collators_reject_declare_messages() { setup_system(virtual_overseer, &test_state).await; // A validator connected to us - connect_peer(virtual_overseer, peer.clone(), Some(validator_id)).await; + connect_peer(virtual_overseer, peer.clone(), CollationVersion::V1, Some(validator_id)) + .await; expect_declare_msg(virtual_overseer, &test_state, &peer).await; overseer_send( @@ -879,19 +1090,21 @@ where test_harness(local_peer_id, collator_pair, |mut test_harness| async move { let virtual_overseer = &mut test_harness.virtual_overseer; - let req_cfg = &mut test_harness.req_cfg; + let req_cfg = &mut test_harness.req_v1_cfg; setup_system(virtual_overseer, &test_state).await; let DistributeCollation { candidate, pov_block } = - distribute_collation(virtual_overseer, &test_state, true).await; + distribute_collation(virtual_overseer, &test_state, test_state.relay_parent, true) + .await; for (val, peer) in test_state .current_group_validator_authority_ids() .into_iter() .zip(test_state.current_group_validator_peer_ids()) { - connect_peer(virtual_overseer, peer.clone(), Some(val.clone())).await; + connect_peer(virtual_overseer, peer.clone(), CollationVersion::V1, Some(val.clone())) + .await; } // We declare to the connected validators that we are a collator. @@ -910,10 +1123,20 @@ where // The peer is interested in a leaf that we have a collation for; // advertise it. - expect_advertise_collation_msg(virtual_overseer, &validator_0, test_state.relay_parent) - .await; - expect_advertise_collation_msg(virtual_overseer, &validator_1, test_state.relay_parent) - .await; + expect_advertise_collation_msg( + virtual_overseer, + &validator_0, + test_state.relay_parent, + None, + ) + .await; + expect_advertise_collation_msg( + virtual_overseer, + &validator_1, + test_state.relay_parent, + None, + ) + .await; // Request a collation. let (pending_response, rx) = oneshot::channel(); @@ -923,7 +1146,7 @@ where .unwrap() .send(RawIncomingRequest { peer: validator_0, - payload: CollationFetchingRequest { + payload: request_v1::CollationFetchingRequest { relay_parent: test_state.relay_parent, para_id: test_state.para_id, } @@ -937,8 +1160,8 @@ where let feedback_tx = assert_matches!( rx.await, Ok(full_response) => { - let CollationFetchingResponse::Collation(receipt, pov): CollationFetchingResponse - = CollationFetchingResponse::decode( + let request_v1::CollationFetchingResponse::Collation(receipt, pov): request_v1::CollationFetchingResponse + = request_v1::CollationFetchingResponse::decode( &mut full_response.result .expect("We should have a proper answer").as_ref() ) @@ -958,7 +1181,7 @@ where .unwrap() .send(RawIncomingRequest { peer: validator_1, - payload: CollationFetchingRequest { + payload: request_v1::CollationFetchingRequest { relay_parent: test_state.relay_parent, para_id: test_state.para_id, } @@ -974,8 +1197,8 @@ where assert_matches!( rx.await, Ok(full_response) => { - let CollationFetchingResponse::Collation(receipt, pov): CollationFetchingResponse - = CollationFetchingResponse::decode( + let request_v1::CollationFetchingResponse::Collation(receipt, pov): request_v1::CollationFetchingResponse + = request_v1::CollationFetchingResponse::decode( &mut full_response.result .expect("We should have a proper answer").as_ref() ) @@ -999,7 +1222,8 @@ fn connect_to_buffered_groups() { test_harness(local_peer_id, collator_pair, |test_harness| async move { let mut virtual_overseer = test_harness.virtual_overseer; - let mut req_cfg = test_harness.req_cfg; + let mut req_cfg = test_harness.req_v1_cfg; + let req_vstaging_cfg = test_harness.req_vstaging_cfg; setup_system(&mut virtual_overseer, &test_state).await; @@ -1007,7 +1231,8 @@ fn connect_to_buffered_groups() { let peers_a = test_state.current_group_validator_peer_ids(); assert!(group_a.len() > 1); - distribute_collation(&mut virtual_overseer, &test_state, false).await; + distribute_collation(&mut virtual_overseer, &test_state, test_state.relay_parent, false) + .await; assert_matches!( overseer_recv(&mut virtual_overseer).await, @@ -1021,7 +1246,13 @@ fn connect_to_buffered_groups() { let head_a = test_state.relay_parent; for (val, peer) in group_a.iter().zip(&peers_a) { - connect_peer(&mut virtual_overseer, peer.clone(), Some(val.clone())).await; + connect_peer( + &mut virtual_overseer, + peer.clone(), + CollationVersion::V1, + Some(val.clone()), + ) + .await; } for peer_id in &peers_a { @@ -1031,7 +1262,7 @@ fn connect_to_buffered_groups() { // Update views. for peed_id in &peers_a { send_peer_view_change(&mut virtual_overseer, peed_id, vec![head_a]).await; - expect_advertise_collation_msg(&mut virtual_overseer, peed_id, head_a).await; + expect_advertise_collation_msg(&mut virtual_overseer, peed_id, head_a, None).await; } let peer = peers_a[0]; @@ -1043,7 +1274,7 @@ fn connect_to_buffered_groups() { .unwrap() .send(RawIncomingRequest { peer, - payload: CollationFetchingRequest { + payload: request_v1::CollationFetchingRequest { relay_parent: head_a, para_id: test_state.para_id, } @@ -1055,14 +1286,17 @@ fn connect_to_buffered_groups() { assert_matches!( rx.await, Ok(full_response) => { - let CollationFetchingResponse::Collation(..): CollationFetchingResponse = - CollationFetchingResponse::decode( + let request_v1::CollationFetchingResponse::Collation(..) = + request_v1::CollationFetchingResponse::decode( &mut full_response.result.expect("We should have a proper answer").as_ref(), ) .expect("Decoding should work"); } ); + // Let the subsystem process process the collation event. + test_helpers::Yield::new().await; + test_state.advance_to_new_round(&mut virtual_overseer, true).await; test_state.group_rotation_info = test_state.group_rotation_info.bump_rotation(); @@ -1071,7 +1305,8 @@ fn connect_to_buffered_groups() { assert_ne!(head_a, head_b); assert_ne!(group_a, group_b); - distribute_collation(&mut virtual_overseer, &test_state, false).await; + distribute_collation(&mut virtual_overseer, &test_state, test_state.relay_parent, false) + .await; // Should be connected to both groups except for the validator that fetched advertised // collation. @@ -1088,6 +1323,6 @@ fn connect_to_buffered_groups() { } ); - TestHarness { virtual_overseer, req_cfg } + TestHarness { virtual_overseer, req_v1_cfg: req_cfg, req_vstaging_cfg } }); } diff --git a/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs b/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs new file mode 100644 index 000000000000..d98db5b8eb82 --- /dev/null +++ b/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs @@ -0,0 +1,562 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the collator side with enabled prospective parachains. + +use super::*; + +use polkadot_node_subsystem::messages::{ChainApiMessage, ProspectiveParachainsMessage}; +use polkadot_primitives::v2::{Header, OccupiedCore}; + +const ALLOWED_ANCESTRY: u32 = 3; + +fn get_parent_hash(hash: Hash) -> Hash { + Hash::from_low_u64_be(hash.to_low_u64_be() + 1) +} + +/// Handle a view update. +async fn update_view( + virtual_overseer: &mut VirtualOverseer, + test_state: &TestState, + new_view: Vec<(Hash, u32)>, // Hash and block number. + activated: u8, // How many new heads does this update contain? +) { + let new_view: HashMap = HashMap::from_iter(new_view); + + let our_view = + OurView::new(new_view.keys().map(|hash| (*hash, Arc::new(jaeger::Span::Disabled))), 0); + + overseer_send( + virtual_overseer, + CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange(our_view)), + ) + .await; + + let mut next_overseer_message = None; + for _ in 0..activated { + let (leaf_hash, leaf_number) = assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + parent, + RuntimeApiRequest::Version(tx), + )) => { + tx.send(Ok(RuntimeApiRequest::VALIDITY_CONSTRAINTS)).unwrap(); + (parent, new_view.get(&parent).copied().expect("Unknown parent requested")) + } + ); + + let min_number = leaf_number.saturating_sub(ALLOWED_ANCESTRY); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::GetMinimumRelayParents(parent, tx), + ) if parent == leaf_hash => { + tx.send(vec![(test_state.para_id, min_number)]).unwrap(); + } + ); + + let ancestry_len = leaf_number + 1 - min_number; + let ancestry_hashes = std::iter::successors(Some(leaf_hash), |h| Some(get_parent_hash(*h))) + .take(ancestry_len as usize); + let ancestry_numbers = (min_number..=leaf_number).rev(); + let mut ancestry_iter = ancestry_hashes.clone().zip(ancestry_numbers).peekable(); + + loop { + let (hash, number) = match ancestry_iter.next() { + Some((hash, number)) => (hash, number), + None => break, + }; + + // May be `None` for the last element. + let parent_hash = + ancestry_iter.peek().map(|(h, _)| *h).unwrap_or_else(|| get_parent_hash(hash)); + + let msg = match next_overseer_message.take() { + Some(msg) => Some(msg), + None => + overseer_recv_with_timeout(virtual_overseer, Duration::from_millis(50)).await, + }; + + let msg = match msg { + Some(msg) => msg, + None => { + // We're done. + return + }, + }; + + if !matches!( + &msg, + AllMessages::ChainApi(ChainApiMessage::BlockHeader(_hash, ..)) + if *_hash == hash + ) { + // Ancestry has already been cached for this leaf. + next_overseer_message.replace(msg); + break + } + + assert_matches!( + msg, + AllMessages::ChainApi(ChainApiMessage::BlockHeader(.., tx)) => { + let header = Header { + parent_hash, + number, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + }; + + tx.send(Ok(Some(header))).unwrap(); + } + ); + } + } +} + +/// Check that the next received message is a `Declare` message. +pub(super) async fn expect_declare_msg_vstaging( + virtual_overseer: &mut VirtualOverseer, + test_state: &TestState, + peer: &PeerId, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendCollationMessage( + to, + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + wire_message, + )), + )) => { + assert_eq!(to[0], *peer); + assert_matches!( + wire_message, + protocol_vstaging::CollatorProtocolMessage::Declare( + collator_id, + para_id, + signature, + ) => { + assert!(signature.verify( + &*protocol_vstaging::declare_signature_payload(&test_state.local_peer_id), + &collator_id), + ); + assert_eq!(collator_id, test_state.collator_pair.public()); + assert_eq!(para_id, test_state.para_id); + } + ); + } + ); +} + +/// Test that a collator distributes a collation from the allowed ancestry +/// to correct validators group. +#[test] +fn distribute_collation_from_implicit_view() { + let head_a = Hash::from_low_u64_be(126); + let head_a_num: u32 = 66; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 64; + + // Grandparent of head `b`. + let head_c = Hash::from_low_u64_be(130); + let head_c_num = 62; + + let group_rotation_info = GroupRotationInfo { + session_start_block: head_c_num - 2, + group_rotation_frequency: 3, + now: head_c_num, + }; + + let mut test_state = TestState::default(); + test_state.group_rotation_info = group_rotation_info; + + let local_peer_id = test_state.local_peer_id; + let collator_pair = test_state.collator_pair.clone(); + + test_harness(local_peer_id, collator_pair, |mut test_harness| async move { + let virtual_overseer = &mut test_harness.virtual_overseer; + + // Set collating para id. + overseer_send(virtual_overseer, CollatorProtocolMessage::CollateOn(test_state.para_id)) + .await; + // Activated leaf is `b`, but the collation will be based on `c`. + update_view(virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + + let validator_peer_ids = test_state.current_group_validator_peer_ids(); + for (val, peer) in test_state + .current_group_validator_authority_ids() + .into_iter() + .zip(validator_peer_ids.clone()) + { + connect_peer( + virtual_overseer, + peer.clone(), + CollationVersion::VStaging, + Some(val.clone()), + ) + .await; + } + + // Collator declared itself to each peer. + for peer_id in &validator_peer_ids { + expect_declare_msg_vstaging(virtual_overseer, &test_state, peer_id).await; + } + + let pov = PoV { block_data: BlockData(vec![1, 2, 3]) }; + let parent_head_data_hash = Hash::repeat_byte(0xAA); + let candidate = TestCandidateBuilder { + para_id: test_state.para_id, + relay_parent: head_c, + pov_hash: pov.hash(), + ..Default::default() + } + .build(); + let DistributeCollation { candidate, pov_block: _ } = distribute_collation_with_receipt( + virtual_overseer, + &test_state, + head_c, + false, // Check the group manually. + candidate, + pov, + parent_head_data_hash, + ) + .await; + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::ConnectToValidators { validator_ids, .. } + ) => { + let expected_validators = test_state.current_group_validator_authority_ids(); + + assert_eq!(expected_validators, validator_ids); + } + ); + + let candidate_hash = candidate.hash(); + + // Update peer views. + for peed_id in &validator_peer_ids { + send_peer_view_change(virtual_overseer, peed_id, vec![head_b]).await; + expect_advertise_collation_msg( + virtual_overseer, + peed_id, + head_c, + Some(vec![candidate_hash]), + ) + .await; + } + + // Head `c` goes out of view. + // Build a different candidate for this relay parent and attempt to distribute it. + update_view(virtual_overseer, &test_state, vec![(head_a, head_a_num)], 1).await; + + let pov = PoV { block_data: BlockData(vec![4, 5, 6]) }; + let parent_head_data_hash = Hash::repeat_byte(0xBB); + let candidate = TestCandidateBuilder { + para_id: test_state.para_id, + relay_parent: head_c, + pov_hash: pov.hash(), + ..Default::default() + } + .build(); + overseer_send( + virtual_overseer, + CollatorProtocolMessage::DistributeCollation( + candidate.clone(), + parent_head_data_hash, + pov.clone(), + None, + ), + ) + .await; + + // Parent out of view, nothing happens. + assert!(overseer_recv_with_timeout(virtual_overseer, Duration::from_millis(100)) + .await + .is_none()); + + test_harness + }) +} + +/// Tests that collator can distribute up to `MAX_CANDIDATE_DEPTH + 1` candidates +/// per relay parent. +#[test] +fn distribute_collation_up_to_limit() { + let test_state = TestState::default(); + + let local_peer_id = test_state.local_peer_id; + let collator_pair = test_state.collator_pair.clone(); + + test_harness(local_peer_id, collator_pair, |mut test_harness| async move { + let virtual_overseer = &mut test_harness.virtual_overseer; + + let head_a = Hash::from_low_u64_be(128); + let head_a_num: u32 = 64; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(130); + + // Set collating para id. + overseer_send(virtual_overseer, CollatorProtocolMessage::CollateOn(test_state.para_id)) + .await; + // Activated leaf is `a`, but the collation will be based on `b`. + update_view(virtual_overseer, &test_state, vec![(head_a, head_a_num)], 1).await; + + for i in 0..(MAX_CANDIDATE_DEPTH + 1) { + let pov = PoV { block_data: BlockData(vec![i as u8]) }; + let parent_head_data_hash = Hash::repeat_byte(0xAA); + let candidate = TestCandidateBuilder { + para_id: test_state.para_id, + relay_parent: head_b, + pov_hash: pov.hash(), + ..Default::default() + } + .build(); + distribute_collation_with_receipt( + virtual_overseer, + &test_state, + head_b, + true, + candidate, + pov, + parent_head_data_hash, + ) + .await; + } + + let pov = PoV { block_data: BlockData(vec![10, 12, 6]) }; + let parent_head_data_hash = Hash::repeat_byte(0xBB); + let candidate = TestCandidateBuilder { + para_id: test_state.para_id, + relay_parent: head_b, + pov_hash: pov.hash(), + ..Default::default() + } + .build(); + overseer_send( + virtual_overseer, + CollatorProtocolMessage::DistributeCollation( + candidate.clone(), + parent_head_data_hash, + pov.clone(), + None, + ), + ) + .await; + + // Limit has been reached. + assert!(overseer_recv_with_timeout(virtual_overseer, Duration::from_millis(100)) + .await + .is_none()); + + test_harness + }) +} + +/// Tests that collator correctly handles peer V2 requests. +#[test] +fn advertise_and_send_collation_by_hash() { + let test_state = TestState::default(); + + let local_peer_id = test_state.local_peer_id; + let collator_pair = test_state.collator_pair.clone(); + + test_harness(local_peer_id, collator_pair, |test_harness| async move { + let mut virtual_overseer = test_harness.virtual_overseer; + let req_v1_cfg = test_harness.req_v1_cfg; + let mut req_vstaging_cfg = test_harness.req_vstaging_cfg; + + let head_a = Hash::from_low_u64_be(128); + let head_a_num: u32 = 64; + + // Parent of head `a`. + let head_b = Hash::from_low_u64_be(129); + let head_b_num: u32 = 63; + + // Set collating para id. + overseer_send( + &mut virtual_overseer, + CollatorProtocolMessage::CollateOn(test_state.para_id), + ) + .await; + update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view(&mut virtual_overseer, &test_state, vec![(head_a, head_a_num)], 1).await; + + let candidates: Vec<_> = (0..2) + .map(|i| { + let pov = PoV { block_data: BlockData(vec![i as u8]) }; + let candidate = TestCandidateBuilder { + para_id: test_state.para_id, + relay_parent: head_b, + pov_hash: pov.hash(), + ..Default::default() + } + .build(); + (candidate, pov) + }) + .collect(); + for (candidate, pov) in &candidates { + distribute_collation_with_receipt( + &mut virtual_overseer, + &test_state, + head_b, + true, + candidate.clone(), + pov.clone(), + Hash::zero(), + ) + .await; + } + + let peer = test_state.validator_peer_id[0].clone(); + let validator_id = test_state.current_group_validator_authority_ids()[0].clone(); + connect_peer( + &mut virtual_overseer, + peer.clone(), + CollationVersion::VStaging, + Some(validator_id.clone()), + ) + .await; + expect_declare_msg_vstaging(&mut virtual_overseer, &test_state, &peer).await; + + // Head `b` is not a leaf, but both advertisements are still relevant. + send_peer_view_change(&mut virtual_overseer, &peer, vec![head_b]).await; + let hashes: Vec<_> = candidates.iter().map(|(candidate, _)| candidate.hash()).collect(); + expect_advertise_collation_msg(&mut virtual_overseer, &peer, head_b, Some(hashes)).await; + + for (candidate, pov_block) in candidates { + let (pending_response, rx) = oneshot::channel(); + req_vstaging_cfg + .inbound_queue + .as_mut() + .unwrap() + .send(RawIncomingRequest { + peer, + payload: request_vstaging::CollationFetchingRequest { + relay_parent: head_b, + para_id: test_state.para_id, + candidate_hash: candidate.hash(), + } + .encode(), + pending_response, + }) + .await + .unwrap(); + + assert_matches!( + rx.await, + Ok(full_response) => { + // Response is the same for vstaging. + let request_v1::CollationFetchingResponse::Collation(receipt, pov): request_v1::CollationFetchingResponse + = request_v1::CollationFetchingResponse::decode( + &mut full_response.result + .expect("We should have a proper answer").as_ref() + ) + .expect("Decoding should work"); + assert_eq!(receipt, candidate); + assert_eq!(pov, pov_block); + } + ); + } + + TestHarness { virtual_overseer, req_v1_cfg, req_vstaging_cfg } + }) +} + +/// Tests that collator distributes collation built on top of occupied core. +#[test] +fn advertise_core_occupied() { + let mut test_state = TestState::default(); + let candidate = + TestCandidateBuilder { para_id: test_state.para_id, ..Default::default() }.build(); + test_state.availability_cores[0] = CoreState::Occupied(OccupiedCore { + next_up_on_available: None, + occupied_since: 0, + time_out_at: 0, + next_up_on_time_out: None, + availability: BitVec::default(), + group_responsible: GroupIndex(0), + candidate_hash: candidate.hash(), + candidate_descriptor: candidate.descriptor, + }); + + let local_peer_id = test_state.local_peer_id; + let collator_pair = test_state.collator_pair.clone(); + + test_harness(local_peer_id, collator_pair, |mut test_harness| async move { + let virtual_overseer = &mut test_harness.virtual_overseer; + + let head_a = Hash::from_low_u64_be(128); + let head_a_num: u32 = 64; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(130); + + // Set collating para id. + overseer_send(virtual_overseer, CollatorProtocolMessage::CollateOn(test_state.para_id)) + .await; + // Activated leaf is `a`, but the collation will be based on `b`. + update_view(virtual_overseer, &test_state, vec![(head_a, head_a_num)], 1).await; + + let pov = PoV { block_data: BlockData(vec![1, 2, 3]) }; + let candidate = TestCandidateBuilder { + para_id: test_state.para_id, + relay_parent: head_b, + pov_hash: pov.hash(), + ..Default::default() + } + .build(); + let candidate_hash = candidate.hash(); + distribute_collation_with_receipt( + virtual_overseer, + &test_state, + head_b, + true, + candidate, + pov, + Hash::zero(), + ) + .await; + + let validators = test_state.current_group_validator_authority_ids(); + let peer_ids = test_state.current_group_validator_peer_ids(); + + connect_peer( + virtual_overseer, + peer_ids[0], + CollationVersion::VStaging, + Some(validators[0].clone()), + ) + .await; + expect_declare_msg_vstaging(virtual_overseer, &test_state, &peer_ids[0]).await; + // Peer is aware of the leaf. + send_peer_view_change(virtual_overseer, &peer_ids[0], vec![head_a]).await; + + // Collation is advertised. + expect_advertise_collation_msg( + virtual_overseer, + &peer_ids[0], + head_b, + Some(vec![candidate_hash]), + ) + .await; + + test_harness + }) +} diff --git a/node/network/collator-protocol/src/collator_side/validators_buffer.rs b/node/network/collator-protocol/src/collator_side/validators_buffer.rs index 5bb31c72d6c5..9f1817aa2051 100644 --- a/node/network/collator-protocol/src/collator_side/validators_buffer.rs +++ b/node/network/collator-protocol/src/collator_side/validators_buffer.rs @@ -23,21 +23,27 @@ //! We keep a simple FIFO buffer of N validator groups and a bitvec for each advertisement, //! 1 indicating we want to be connected to i-th validator in a buffer, 0 otherwise. //! -//! The bit is set to 1 for the whole **group** whenever it's inserted into the buffer. Given a relay -//! parent, one can reset a bit back to 0 for particular **validator**. For example, if a collation +//! The bit is set to 1 for the whole **group** whenever it's inserted into the buffer. Given a candidate +//! hash, one can reset a bit back to 0 for particular **validator**. For example, if a collation //! was fetched or some timeout has been hit. //! //! The bitwise OR over known advertisements gives us validators indices for connection request. use std::{ collections::{HashMap, VecDeque}, + future::Future, num::NonZeroUsize, ops::Range, + pin::Pin, + task::{Context, Poll}, + time::Duration, }; use bitvec::{bitvec, vec::BitVec}; +use futures::FutureExt; -use polkadot_primitives::v2::{AuthorityDiscoveryId, GroupIndex, Hash, SessionIndex}; +use polkadot_node_network_protocol::PeerId; +use polkadot_primitives::v2::{AuthorityDiscoveryId, CandidateHash, GroupIndex, SessionIndex}; /// The ring buffer stores at most this many unique validator groups. /// @@ -66,9 +72,9 @@ pub struct ValidatorGroupsBuffer { group_infos: VecDeque, /// Continuous buffer of validators discovery keys. validators: VecDeque, - /// Mapping from relay-parent to bit-vectors with bits for all `validators`. + /// Mapping from candidate hashes to bit-vectors with bits for all `validators`. /// Invariants kept: All bit-vectors are guaranteed to have the same size. - should_be_connected: HashMap, + should_be_connected: HashMap, /// Buffer capacity, limits the number of **groups** tracked. cap: NonZeroUsize, } @@ -107,7 +113,7 @@ impl ValidatorGroupsBuffer { /// of the buffer. pub fn note_collation_advertised( &mut self, - relay_parent: Hash, + candidate_hash: CandidateHash, session_index: SessionIndex, group_index: GroupIndex, validators: &[AuthorityDiscoveryId], @@ -121,19 +127,19 @@ impl ValidatorGroupsBuffer { }) { Some((idx, group)) => { let group_start_idx = self.group_lengths_iter().take(idx).sum(); - self.set_bits(relay_parent, group_start_idx..(group_start_idx + group.len)); + self.set_bits(candidate_hash, group_start_idx..(group_start_idx + group.len)); }, - None => self.push(relay_parent, session_index, group_index, validators), + None => self.push(candidate_hash, session_index, group_index, validators), } } /// Note that a validator is no longer interested in a given relay parent. pub fn reset_validator_interest( &mut self, - relay_parent: Hash, + candidate_hash: CandidateHash, authority_id: &AuthorityDiscoveryId, ) { - let bits = match self.should_be_connected.get_mut(&relay_parent) { + let bits = match self.should_be_connected.get_mut(&candidate_hash) { Some(bits) => bits, None => return, }; @@ -145,17 +151,12 @@ impl ValidatorGroupsBuffer { } } - /// Remove relay parent from the buffer. + /// Remove advertised candidate from the buffer. /// /// The buffer will no longer track which validators are interested in a corresponding /// advertisement. - pub fn remove_relay_parent(&mut self, relay_parent: &Hash) { - self.should_be_connected.remove(relay_parent); - } - - /// Removes all advertisements from the buffer. - pub fn clear_advertisements(&mut self) { - self.should_be_connected.clear(); + pub fn remove_candidate(&mut self, candidate_hash: &CandidateHash) { + self.should_be_connected.remove(candidate_hash); } /// Pushes a new group to the buffer along with advertisement, setting all validators @@ -164,7 +165,7 @@ impl ValidatorGroupsBuffer { /// If the buffer is full, drops group from the tail. fn push( &mut self, - relay_parent: Hash, + candidate_hash: CandidateHash, session_index: SessionIndex, group_index: GroupIndex, validators: &[AuthorityDiscoveryId], @@ -193,17 +194,17 @@ impl ValidatorGroupsBuffer { self.should_be_connected .values_mut() .for_each(|bits| bits.resize(new_len, false)); - self.set_bits(relay_parent, group_start_idx..(group_start_idx + validators.len())); + self.set_bits(candidate_hash, group_start_idx..(group_start_idx + validators.len())); } /// Sets advertisement bits to 1 in a given range (usually corresponding to some group). /// If the relay parent is unknown, inserts 0-initialized bitvec first. /// /// The range must be ensured to be within bounds. - fn set_bits(&mut self, relay_parent: Hash, range: Range) { + fn set_bits(&mut self, candidate_hash: CandidateHash, range: Range) { let bits = self .should_be_connected - .entry(relay_parent) + .entry(candidate_hash) .or_insert_with(|| bitvec![0; self.validators.len()]); bits[range].fill(true); @@ -217,9 +218,40 @@ impl ValidatorGroupsBuffer { } } +/// A timeout for resetting validators' interests in collations. +pub const RESET_INTEREST_TIMEOUT: Duration = Duration::from_secs(6); + +/// A future that returns a candidate hash along with validator discovery +/// keys once a timeout hit. +/// +/// If a validator doesn't manage to fetch a collation within this timeout +/// we should reset its interest in this advertisement in a buffer. For example, +/// when the PoV was already requested from another peer. +pub struct ResetInterestTimeout { + fut: futures_timer::Delay, + candidate_hash: CandidateHash, + peer_id: PeerId, +} + +impl ResetInterestTimeout { + /// Returns new `ResetInterestTimeout` that resolves after given timeout. + pub fn new(candidate_hash: CandidateHash, peer_id: PeerId, delay: Duration) -> Self { + Self { fut: futures_timer::Delay::new(delay), candidate_hash, peer_id } + } +} + +impl Future for ResetInterestTimeout { + type Output = (CandidateHash, PeerId); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.fut.poll_unpin(cx).map(|_| (self.candidate_hash, self.peer_id)) + } +} + #[cfg(test)] mod tests { use super::*; + use polkadot_primitives::v2::Hash; use sp_keyring::Sr25519Keyring; #[test] @@ -227,8 +259,8 @@ mod tests { let cap = NonZeroUsize::new(1).unwrap(); let mut buf = ValidatorGroupsBuffer::with_capacity(cap); - let hash_a = Hash::repeat_byte(0x1); - let hash_b = Hash::repeat_byte(0x2); + let hash_a = CandidateHash(Hash::repeat_byte(0x1)); + let hash_b = CandidateHash(Hash::repeat_byte(0x2)); let validators: Vec<_> = [ Sr25519Keyring::Alice, @@ -263,7 +295,7 @@ mod tests { let cap = NonZeroUsize::new(3).unwrap(); let mut buf = ValidatorGroupsBuffer::with_capacity(cap); - let hashes: Vec<_> = (0..5).map(Hash::repeat_byte).collect(); + let hashes: Vec<_> = (0..5).map(|i| CandidateHash(Hash::repeat_byte(i))).collect(); let validators: Vec<_> = [ Sr25519Keyring::Alice, diff --git a/node/network/collator-protocol/src/error.rs b/node/network/collator-protocol/src/error.rs index b1c86fa81c5a..4003ac438c92 100644 --- a/node/network/collator-protocol/src/error.rs +++ b/node/network/collator-protocol/src/error.rs @@ -17,10 +17,12 @@ //! Error handling related code and Error/Result definitions. +use futures::channel::oneshot; + use polkadot_node_network_protocol::request_response::incoming; use polkadot_node_primitives::UncheckedSignedFullStatement; -use polkadot_node_subsystem::errors::SubsystemError; -use polkadot_node_subsystem_util::runtime; +use polkadot_node_subsystem::{errors::SubsystemError, RuntimeApiError}; +use polkadot_node_subsystem_util::{backing_implicit_view, runtime}; use crate::LOG_TARGET; @@ -44,10 +46,84 @@ pub enum Error { #[error("Error while accessing runtime information")] Runtime(#[from] runtime::Error), + #[error("Error while accessing Runtime API")] + RuntimeApi(#[from] RuntimeApiError), + + #[error(transparent)] + ImplicitViewFetchError(backing_implicit_view::FetchError), + + #[error("Response receiver for Runtime API version request cancelled")] + CancelledRuntimeApiVersion(oneshot::Canceled), + + #[error("Response receiver for active validators request cancelled")] + CancelledActiveValidators(oneshot::Canceled), + + #[error("Response receiver for validator groups request cancelled")] + CancelledValidatorGroups(oneshot::Canceled), + + #[error("Response receiver for availability cores request cancelled")] + CancelledAvailabilityCores(oneshot::Canceled), + #[error("CollationSeconded contained statement with invalid signature")] InvalidStatementSignature(UncheckedSignedFullStatement), } +/// An error happened on the validator side of the protocol when attempting +/// to start seconding a candidate. +#[derive(Debug, thiserror::Error)] +pub enum SecondingError { + #[error("Failed to fetch a collation")] + FailedToFetch(#[from] oneshot::Canceled), + + #[error("Error while accessing Runtime API")] + RuntimeApi(#[from] RuntimeApiError), + + #[error("Response receiver for persisted validation data request cancelled")] + CancelledRuntimePersistedValidationData(oneshot::Canceled), + + #[error("Response receiver for prospective validation data request cancelled")] + CancelledProspectiveValidationData(oneshot::Canceled), + + #[error("Persisted validation data is not available")] + PersistedValidationDataNotFound, + + #[error("Persisted validation data hash doesn't match one in the candidate receipt.")] + PersistedValidationDataMismatch, + + #[error("Candidate hash doesn't match the advertisement")] + CandidateHashMismatch, + + #[error("Received duplicate collation from the peer")] + Duplicate, +} + +impl SecondingError { + /// Returns true if an error indicates that a peer is malicious. + pub fn is_malicious(&self) -> bool { + use SecondingError::*; + matches!(self, PersistedValidationDataMismatch | CandidateHashMismatch | Duplicate) + } +} + +/// A validator failed to request a collation due to an error. +#[derive(Debug, thiserror::Error)] +pub enum FetchError { + #[error("Collation was not previously advertised")] + NotAdvertised, + + #[error("Peer is unknown")] + UnknownPeer, + + #[error("Collation was already requested")] + AlreadyRequested, + + #[error("Relay parent went out of view")] + RelayParentOutOfView, + + #[error("Peer's protocol doesn't match the advertisement")] + ProtocolMismatch, +} + /// Utility for eating top level errors and log them. /// /// We basically always want to try and continue on error. This utility function is meant to diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index b71acc127c88..85f560ceaa6e 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -31,13 +31,15 @@ use futures::{ use sp_keystore::SyncCryptoStorePtr; use polkadot_node_network_protocol::{ - request_response::{v1 as request_v1, IncomingRequestReceiver}, + request_response::{v1 as request_v1, vstaging as protocol_vstaging, IncomingRequestReceiver}, PeerId, UnifiedReputationChange as Rep, }; -use polkadot_primitives::v2::CollatorPair; +use polkadot_primitives::v2::{CollatorPair, Hash}; use polkadot_node_subsystem::{ - errors::SubsystemError, messages::NetworkBridgeTxMessage, overseer, SpawnedSubsystem, + errors::SubsystemError, + messages::{NetworkBridgeTxMessage, RuntimeApiMessage, RuntimeApiRequest}, + overseer, SpawnedSubsystem, }; mod error; @@ -47,6 +49,15 @@ mod validator_side; const LOG_TARGET: &'static str = "parachain::collator-protocol"; +/// The maximum depth a candidate can occupy for any relay parent. +/// 'depth' is defined as the amount of blocks between the para +/// head in a relay-chain block's state and a candidate with a +/// particular relay-parent. +/// +/// This value is only used for limiting the number of candidates +/// we accept and distribute per relay parent. +const MAX_CANDIDATE_DEPTH: usize = 4; + /// A collator eviction policy - how fast to evict collators which are inactive. #[derive(Debug, Clone, Copy)] pub struct CollatorEvictionPolicy { @@ -77,12 +88,19 @@ pub enum ProtocolSide { metrics: validator_side::Metrics, }, /// Collators operate on a parachain. - Collator( - PeerId, - CollatorPair, - IncomingRequestReceiver, - collator_side::Metrics, - ), + Collator { + /// Local peer id. + peer_id: PeerId, + /// Parachain collator pair. + collator_pair: CollatorPair, + /// Receiver for v1 collation fetching requests. + request_receiver_v1: IncomingRequestReceiver, + /// Receiver for vstaging collation fetching requests. + request_receiver_vstaging: + IncomingRequestReceiver, + /// Metrics. + metrics: collator_side::Metrics, + }, } /// The collator protocol subsystem. @@ -104,8 +122,22 @@ impl CollatorProtocolSubsystem { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => validator_side::run(ctx, keystore, eviction_policy, metrics).await, - ProtocolSide::Collator(local_peer_id, collator_pair, req_receiver, metrics) => - collator_side::run(ctx, local_peer_id, collator_pair, req_receiver, metrics).await, + ProtocolSide::Collator { + peer_id, + collator_pair, + request_receiver_v1, + request_receiver_vstaging, + metrics, + } => + collator_side::run( + ctx, + peer_id, + collator_pair, + request_receiver_v1, + request_receiver_vstaging, + metrics, + ) + .await, } } } @@ -157,3 +189,48 @@ fn tick_stream(period: Duration) -> impl FusedStream { }) .fuse() } + +#[derive(Debug, Clone, Copy, PartialEq)] +enum ProspectiveParachainsMode { + // v2 runtime API: no prospective parachains. + Disabled, + // vstaging runtime API: prospective parachains. + Enabled, +} + +impl ProspectiveParachainsMode { + fn is_enabled(&self) -> bool { + matches!(self, Self::Enabled) + } +} + +async fn prospective_parachains_mode( + sender: &mut Sender, + leaf_hash: Hash, +) -> Result +where + Sender: polkadot_node_subsystem::CollatorProtocolSenderTrait, +{ + let (tx, rx) = futures::channel::oneshot::channel(); + sender + .send_message(RuntimeApiMessage::Request(leaf_hash, RuntimeApiRequest::Version(tx))) + .await; + + let version = rx + .await + .map_err(error::Error::CancelledRuntimeApiVersion)? + .map_err(error::Error::RuntimeApi)?; + + if version >= RuntimeApiRequest::VALIDITY_CONSTRAINTS { + Ok(ProspectiveParachainsMode::Enabled) + } else { + if version < 2 { + gum::warn!( + target: LOG_TARGET, + "Runtime API version is {}, it is expected to be at least 2. Prospective parachains are disabled", + version + ); + } + Ok(ProspectiveParachainsMode::Disabled) + } +} diff --git a/node/network/collator-protocol/src/validator_side/collation.rs b/node/network/collator-protocol/src/validator_side/collation.rs new file mode 100644 index 000000000000..c5cb848d2815 --- /dev/null +++ b/node/network/collator-protocol/src/validator_side/collation.rs @@ -0,0 +1,254 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Primitives for tracking collations-related data. +//! +//! Usually a path of collations is as follows: +//! 1. First, collation must be advertised by collator. +//! 2. If the advertisement was accepted, it's queued for fetch (per relay parent). +//! 3. Once it's requested, the collation is said to be Pending. +//! 4. Pending collation becomes Fetched once received, we send it to backing for validation. +//! 5. If it turns to be invalid or async backing allows seconding another candidate, carry on with +//! the next advertisement, otherwise we're done with this relay parent. +//! +//! ┌──────────────────────────────────────────┐ +//! └─▶Advertised ─▶ Pending ─▶ Fetched ─▶ Validated + +use futures::channel::oneshot; +use std::collections::{HashMap, VecDeque}; + +use polkadot_node_network_protocol::PeerId; +use polkadot_node_primitives::PoV; +use polkadot_primitives::v2::{ + CandidateHash, CandidateReceipt, CollatorId, Hash, Id as ParaId, PersistedValidationData, +}; + +use crate::{error::SecondingError, ProspectiveParachainsMode, LOG_TARGET, MAX_CANDIDATE_DEPTH}; + +/// Candidate supplied with a para head it's built on top of. +#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] +pub struct ProspectiveCandidate { + /// Candidate hash. + pub candidate_hash: CandidateHash, + /// Parent head-data hash as supplied in advertisement. + pub parent_head_data_hash: Hash, +} + +impl ProspectiveCandidate { + pub fn candidate_hash(&self) -> CandidateHash { + self.candidate_hash + } +} + +/// Identifier of a fetched collation. +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +pub struct FetchedCollation { + /// Candidate's relay parent. + pub relay_parent: Hash, + /// Parachain id. + pub para_id: ParaId, + /// Candidate hash. + pub candidate_hash: CandidateHash, + /// Id of the collator the collation was fetched from. + pub collator_id: CollatorId, +} + +impl From<&CandidateReceipt> for FetchedCollation { + fn from(receipt: &CandidateReceipt) -> Self { + let descriptor = receipt.descriptor(); + Self { + relay_parent: descriptor.relay_parent, + para_id: descriptor.para_id, + candidate_hash: receipt.hash(), + collator_id: descriptor.collator.clone(), + } + } +} + +/// Identifier of a collation being requested. +#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] +pub struct PendingCollation { + /// Candidate's relay parent. + pub relay_parent: Hash, + /// Parachain id. + pub para_id: ParaId, + /// Peer that advertised this collation. + pub peer_id: PeerId, + /// Optional candidate hash and parent head-data hash if were + /// supplied in advertisement. + pub prospective_candidate: Option, + /// Hash of the candidate's commitments. + pub commitments_hash: Option, +} + +impl PendingCollation { + pub fn new( + relay_parent: Hash, + para_id: ParaId, + peer_id: &PeerId, + prospective_candidate: Option, + ) -> Self { + Self { + relay_parent, + para_id, + peer_id: peer_id.clone(), + prospective_candidate, + commitments_hash: None, + } + } +} + +/// Performs a sanity check between advertised and fetched collations. +/// +/// Since the persisted validation data is constructed using the advertised +/// parent head data hash, the latter doesn't require an additional check. +pub fn fetched_collation_sanity_check( + advertised: &PendingCollation, + fetched: &CandidateReceipt, + persisted_validation_data: &PersistedValidationData, +) -> Result<(), SecondingError> { + if persisted_validation_data.hash() != fetched.descriptor().persisted_validation_data_hash { + Err(SecondingError::PersistedValidationDataMismatch) + } else if advertised + .prospective_candidate + .map_or(false, |pc| pc.candidate_hash() != fetched.hash()) + { + Err(SecondingError::CandidateHashMismatch) + } else { + Ok(()) + } +} + +pub type CollationEvent = (CollatorId, PendingCollation); + +pub type PendingCollationFetch = + (CollationEvent, std::result::Result<(CandidateReceipt, PoV), oneshot::Canceled>); + +/// The status of the collations in [`CollationsPerRelayParent`]. +#[derive(Debug, Clone, Copy)] +pub enum CollationStatus { + /// We are waiting for a collation to be advertised to us. + Waiting, + /// We are currently fetching a collation. + Fetching, + /// We are waiting that a collation is being validated. + WaitingOnValidation, + /// We have seconded a collation. + Seconded, +} + +impl Default for CollationStatus { + fn default() -> Self { + Self::Waiting + } +} + +impl CollationStatus { + /// Downgrades to `Waiting`, but only if `self != Seconded`. + fn back_to_waiting(&mut self, relay_parent_mode: ProspectiveParachainsMode) { + match self { + Self::Seconded => + if relay_parent_mode.is_enabled() { + // With async backing enabled it's allowed to + // second more candidates. + *self = Self::Waiting + }, + _ => *self = Self::Waiting, + } + } +} + +/// Information about collations per relay parent. +#[derive(Default)] +pub struct Collations { + /// What is the current status in regards to a collation for this relay parent? + pub status: CollationStatus, + /// Collator we're fetching from, optionally which candidate was requested. + /// + /// This is the currently last started fetch, which did not exceed `MAX_UNSHARED_DOWNLOAD_TIME` + /// yet. + pub fetching_from: Option<(CollatorId, Option)>, + /// Collation that were advertised to us, but we did not yet fetch. + pub waiting_queue: VecDeque<(PendingCollation, CollatorId)>, + /// How many collations have been seconded per parachain. + pub seconded_count: HashMap, +} + +impl Collations { + /// Note a seconded collation for a given para. + pub(super) fn note_seconded(&mut self, para_id: ParaId) { + *self.seconded_count.entry(para_id).or_insert(0) += 1 + } + + /// Returns the next collation to fetch from the `waiting_queue`. + /// + /// This will reset the status back to `Waiting` using [`CollationStatus::back_to_waiting`]. + /// + /// Returns `Some(_)` if there is any collation to fetch, the `status` is not `Seconded` and + /// the passed in `finished_one` is the currently `waiting_collation`. + pub(super) fn get_next_collation_to_fetch( + &mut self, + finished_one: &(CollatorId, Option), + relay_parent_mode: ProspectiveParachainsMode, + ) -> Option<(PendingCollation, CollatorId)> { + // If finished one does not match waiting_collation, then we already dequeued another fetch + // to replace it. + if let Some((collator_id, maybe_candidate_hash)) = self.fetching_from.as_ref() { + // If a candidate hash was saved previously, `finished_one` must include this too. + if collator_id != &finished_one.0 && + maybe_candidate_hash.map_or(true, |hash| Some(&hash) != finished_one.1.as_ref()) + { + gum::trace!( + target: LOG_TARGET, + waiting_collation = ?self.fetching_from, + ?finished_one, + "Not proceeding to the next collation - has already been done." + ); + return None + } + } + self.status.back_to_waiting(relay_parent_mode); + + match self.status { + // We don't need to fetch any other collation when we already have seconded one. + CollationStatus::Seconded => None, + CollationStatus::Waiting => { + while let Some(next) = self.waiting_queue.pop_front() { + let para_id = next.0.para_id; + if !self.is_seconded_limit_reached(relay_parent_mode, para_id) { + continue + } + + return Some(next) + } + None + }, + CollationStatus::WaitingOnValidation | CollationStatus::Fetching => + unreachable!("We have reset the status above!"), + } + } + + /// Checks the limit of seconded candidates for a given para. + pub(super) fn is_seconded_limit_reached( + &self, + relay_parent_mode: ProspectiveParachainsMode, + para_id: ParaId, + ) -> bool { + let seconded_limit = + if relay_parent_mode.is_enabled() { MAX_CANDIDATE_DEPTH + 1 } else { 1 }; + self.seconded_count.get(¶_id).map_or(true, |&num| num < seconded_limit) + } +} diff --git a/node/network/collator-protocol/src/validator_side/metrics.rs b/node/network/collator-protocol/src/validator_side/metrics.rs new file mode 100644 index 000000000000..a011a5f3b43e --- /dev/null +++ b/node/network/collator-protocol/src/validator_side/metrics.rs @@ -0,0 +1,123 @@ +// Copyright 2017-2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_node_subsystem_util::metrics::{self, prometheus}; + +#[derive(Clone, Default)] +pub struct Metrics(Option); + +impl Metrics { + pub fn on_request(&self, succeeded: std::result::Result<(), ()>) { + if let Some(metrics) = &self.0 { + match succeeded { + Ok(()) => metrics.collation_requests.with_label_values(&["succeeded"]).inc(), + Err(()) => metrics.collation_requests.with_label_values(&["failed"]).inc(), + } + } + } + + /// Provide a timer for `process_msg` which observes on drop. + pub fn time_process_msg(&self) -> Option { + self.0.as_ref().map(|metrics| metrics.process_msg.start_timer()) + } + + /// Provide a timer for `handle_collation_request_result` which observes on drop. + pub fn time_handle_collation_request_result( + &self, + ) -> Option { + self.0 + .as_ref() + .map(|metrics| metrics.handle_collation_request_result.start_timer()) + } + + /// Note the current number of collator peers. + pub fn note_collator_peer_count(&self, collator_peers: usize) { + self.0 + .as_ref() + .map(|metrics| metrics.collator_peer_count.set(collator_peers as u64)); + } + + /// Provide a timer for `PerRequest` structure which observes on drop. + pub fn time_collation_request_duration( + &self, + ) -> Option { + self.0.as_ref().map(|metrics| metrics.collation_request_duration.start_timer()) + } +} + +#[derive(Clone)] +struct MetricsInner { + collation_requests: prometheus::CounterVec, + process_msg: prometheus::Histogram, + handle_collation_request_result: prometheus::Histogram, + collator_peer_count: prometheus::Gauge, + collation_request_duration: prometheus::Histogram, +} + +impl metrics::Metrics for Metrics { + fn try_register( + registry: &prometheus::Registry, + ) -> std::result::Result { + let metrics = MetricsInner { + collation_requests: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_collation_requests_total", + "Number of collations requested from Collators.", + ), + &["success"], + )?, + registry, + )?, + process_msg: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_collator_protocol_validator_process_msg", + "Time spent within `collator_protocol_validator::process_msg`", + ) + )?, + registry, + )?, + handle_collation_request_result: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_collator_protocol_validator_handle_collation_request_result", + "Time spent within `collator_protocol_validator::handle_collation_request_result`", + ) + )?, + registry, + )?, + collator_peer_count: prometheus::register( + prometheus::Gauge::new( + "polkadot_parachain_collator_peer_count", + "Amount of collator peers connected", + )?, + registry, + )?, + collation_request_duration: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_collator_protocol_validator_collation_request_duration", + "Lifetime of the `PerRequest` structure", + ).buckets(vec![0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.75, 0.9, 1.0, 1.2, 1.5, 1.75]), + )?, + registry, + )?, + }; + + Ok(Metrics(Some(metrics))) + } +} diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index 58e2eb5b6c3f..2454a23e3581 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -25,7 +25,8 @@ use futures::{ use futures_timer::Delay; use std::{ collections::{hash_map::Entry, HashMap, HashSet}, - sync::Arc, + convert::TryInto, + iter::FromIterator, task::Poll, time::{Duration, Instant}, }; @@ -34,37 +35,52 @@ use sp_keystore::SyncCryptoStorePtr; use polkadot_node_network_protocol::{ self as net_protocol, - peer_set::PeerSet, + peer_set::{CollationVersion, PeerSet}, request_response as req_res, request_response::{ outgoing::{Recipient, RequestError}, - v1::{CollationFetchingRequest, CollationFetchingResponse}, - OutgoingRequest, Requests, + v1 as request_v1, vstaging as request_vstaging, OutgoingRequest, Requests, }, - v1 as protocol_v1, OurView, PeerId, UnifiedReputationChange as Rep, Versioned, View, + v1 as protocol_v1, vstaging as protocol_vstaging, OurView, PeerId, + UnifiedReputationChange as Rep, Versioned, View, }; -use polkadot_node_primitives::{PoV, SignedFullStatement}; +use polkadot_node_primitives::{PoV, SignedFullStatement, Statement}; use polkadot_node_subsystem::{ jaeger, messages::{ CandidateBackingMessage, CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, - NetworkBridgeTxMessage, RuntimeApiMessage, + NetworkBridgeTxMessage, ProspectiveParachainsMessage, ProspectiveValidationDataRequest, }, - overseer, FromOrchestra, OverseerSignal, PerLeafSpan, SubsystemSender, + overseer, CollatorProtocolSenderTrait, FromOrchestra, OverseerSignal, PerLeafSpan, +}; +use polkadot_node_subsystem_util::{ + backing_implicit_view::View as ImplicitView, metrics::prometheus::prometheus::HistogramTimer, }; -use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v2::{ - CandidateReceipt, CollatorId, Hash, Id as ParaId, OccupiedCoreAssumption, - PersistedValidationData, + CandidateHash, CandidateReceipt, CollatorId, CoreState, Hash, Id as ParaId, + OccupiedCoreAssumption, PersistedValidationData, }; -use crate::error::Result; +use crate::error::{Error, FetchError, Result, SecondingError}; + +use super::{ + modify_reputation, prospective_parachains_mode, tick_stream, ProspectiveParachainsMode, + LOG_TARGET, MAX_CANDIDATE_DEPTH, +}; -use super::{modify_reputation, tick_stream, LOG_TARGET}; +mod collation; +mod metrics; + +use collation::{ + fetched_collation_sanity_check, CollationEvent, CollationStatus, Collations, FetchedCollation, + PendingCollation, PendingCollationFetch, ProspectiveCandidate, +}; #[cfg(test)] mod tests; +pub use metrics::Metrics; + const COST_UNEXPECTED_MESSAGE: Rep = Rep::CostMinor("An unexpected message"); /// Message could not be decoded properly. const COST_CORRUPTED_MESSAGE: Rep = Rep::CostMinor("Message was corrupt"); @@ -104,131 +120,33 @@ const ACTIVITY_POLL: Duration = Duration::from_millis(10); // How often to poll collation responses. // This is a hack that should be removed in a refactoring. // See https://github.com/paritytech/polkadot/issues/4182 -const CHECK_COLLATIONS_POLL: Duration = Duration::from_millis(50); - -#[derive(Clone, Default)] -pub struct Metrics(Option); - -impl Metrics { - fn on_request(&self, succeeded: std::result::Result<(), ()>) { - if let Some(metrics) = &self.0 { - match succeeded { - Ok(()) => metrics.collation_requests.with_label_values(&["succeeded"]).inc(), - Err(()) => metrics.collation_requests.with_label_values(&["failed"]).inc(), - } - } - } - - /// Provide a timer for `process_msg` which observes on drop. - fn time_process_msg(&self) -> Option { - self.0.as_ref().map(|metrics| metrics.process_msg.start_timer()) - } - - /// Provide a timer for `handle_collation_request_result` which observes on drop. - fn time_handle_collation_request_result( - &self, - ) -> Option { - self.0 - .as_ref() - .map(|metrics| metrics.handle_collation_request_result.start_timer()) - } - - /// Note the current number of collator peers. - fn note_collator_peer_count(&self, collator_peers: usize) { - self.0 - .as_ref() - .map(|metrics| metrics.collator_peer_count.set(collator_peers as u64)); - } - - /// Provide a timer for `PerRequest` structure which observes on drop. - fn time_collation_request_duration( - &self, - ) -> Option { - self.0.as_ref().map(|metrics| metrics.collation_request_duration.start_timer()) - } -} - -#[derive(Clone)] -struct MetricsInner { - collation_requests: prometheus::CounterVec, - process_msg: prometheus::Histogram, - handle_collation_request_result: prometheus::Histogram, - collator_peer_count: prometheus::Gauge, - collation_request_duration: prometheus::Histogram, -} - -impl metrics::Metrics for Metrics { - fn try_register( - registry: &prometheus::Registry, - ) -> std::result::Result { - let metrics = MetricsInner { - collation_requests: prometheus::register( - prometheus::CounterVec::new( - prometheus::Opts::new( - "polkadot_parachain_collation_requests_total", - "Number of collations requested from Collators.", - ), - &["success"], - )?, - registry, - )?, - process_msg: prometheus::register( - prometheus::Histogram::with_opts( - prometheus::HistogramOpts::new( - "polkadot_parachain_collator_protocol_validator_process_msg", - "Time spent within `collator_protocol_validator::process_msg`", - ) - )?, - registry, - )?, - handle_collation_request_result: prometheus::register( - prometheus::Histogram::with_opts( - prometheus::HistogramOpts::new( - "polkadot_parachain_collator_protocol_validator_handle_collation_request_result", - "Time spent within `collator_protocol_validator::handle_collation_request_result`", - ) - )?, - registry, - )?, - collator_peer_count: prometheus::register( - prometheus::Gauge::new( - "polkadot_parachain_collator_peer_count", - "Amount of collator peers connected", - )?, - registry, - )?, - collation_request_duration: prometheus::register( - prometheus::Histogram::with_opts( - prometheus::HistogramOpts::new( - "polkadot_parachain_collator_protocol_validator_collation_request_duration", - "Lifetime of the `PerRequest` structure", - ).buckets(vec![0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.75, 0.9, 1.0, 1.2, 1.5, 1.75]), - )?, - registry, - )?, - }; - - Ok(Metrics(Some(metrics))) - } -} +const CHECK_COLLATIONS_POLL: Duration = Duration::from_millis(5); struct PerRequest { /// Responses from collator. - from_collator: Fuse>>, + /// + /// The response payload is the same for both versions of protocol + /// and doesn't have vstaging alias for simplicity. + from_collator: + Fuse>>, /// Sender to forward to initial requester. to_requester: oneshot::Sender<(CandidateReceipt, PoV)>, /// A jaeger span corresponding to the lifetime of the request. span: Option, /// A metric histogram for the lifetime of the request - _lifetime_timer: Option, + _lifetime_timer: Option, } #[derive(Debug)] struct CollatingPeerState { collator_id: CollatorId, para_id: ParaId, - // Advertised relay parents. - advertisements: HashSet, + /// Collations advertised by peer per relay parent. + /// + /// V1 network protocol doesn't include candidate hash in + /// advertisements, we store an empty set in this case to occupy + /// a slot in map. + advertisements: HashMap>, last_active: Instant, } @@ -241,38 +159,85 @@ enum PeerState { } #[derive(Debug)] -enum AdvertisementError { +enum InsertAdvertisementError { + /// Advertisement is already known. Duplicate, + /// Collation relay parent is out of our view. OutOfOurView, + /// No prior declare message received. UndeclaredCollator, + /// A limit for announcements per peer is reached. + PeerLimitReached, + /// Mismatch of relay parent mode and advertisement arguments. + /// An internal error that should not happen. + ProtocolMismatch, } #[derive(Debug)] struct PeerData { view: View, state: PeerState, + version: CollationVersion, } impl PeerData { - fn new(view: View) -> Self { - PeerData { view, state: PeerState::Connected(Instant::now()) } - } - /// Update the view, clearing all advertisements that are no longer in the /// current view. - fn update_view(&mut self, new_view: View) { + fn update_view( + &mut self, + implicit_view: &ImplicitView, + active_leaves: &HashMap, + per_relay_parent: &HashMap, + new_view: View, + ) { let old_view = std::mem::replace(&mut self.view, new_view); if let PeerState::Collating(ref mut peer_state) = self.state { for removed in old_view.difference(&self.view) { - let _ = peer_state.advertisements.remove(&removed); + // Remove relay parent advertisements if it went out + // of our (implicit) view. + let keep = per_relay_parent + .get(removed) + .map(|s| { + is_relay_parent_in_implicit_view( + removed, + s.prospective_parachains_mode, + implicit_view, + active_leaves, + peer_state.para_id, + ) + }) + .unwrap_or(false); + + if !keep { + peer_state.advertisements.remove(&removed); + } } } } /// Prune old advertisements relative to our view. - fn prune_old_advertisements(&mut self, our_view: &View) { + fn prune_old_advertisements( + &mut self, + implicit_view: &ImplicitView, + active_leaves: &HashMap, + per_relay_parent: &HashMap, + ) { if let PeerState::Collating(ref mut peer_state) = self.state { - peer_state.advertisements.retain(|a| our_view.contains(a)); + peer_state.advertisements.retain(|hash, _| { + // Either + // - Relay parent is an active leaf + // - It belongs to allowed ancestry under some leaf + // Discard otherwise. + per_relay_parent.get(hash).map_or(false, |s| { + is_relay_parent_in_implicit_view( + hash, + s.prospective_parachains_mode, + implicit_view, + active_leaves, + peer_state.para_id, + ) + }) + }); } } @@ -282,18 +247,54 @@ impl PeerData { fn insert_advertisement( &mut self, on_relay_parent: Hash, - our_view: &View, - ) -> std::result::Result<(CollatorId, ParaId), AdvertisementError> { + relay_parent_mode: ProspectiveParachainsMode, + candidate_hash: Option, + implicit_view: &ImplicitView, + active_leaves: &HashMap, + ) -> std::result::Result<(CollatorId, ParaId), InsertAdvertisementError> { match self.state { - PeerState::Connected(_) => Err(AdvertisementError::UndeclaredCollator), - _ if !our_view.contains(&on_relay_parent) => Err(AdvertisementError::OutOfOurView), - PeerState::Collating(ref mut state) => - if state.advertisements.insert(on_relay_parent) { - state.last_active = Instant::now(); - Ok((state.collator_id.clone(), state.para_id.clone())) - } else { - Err(AdvertisementError::Duplicate) - }, + PeerState::Connected(_) => Err(InsertAdvertisementError::UndeclaredCollator), + PeerState::Collating(ref mut state) => { + if !is_relay_parent_in_implicit_view( + &on_relay_parent, + relay_parent_mode, + implicit_view, + active_leaves, + state.para_id, + ) { + return Err(InsertAdvertisementError::OutOfOurView) + } + + match (relay_parent_mode, candidate_hash) { + (ProspectiveParachainsMode::Disabled, candidate_hash) => { + if state.advertisements.contains_key(&on_relay_parent) { + return Err(InsertAdvertisementError::Duplicate) + } + state + .advertisements + .insert(on_relay_parent, HashSet::from_iter(candidate_hash)); + }, + (ProspectiveParachainsMode::Enabled, Some(candidate_hash)) => { + if state + .advertisements + .get(&on_relay_parent) + .map_or(false, |candidates| candidates.contains(&candidate_hash)) + { + return Err(InsertAdvertisementError::Duplicate) + } + let candidates = state.advertisements.entry(on_relay_parent).or_default(); + + if candidates.len() >= MAX_CANDIDATE_DEPTH + 1 { + return Err(InsertAdvertisementError::PeerLimitReached) + } + candidates.insert(candidate_hash); + }, + _ => return Err(InsertAdvertisementError::ProtocolMismatch), + } + + state.last_active = Instant::now(); + Ok((state.collator_id.clone(), state.para_id)) + }, } } @@ -305,7 +306,7 @@ impl PeerData { } } - /// Note that a peer is now collating with the given collator and para ids. + /// Note that a peer is now collating with the given collator and para id. /// /// This will overwrite any previous call to `set_collating` and should only be called /// if `is_collating` is false. @@ -313,7 +314,7 @@ impl PeerData { self.state = PeerState::Collating(CollatingPeerState { collator_id, para_id, - advertisements: HashSet::new(), + advertisements: HashMap::new(), last_active: Instant::now(), }); } @@ -333,10 +334,23 @@ impl PeerData { } /// Whether the peer has advertised the given collation. - fn has_advertised(&self, relay_parent: &Hash) -> bool { - match self.state { - PeerState::Connected(_) => false, - PeerState::Collating(ref state) => state.advertisements.contains(relay_parent), + fn has_advertised( + &self, + relay_parent: &Hash, + maybe_candidate_hash: Option, + ) -> bool { + let collating_state = match self.state { + PeerState::Connected(_) => return false, + PeerState::Collating(ref state) => state, + }; + + if let Some(ref candidate_hash) = maybe_candidate_hash { + collating_state + .advertisements + .get(relay_parent) + .map_or(false, |candidates| candidates.contains(candidate_hash)) + } else { + collating_state.advertisements.contains_key(relay_parent) } } @@ -350,236 +364,45 @@ impl PeerData { } } -impl Default for PeerData { - fn default() -> Self { - PeerData::new(Default::default()) - } -} - -struct GroupAssignments { - current: Option, -} - -#[derive(Default)] -struct ActiveParas { - relay_parent_assignments: HashMap, - current_assignments: HashMap, -} - -impl ActiveParas { - async fn assign_incoming( - &mut self, - sender: &mut impl SubsystemSender, - keystore: &SyncCryptoStorePtr, - new_relay_parents: impl IntoIterator, - ) { - for relay_parent in new_relay_parents { - let mv = polkadot_node_subsystem_util::request_validators(relay_parent, sender) - .await - .await - .ok() - .map(|x| x.ok()) - .flatten(); - - let mg = polkadot_node_subsystem_util::request_validator_groups(relay_parent, sender) - .await - .await - .ok() - .map(|x| x.ok()) - .flatten(); - - let mc = polkadot_node_subsystem_util::request_availability_cores(relay_parent, sender) - .await - .await - .ok() - .map(|x| x.ok()) - .flatten(); - - let (validators, groups, rotation_info, cores) = match (mv, mg, mc) { - (Some(v), Some((g, r)), Some(c)) => (v, g, r, c), - _ => { - gum::debug!( - target: LOG_TARGET, - ?relay_parent, - "Failed to query runtime API for relay-parent", - ); - - continue - }, - }; - - let para_now = - match polkadot_node_subsystem_util::signing_key_and_index(&validators, keystore) - .await - .and_then(|(_, index)| { - polkadot_node_subsystem_util::find_validator_group(&groups, index) - }) { - Some(group) => { - let core_now = rotation_info.core_for_group(group, cores.len()); - - cores.get(core_now.0 as usize).and_then(|c| c.para_id()) - }, - None => { - gum::trace!(target: LOG_TARGET, ?relay_parent, "Not a validator"); - - continue - }, - }; - - // This code won't work well, if at all for parathreads. For parathreads we'll - // have to be aware of which core the parathread claim is going to be multiplexed - // onto. The parathread claim will also have a known collator, and we should always - // allow an incoming connection from that collator. If not even connecting to them - // directly. - // - // However, this'll work fine for parachains, as each parachain gets a dedicated - // core. - if let Some(para_now) = para_now { - let entry = self.current_assignments.entry(para_now).or_default(); - *entry += 1; - if *entry == 1 { - gum::debug!( - target: LOG_TARGET, - ?relay_parent, - para_id = ?para_now, - "Assigned to a parachain", - ); - } - } - - self.relay_parent_assignments - .insert(relay_parent, GroupAssignments { current: para_now }); - } - } - - fn remove_outgoing(&mut self, old_relay_parents: impl IntoIterator) { - for old_relay_parent in old_relay_parents { - if let Some(assignments) = self.relay_parent_assignments.remove(&old_relay_parent) { - let GroupAssignments { current } = assignments; - - if let Some(cur) = current { - if let Entry::Occupied(mut occupied) = self.current_assignments.entry(cur) { - *occupied.get_mut() -= 1; - if *occupied.get() == 0 { - occupied.remove_entry(); - gum::debug!( - target: LOG_TARGET, - para_id = ?cur, - "Unassigned from a parachain", - ); - } - } - } - } - } - } - - fn is_current(&self, id: &ParaId) -> bool { - self.current_assignments.contains_key(id) - } -} - -#[derive(Debug, Clone, Hash, Eq, PartialEq)] -struct PendingCollation { - relay_parent: Hash, - para_id: ParaId, - peer_id: PeerId, - commitments_hash: Option, +#[derive(Debug, Copy, Clone)] +enum AssignedCoreState { + Scheduled, + Occupied, } -impl PendingCollation { - fn new(relay_parent: Hash, para_id: &ParaId, peer_id: &PeerId) -> Self { - Self { - relay_parent, - para_id: para_id.clone(), - peer_id: peer_id.clone(), - commitments_hash: None, - } +impl AssignedCoreState { + fn is_occupied(&self) -> bool { + matches!(self, AssignedCoreState::Occupied) } } -type CollationEvent = (CollatorId, PendingCollation); - -type PendingCollationFetch = - (CollationEvent, std::result::Result<(CandidateReceipt, PoV), oneshot::Canceled>); - -/// The status of the collations in [`CollationsPerRelayParent`]. -#[derive(Debug, Clone, Copy)] -enum CollationStatus { - /// We are waiting for a collation to be advertised to us. - Waiting, - /// We are currently fetching a collation. - Fetching, - /// We are waiting that a collation is being validated. - WaitingOnValidation, - /// We have seconded a collation. - Seconded, -} - -impl Default for CollationStatus { - fn default() -> Self { - Self::Waiting - } -} - -impl CollationStatus { - /// Downgrades to `Waiting`, but only if `self != Seconded`. - fn back_to_waiting(&mut self) { - match self { - Self::Seconded => {}, - _ => *self = Self::Waiting, - } - } +#[derive(Debug, Clone)] +struct GroupAssignments { + /// Current assignment. + current: Option<(ParaId, AssignedCoreState)>, + /// Paras we're implicitly assigned to with respect to ancestry. + /// This only includes paras from children relay chain blocks assignments. + /// + /// Implicit assignments are not reference-counted since they're accumulated + /// from the most recent leaf. + /// + /// Should be relatively small depending on the group rotation frequency and + /// allowed ancestry length. + implicit: Vec, } -/// Information about collations per relay parent. -#[derive(Default)] -struct CollationsPerRelayParent { - /// What is the current status in regards to a collation for this relay parent? - status: CollationStatus, - /// Collation currently being fetched. - /// - /// This is the currently last started fetch, which did not exceed `MAX_UNSHARED_DOWNLOAD_TIME` - /// yet. - waiting_collation: Option, - /// Collation that were advertised to us, but we did not yet fetch. - unfetched_collations: Vec<(PendingCollation, CollatorId)>, +struct PerRelayParent { + prospective_parachains_mode: ProspectiveParachainsMode, + assignment: GroupAssignments, + collations: Collations, } -impl CollationsPerRelayParent { - /// Returns the next collation to fetch from the `unfetched_collations`. - /// - /// This will reset the status back to `Waiting` using [`CollationStatus::back_to_waiting`]. - /// - /// Returns `Some(_)` if there is any collation to fetch, the `status` is not `Seconded` and - /// the passed in `finished_one` is the currently `waiting_collation`. - pub fn get_next_collation_to_fetch( - &mut self, - finished_one: Option<&CollatorId>, - ) -> Option<(PendingCollation, CollatorId)> { - // If finished one does not match waiting_collation, then we already dequeued another fetch - // to replace it. - if self.waiting_collation.as_ref() != finished_one { - gum::trace!( - target: LOG_TARGET, - waiting_collation = ?self.waiting_collation, - ?finished_one, - "Not proceeding to the next collation - has already been done." - ); - return None - } - self.status.back_to_waiting(); - - match self.status { - // We don't need to fetch any other collation when we already have seconded one. - CollationStatus::Seconded => None, - CollationStatus::Waiting => { - let next = self.unfetched_collations.pop(); - self.waiting_collation = next.as_ref().map(|(_, collator_id)| collator_id.clone()); - next - }, - CollationStatus::WaitingOnValidation | CollationStatus::Fetching => - unreachable!("We have reset the status above!"), +impl PerRelayParent { + fn new(mode: ProspectiveParachainsMode) -> Self { + Self { + prospective_parachains_mode: mode, + assignment: GroupAssignments { current: None, implicit: Vec::new() }, + collations: Collations::default(), } } } @@ -587,15 +410,32 @@ impl CollationsPerRelayParent { /// All state relevant for the validator side of the protocol lives here. #[derive(Default)] struct State { - /// Our own view. - view: OurView, + /// Leaves that do support asynchronous backing along with + /// implicit ancestry. Leaves from the implicit view are present in + /// `active_leaves`, the opposite doesn't hold true. + /// + /// Relay-chain blocks which don't support prospective parachains are + /// never included in the fragment trees of active leaves which do. In + /// particular, this means that if a given relay parent belongs to implicit + /// ancestry of some active leaf, then it does support prospective parachains. + implicit_view: ImplicitView, + + /// All active leaves observed by us, including both that do and do not + /// support prospective parachains. This mapping works as a replacement for + /// [`polkadot_node_network_protocol::View`] and can be dropped once the transition + /// to asynchronous backing is done. + active_leaves: HashMap, - /// Active paras based on our view. We only accept collators from these paras. - active_paras: ActiveParas, + /// State tracked per relay parent. + per_relay_parent: HashMap, /// Track all active collators and their data. peer_data: HashMap, + /// Parachains we're currently assigned to. With async backing enabled + /// this includes assignments from the implicit view. + current_assignments: HashMap, + /// The collations we have requested by relay parent and para id. /// /// For each relay parent and para id we may be connected to a number @@ -617,13 +457,124 @@ struct State { /// /// A triggering timer means that the fetching took too long for our taste and we should give /// another collator the chance to be faster (dequeue next fetch request as well). - collation_fetch_timeouts: FuturesUnordered>, + collation_fetch_timeouts: + FuturesUnordered, Hash)>>, - /// Information about the collations per relay parent. - collations_per_relay_parent: HashMap, + /// Collations that we have successfully requested from peers and waiting + /// on validation. + fetched_candidates: HashMap, +} - /// Keep track of all pending candidate collations - pending_candidates: HashMap, +fn is_relay_parent_in_implicit_view( + relay_parent: &Hash, + relay_parent_mode: ProspectiveParachainsMode, + implicit_view: &ImplicitView, + active_leaves: &HashMap, + para_id: ParaId, +) -> bool { + match relay_parent_mode { + ProspectiveParachainsMode::Disabled => active_leaves.contains_key(relay_parent), + ProspectiveParachainsMode::Enabled => active_leaves.iter().any(|(hash, mode)| { + mode.is_enabled() && + implicit_view + .known_allowed_relay_parents_under(hash, Some(para_id)) + .unwrap_or_default() + .contains(relay_parent) + }), + } +} + +async fn assign_incoming( + sender: &mut Sender, + group_assignment: &mut GroupAssignments, + current_assignments: &mut HashMap, + keystore: &SyncCryptoStorePtr, + relay_parent: Hash, +) -> Result<()> +where + Sender: CollatorProtocolSenderTrait, +{ + let validators = polkadot_node_subsystem_util::request_validators(relay_parent, sender) + .await + .await + .map_err(Error::CancelledActiveValidators)??; + + let (groups, rotation_info) = + polkadot_node_subsystem_util::request_validator_groups(relay_parent, sender) + .await + .await + .map_err(Error::CancelledValidatorGroups)??; + + let cores = polkadot_node_subsystem_util::request_availability_cores(relay_parent, sender) + .await + .await + .map_err(Error::CancelledAvailabilityCores)??; + + let para_now = match polkadot_node_subsystem_util::signing_key_and_index(&validators, keystore) + .await + .and_then(|(_, index)| polkadot_node_subsystem_util::find_validator_group(&groups, index)) + { + Some(group) => { + let core_now = rotation_info.core_for_group(group, cores.len()); + + cores.get(core_now.0 as usize).and_then(|c| match c { + CoreState::Occupied(core) => Some((core.para_id(), AssignedCoreState::Occupied)), + CoreState::Scheduled(core) => Some((core.para_id, AssignedCoreState::Scheduled)), + CoreState::Free => None, + }) + }, + None => { + gum::trace!(target: LOG_TARGET, ?relay_parent, "Not a validator"); + + return Ok(()) + }, + }; + + // This code won't work well, if at all for parathreads. For parathreads we'll + // have to be aware of which core the parathread claim is going to be multiplexed + // onto. The parathread claim will also have a known collator, and we should always + // allow an incoming connection from that collator. If not even connecting to them + // directly. + // + // However, this'll work fine for parachains, as each parachain gets a dedicated + // core. + if let Some((para_id, _)) = para_now.as_ref() { + let entry = current_assignments.entry(*para_id).or_default(); + *entry += 1; + if *entry == 1 { + gum::debug!( + target: LOG_TARGET, + ?relay_parent, + para_id = ?para_id, + "Assigned to a parachain", + ); + } + } + + *group_assignment = GroupAssignments { current: para_now, implicit: Vec::new() }; + + Ok(()) +} + +fn remove_outgoing( + current_assignments: &mut HashMap, + per_relay_parent: PerRelayParent, +) { + let GroupAssignments { current, .. } = per_relay_parent.assignment; + + if let Some((cur, _)) = current { + if let Entry::Occupied(mut occupied) = current_assignments.entry(cur) { + *occupied.get_mut() -= 1; + if *occupied.get() == 0 { + occupied.remove_entry(); + gum::debug!( + target: LOG_TARGET, + para_id = ?cur, + "Unassigned from a parachain", + ); + } + } + } } // O(n) search for collator ID by iterating through the peers map. This should be fast enough @@ -649,42 +600,29 @@ async fn fetch_collation( state: &mut State, pc: PendingCollation, id: CollatorId, -) { +) -> std::result::Result<(), FetchError> { let (tx, rx) = oneshot::channel(); - let PendingCollation { relay_parent, para_id, peer_id, .. } = pc; + let PendingCollation { relay_parent, peer_id, prospective_candidate, .. } = pc; + let candidate_hash = prospective_candidate.as_ref().map(ProspectiveCandidate::candidate_hash); - let timeout = |collator_id, relay_parent| async move { - Delay::new(MAX_UNSHARED_DOWNLOAD_TIME).await; - (collator_id, relay_parent) - }; - state - .collation_fetch_timeouts - .push(timeout(id.clone(), relay_parent.clone()).boxed()); + let peer_data = state.peer_data.get(&peer_id).ok_or(FetchError::UnknownPeer)?; - if let Some(peer_data) = state.peer_data.get(&peer_id) { - if peer_data.has_advertised(&relay_parent) { - request_collation(sender, state, relay_parent, para_id, peer_id, tx).await; - } else { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?para_id, - ?relay_parent, - "Collation is not advertised for the relay parent by the peer, do not request it", - ); - } + if peer_data.has_advertised(&relay_parent, candidate_hash) { + request_collation(sender, state, pc, id.clone(), peer_data.version, tx).await?; + let timeout = |collator_id, candidate_hash, relay_parent| async move { + Delay::new(MAX_UNSHARED_DOWNLOAD_TIME).await; + (collator_id, candidate_hash, relay_parent) + }; + state + .collation_fetch_timeouts + .push(timeout(id.clone(), candidate_hash, relay_parent).boxed()); + state.collation_fetches.push(rx.map(move |r| ((id, pc), r)).boxed()); + + Ok(()) } else { - gum::warn!( - target: LOG_TARGET, - ?peer_id, - ?para_id, - ?relay_parent, - "Requested to fetch a collation from an unknown peer", - ); + Err(FetchError::NotAdvertised) } - - state.collation_fetches.push(rx.map(|r| ((id, pc), r)).boxed()); } /// Report a collator for some malicious actions. @@ -713,33 +651,46 @@ async fn note_good_collation( async fn notify_collation_seconded( sender: &mut impl overseer::CollatorProtocolSenderTrait, peer_id: PeerId, + version: CollationVersion, relay_parent: Hash, statement: SignedFullStatement, ) { - let wire_message = - protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement.into()); + let statement = statement.into(); + let wire_message = match version { + CollationVersion::V1 => Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol( + protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement), + )), + CollationVersion::VStaging => + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + protocol_vstaging::CollatorProtocolMessage::CollationSeconded( + relay_parent, + statement, + ), + )), + }; sender - .send_message(NetworkBridgeTxMessage::SendCollationMessage( - vec![peer_id], - Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)), - )) + .send_message(NetworkBridgeTxMessage::SendCollationMessage(vec![peer_id], wire_message)) .await; - - modify_reputation(sender, peer_id, BENEFIT_NOTIFY_GOOD).await; } /// A peer's view has changed. A number of things should be done: /// - Ongoing collation requests have to be canceled. /// - Advertisements by this peer that are no longer relevant have to be removed. -async fn handle_peer_view_change(state: &mut State, peer_id: PeerId, view: View) -> Result<()> { - let peer_data = state.peer_data.entry(peer_id.clone()).or_default(); +fn handle_peer_view_change(state: &mut State, peer_id: PeerId, view: View) { + let peer_data = match state.peer_data.get_mut(&peer_id) { + Some(peer_data) => peer_data, + None => return, + }; - peer_data.update_view(view); + peer_data.update_view( + &state.implicit_view, + &state.active_leaves, + &state.per_relay_parent, + view, + ); state .requested_collations - .retain(|pc, _| pc.peer_id != peer_id || peer_data.has_advertised(&pc.relay_parent)); - - Ok(()) + .retain(|pc, _| pc.peer_id != peer_id || peer_data.has_advertised(&pc.relay_parent, None)); } /// Request a collation from the network. @@ -751,41 +702,49 @@ async fn handle_peer_view_change(state: &mut State, peer_id: PeerId, view: View) async fn request_collation( sender: &mut impl overseer::CollatorProtocolSenderTrait, state: &mut State, - relay_parent: Hash, - para_id: ParaId, - peer_id: PeerId, + pending_collation: PendingCollation, + collator_id: CollatorId, + peer_protocol_version: CollationVersion, result: oneshot::Sender<(CandidateReceipt, PoV)>, -) { - if !state.view.contains(&relay_parent) { - gum::debug!( - target: LOG_TARGET, - peer_id = %peer_id, - para_id = %para_id, - relay_parent = %relay_parent, - "collation is no longer in view", - ); - return - } - let pending_collation = PendingCollation::new(relay_parent, ¶_id, &peer_id); +) -> std::result::Result<(), FetchError> { if state.requested_collations.contains_key(&pending_collation) { - gum::warn!( - target: LOG_TARGET, - peer_id = %pending_collation.peer_id, - %pending_collation.para_id, - ?pending_collation.relay_parent, - "collation has already been requested", - ); - return + return Err(FetchError::AlreadyRequested) } - let (full_request, response_recv) = OutgoingRequest::new( - Recipient::Peer(peer_id), - CollationFetchingRequest { relay_parent, para_id }, - ); - let requests = Requests::CollationFetchingV1(full_request); + let PendingCollation { relay_parent, para_id, peer_id, prospective_candidate, .. } = + pending_collation; + let per_relay_parent = state + .per_relay_parent + .get_mut(&relay_parent) + .ok_or(FetchError::RelayParentOutOfView)?; + + // Relay parent mode is checked in `handle_advertisement`. + let (requests, response_recv) = match (peer_protocol_version, prospective_candidate) { + (CollationVersion::V1, _) => { + let (req, response_recv) = OutgoingRequest::new( + Recipient::Peer(peer_id), + request_v1::CollationFetchingRequest { relay_parent, para_id }, + ); + let requests = Requests::CollationFetchingV1(req); + (requests, response_recv.boxed()) + }, + (CollationVersion::VStaging, Some(ProspectiveCandidate { candidate_hash, .. })) => { + let (req, response_recv) = OutgoingRequest::new( + Recipient::Peer(peer_id), + request_vstaging::CollationFetchingRequest { + relay_parent, + para_id, + candidate_hash, + }, + ); + let requests = Requests::CollationFetchingVStaging(req); + (requests, response_recv.boxed()) + }, + _ => return Err(FetchError::ProtocolMismatch), + }; let per_request = PerRequest { - from_collator: response_recv.boxed().fuse(), + from_collator: response_recv.fuse(), to_requester: result, span: state .span_per_relay_parent @@ -794,9 +753,7 @@ async fn request_collation( _lifetime_timer: state.metrics.time_collation_request_duration(), }; - state - .requested_collations - .insert(PendingCollation::new(relay_parent, ¶_id, &peer_id), per_request); + state.requested_collations.insert(pending_collation, per_request); gum::debug!( target: LOG_TARGET, @@ -806,12 +763,21 @@ async fn request_collation( "Requesting collation", ); + let maybe_candidate_hash = + prospective_candidate.as_ref().map(ProspectiveCandidate::candidate_hash); + per_relay_parent.collations.status = CollationStatus::Fetching; + per_relay_parent + .collations + .fetching_from + .replace((collator_id, maybe_candidate_hash)); + sender .send_message(NetworkBridgeTxMessage::SendRequests( vec![requests], IfDisconnected::ImmediateError, )) .await; + Ok(()) } /// Networking message has been received. @@ -820,12 +786,18 @@ async fn process_incoming_peer_message( ctx: &mut Context, state: &mut State, origin: PeerId, - msg: protocol_v1::CollatorProtocolMessage, + msg: Versioned< + protocol_v1::CollatorProtocolMessage, + protocol_vstaging::CollatorProtocolMessage, + >, ) { - use protocol_v1::CollatorProtocolMessage::*; + use protocol_v1::CollatorProtocolMessage as V1; + use protocol_vstaging::CollatorProtocolMessage as VStaging; use sp_runtime::traits::AppVerify; + match msg { - Declare(collator_id, para_id, signature) => { + Versioned::V1(V1::Declare(collator_id, para_id, signature)) | + Versioned::VStaging(VStaging::Declare(collator_id, para_id, signature)) => { if collator_peer_id(&state.peer_data, &collator_id).is_some() { modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; return @@ -850,7 +822,7 @@ async fn process_incoming_peer_message( target: LOG_TARGET, peer_id = ?origin, ?para_id, - "Peer is not in the collating state", + "Peer is already in the collating state", ); modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; return @@ -867,7 +839,7 @@ async fn process_incoming_peer_message( return } - if state.active_paras.is_current(¶_id) { + if state.current_assignments.contains_key(¶_id) { gum::debug!( target: LOG_TARGET, peer_id = ?origin, @@ -891,165 +863,383 @@ async fn process_incoming_peer_message( disconnect_peer(ctx.sender(), origin).await; } }, - AdvertiseCollation(relay_parent) => { - let _span = state - .span_per_relay_parent - .get(&relay_parent) - .map(|s| s.child("advertise-collation")); - if !state.view.contains(&relay_parent) { + Versioned::V1(V1::AdvertiseCollation(relay_parent)) => + if let Err(err) = + handle_advertisement(ctx.sender(), state, relay_parent, &origin, None).await + { gum::debug!( target: LOG_TARGET, peer_id = ?origin, ?relay_parent, - "Advertise collation out of view", + error = ?err, + "Rejected v1 advertisement", ); - modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; - return + if let Some(rep) = err.reputation_changes() { + modify_reputation(ctx.sender(), origin.clone(), rep).await; + } + }, + Versioned::VStaging(VStaging::AdvertiseCollation { + relay_parent, + candidate_hash, + parent_head_data_hash, + }) => + if let Err(err) = handle_advertisement( + ctx.sender(), + state, + relay_parent, + &origin, + Some((candidate_hash, parent_head_data_hash)), + ) + .await + { + gum::debug!( + target: LOG_TARGET, + peer_id = ?origin, + ?relay_parent, + ?candidate_hash, + error = ?err, + "Rejected vstaging advertisement", + ); + + if let Some(rep) = err.reputation_changes() { + modify_reputation(ctx.sender(), origin.clone(), rep).await; + } + }, + Versioned::V1(V1::CollationSeconded(..)) | + Versioned::VStaging(VStaging::CollationSeconded(..)) => { + gum::warn!( + target: LOG_TARGET, + peer_id = ?origin, + "Unexpected `CollationSeconded` message, decreasing reputation", + ); + + modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; + }, + } +} + +async fn is_seconding_allowed( + _sender: &mut Sender, + _relay_parent: Hash, + _candidate_hash: CandidateHash, + _parent_head_data_hash: Hash, + _para_id: ParaId, + _active_leaves: impl IntoIterator, +) -> Option +where + Sender: CollatorProtocolSenderTrait, +{ + // TODO https://github.com/paritytech/polkadot/issues/5923 + Some(true) +} + +#[derive(Debug)] +enum AdvertisementError { + /// Relay parent is unknown. + RelayParentUnknown, + /// Peer is not present in the subsystem state. + UnknownPeer, + /// Peer has not declared its para id. + UndeclaredCollator, + /// We're assigned to a different para at the given relay parent. + InvalidAssignment, + /// Collator is trying to build on top of occupied core + /// when async backing is disabled. + CoreOccupied, + /// An advertisement format doesn't match the relay parent. + ProtocolMismatch, + /// Para reached a limit of seconded candidates for this relay parent. + SecondedLimitReached, + /// Advertisement is invalid. + Invalid(InsertAdvertisementError), + /// Failed to query prospective parachains subsystem. + ProspectiveParachainsUnavailable, +} + +impl AdvertisementError { + fn reputation_changes(&self) -> Option { + use AdvertisementError::*; + match self { + InvalidAssignment => Some(COST_WRONG_PARA), + RelayParentUnknown | UndeclaredCollator | CoreOccupied | Invalid(_) => + Some(COST_UNEXPECTED_MESSAGE), + UnknownPeer | + ProtocolMismatch | + SecondedLimitReached | + ProspectiveParachainsUnavailable => None, + } + } +} + +async fn handle_advertisement( + sender: &mut Sender, + state: &mut State, + relay_parent: Hash, + peer_id: &PeerId, + prospective_candidate: Option<(CandidateHash, Hash)>, +) -> std::result::Result<(), AdvertisementError> +where + Sender: CollatorProtocolSenderTrait, +{ + let _span = state + .span_per_relay_parent + .get(&relay_parent) + .map(|s| s.child("advertise-collation")); + + let per_relay_parent = state + .per_relay_parent + .get_mut(&relay_parent) + .ok_or(AdvertisementError::RelayParentUnknown)?; + + let relay_parent_mode = per_relay_parent.prospective_parachains_mode; + let assignment = &per_relay_parent.assignment; + + let peer_data = state.peer_data.get_mut(&peer_id).ok_or(AdvertisementError::UnknownPeer)?; + let collator_para_id = + peer_data.collating_para().ok_or(AdvertisementError::UndeclaredCollator)?; + + match assignment.current { + Some((id, core_state)) if id == collator_para_id => { + // Disallow building on top occupied core if async + // backing is disabled. + if !relay_parent_mode.is_enabled() && core_state.is_occupied() { + return Err(AdvertisementError::CoreOccupied) } + }, + _ if assignment.implicit.contains(&collator_para_id) => { + // This relay parent is a part of implicit ancestry, + // thus async backing is enabled. + }, + _ => return Err(AdvertisementError::InvalidAssignment), + }; - let peer_data = match state.peer_data.get_mut(&origin) { - None => { - gum::debug!( - target: LOG_TARGET, - peer_id = ?origin, - ?relay_parent, - "Advertise collation message has been received from an unknown peer", - ); - modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; - return - }, - Some(p) => p, - }; + // TODO: only fetch a collation if it's built on top of backed nodes in fragment tree. + // https://github.com/paritytech/polkadot/issues/5923 + let is_seconding_allowed = match (relay_parent_mode, prospective_candidate) { + (ProspectiveParachainsMode::Disabled, _) => true, + (ProspectiveParachainsMode::Enabled, Some((candidate_hash, parent_head_data_hash))) => { + let active_leaves = state.active_leaves.keys().copied(); + is_seconding_allowed( + sender, + relay_parent, + candidate_hash, + parent_head_data_hash, + collator_para_id, + active_leaves, + ) + .await + .ok_or(AdvertisementError::ProspectiveParachainsUnavailable)? + }, + _ => return Err(AdvertisementError::ProtocolMismatch), + }; - match peer_data.insert_advertisement(relay_parent, &state.view) { - Ok((id, para_id)) => { - gum::debug!( + if !is_seconding_allowed { + // TODO + return Ok(()) + } + + let candidate_hash = prospective_candidate.map(|(hash, ..)| hash); + let insert_result = peer_data.insert_advertisement( + relay_parent, + relay_parent_mode, + candidate_hash, + &state.implicit_view, + &state.active_leaves, + ); + + match insert_result { + Ok((id, para_id)) => { + gum::debug!( + target: LOG_TARGET, + peer_id = ?peer_id, + %para_id, + ?relay_parent, + "Received advertise collation", + ); + let prospective_candidate = + prospective_candidate.map(|(candidate_hash, parent_head_data_hash)| { + ProspectiveCandidate { candidate_hash, parent_head_data_hash } + }); + + let collations = &mut per_relay_parent.collations; + if !collations.is_seconded_limit_reached(relay_parent_mode, collator_para_id) { + return Err(AdvertisementError::SecondedLimitReached) + } + + let pending_collation = + PendingCollation::new(relay_parent, para_id, peer_id, prospective_candidate); + + match collations.status { + CollationStatus::Fetching | CollationStatus::WaitingOnValidation => { + gum::trace!( target: LOG_TARGET, - peer_id = ?origin, + peer_id = ?peer_id, %para_id, ?relay_parent, - "Received advertise collation", + "Added collation to the pending list" ); - - let pending_collation = PendingCollation::new(relay_parent, ¶_id, &origin); - - let collations = - state.collations_per_relay_parent.entry(relay_parent).or_default(); - - match collations.status { - CollationStatus::Fetching | CollationStatus::WaitingOnValidation => { - gum::trace!( - target: LOG_TARGET, - peer_id = ?origin, - %para_id, - ?relay_parent, - "Added collation to the pending list" - ); - collations.unfetched_collations.push((pending_collation, id)); - }, - CollationStatus::Waiting => { - collations.status = CollationStatus::Fetching; - collations.waiting_collation = Some(id.clone()); - - fetch_collation(ctx.sender(), state, pending_collation.clone(), id) - .await; - }, - CollationStatus::Seconded => { - gum::trace!( - target: LOG_TARGET, - peer_id = ?origin, - %para_id, - ?relay_parent, - "Valid seconded collation" - ); - }, - } + collations.waiting_queue.push_back((pending_collation, id)); }, - Err(error) => { - gum::debug!( + CollationStatus::Waiting => { + let _ = fetch_collation(sender, state, pending_collation, id).await; + }, + CollationStatus::Seconded if relay_parent_mode.is_enabled() => { + // Limit is not reached, it's allowed to second another + // collation. + let _ = fetch_collation(sender, state, pending_collation, id).await; + }, + CollationStatus::Seconded => { + gum::trace!( target: LOG_TARGET, - peer_id = ?origin, + peer_id = ?peer_id, + %para_id, ?relay_parent, - ?error, - "Invalid advertisement", + ?relay_parent_mode, + "A collation has already been seconded", ); - - modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await; }, } }, - CollationSeconded(_, _) => { - gum::warn!( - target: LOG_TARGET, - peer_id = ?origin, - "Unexpected `CollationSeconded` message, decreasing reputation", - ); + Err(InsertAdvertisementError::ProtocolMismatch) => { + // Checked above. + return Err(AdvertisementError::ProtocolMismatch) }, + Err(error) => return Err(AdvertisementError::Invalid(error)), } -} -/// A leaf has become inactive so we want to -/// - Cancel all ongoing collation requests that are on top of that leaf. -/// - Remove all stored collations relevant to that leaf. -async fn remove_relay_parent(state: &mut State, relay_parent: Hash) -> Result<()> { - state.requested_collations.retain(|k, _| k.relay_parent != relay_parent); - - state.pending_candidates.retain(|k, _| k != &relay_parent); - - state.collations_per_relay_parent.remove(&relay_parent); Ok(()) } /// Our view has changed. -#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] -async fn handle_our_view_change( - ctx: &mut Context, +async fn handle_our_view_change( + sender: &mut Sender, state: &mut State, keystore: &SyncCryptoStorePtr, view: OurView, -) -> Result<()> { - let old_view = std::mem::replace(&mut state.view, view); - - let added: HashMap> = state - .view - .span_per_head() - .iter() - .filter(|v| !old_view.contains(&v.0)) - .map(|v| (v.0.clone(), v.1.clone())) - .collect(); - - added.into_iter().for_each(|(h, s)| { - state.span_per_relay_parent.insert(h, PerLeafSpan::new(s, "validator-side")); - }); - - let added = state.view.difference(&old_view).cloned().collect::>(); - let removed = old_view.difference(&state.view).cloned().collect::>(); - - for removed in removed.iter().cloned() { - remove_relay_parent(state, removed).await?; - state.span_per_relay_parent.remove(&removed); +) -> Result<()> +where + Sender: CollatorProtocolSenderTrait, +{ + let current_leaves = state.active_leaves.clone(); + + let removed = current_leaves.iter().filter(|(h, _)| !view.contains(*h)); + let added = view.iter().filter(|h| !current_leaves.contains_key(h)); + + for leaf in added { + let mode = prospective_parachains_mode(sender, *leaf).await?; + + if let Some(span) = view.span_per_head().get(leaf).cloned() { + let per_leaf_span = PerLeafSpan::new(span, "validator-side"); + state.span_per_relay_parent.insert(*leaf, per_leaf_span); + } + + let mut per_relay_parent = PerRelayParent::new(mode); + assign_incoming( + sender, + &mut per_relay_parent.assignment, + &mut state.current_assignments, + keystore, + *leaf, + ) + .await?; + + state.active_leaves.insert(*leaf, mode); + + let mut implicit_assignment = + Vec::from_iter(per_relay_parent.assignment.current.map(|(para, _)| para)); + state.per_relay_parent.insert(*leaf, per_relay_parent); + + if mode.is_enabled() { + state + .implicit_view + .activate_leaf(sender, *leaf) + .await + .map_err(Error::ImplicitViewFetchError)?; + + // Order is always descending. + let allowed_ancestry = state + .implicit_view + .known_allowed_relay_parents_under(leaf, None) + .unwrap_or_default(); + for block_hash in allowed_ancestry { + let entry = match state.per_relay_parent.entry(*block_hash) { + Entry::Vacant(entry) => { + let mut per_relay_parent = + PerRelayParent::new(ProspectiveParachainsMode::Enabled); + assign_incoming( + sender, + &mut per_relay_parent.assignment, + &mut state.current_assignments, + keystore, + *block_hash, + ) + .await?; + + entry.insert(per_relay_parent) + }, + Entry::Occupied(entry) => entry.into_mut(), + }; + + let current = entry.assignment.current.map(|(para, _)| para); + let implicit = &mut entry.assignment.implicit; + + // Extend implicitly assigned parachains. + for para in &implicit_assignment { + if !implicit.contains(para) { + implicit.push(*para); + } + } + // Current assignment propagates to parents, meaning that a parachain + // we're assigned to in fresh blocks can submit collations built + // on top of relay parents in the allowed ancestry, but not vice versa. + implicit_assignment.extend(current); + } + } } - state.active_paras.assign_incoming(ctx.sender(), keystore, added).await; - state.active_paras.remove_outgoing(removed); + for (removed, mode) in removed { + state.active_leaves.remove(removed); + // If the leaf is deactivated it still may stay in the view as a part + // of implicit ancestry. Only update the state after the hash is actually + // pruned from the block info storage. + let pruned = if mode.is_enabled() { + state.implicit_view.deactivate_leaf(*removed) + } else { + vec![*removed] + }; + + for removed in pruned { + if let Some(per_relay_parent) = state.per_relay_parent.remove(&removed) { + remove_outgoing(&mut state.current_assignments, per_relay_parent); + } + + state.requested_collations.retain(|k, _| k.relay_parent != removed); + state.fetched_candidates.retain(|k, _| k.relay_parent != removed); + state.span_per_relay_parent.remove(&removed); + } + } for (peer_id, peer_data) in state.peer_data.iter_mut() { - peer_data.prune_old_advertisements(&state.view); + peer_data.prune_old_advertisements( + &state.implicit_view, + &state.active_leaves, + &state.per_relay_parent, + ); // Disconnect peers who are not relevant to our current or next para. // // If the peer hasn't declared yet, they will be disconnected if they do not // declare. if let Some(para_id) = peer_data.collating_para() { - if !state.active_paras.is_current(¶_id) { + if !state.current_assignments.contains_key(¶_id) { gum::trace!( target: LOG_TARGET, ?peer_id, ?para_id, "Disconnecting peer on view change (not current parachain id)" ); - disconnect_peer(ctx.sender(), peer_id.clone()).await; + disconnect_peer(sender, peer_id.clone()).await; } } } @@ -1068,8 +1258,26 @@ async fn handle_network_msg( use NetworkBridgeEvent::*; match bridge_message { - PeerConnected(peer_id, _role, _version, _) => { - state.peer_data.entry(peer_id).or_default(); + PeerConnected(peer_id, observed_role, protocol_version, _) => { + let version = match protocol_version.try_into() { + Ok(version) => version, + Err(err) => { + // Network bridge is expected to handle this. + gum::error!( + target: LOG_TARGET, + ?peer_id, + ?observed_role, + ?err, + "Unsupported protocol version" + ); + return Ok(()) + }, + }; + state.peer_data.entry(peer_id).or_insert_with(|| PeerData { + view: View::default(), + state: PeerState::Connected(Instant::now()), + version, + }); state.metrics.note_collator_peer_count(state.peer_data.len()); }, PeerDisconnected(peer_id) => { @@ -1080,15 +1288,14 @@ async fn handle_network_msg( // impossible! }, PeerViewChange(peer_id, view) => { - handle_peer_view_change(state, peer_id, view).await?; + handle_peer_view_change(state, peer_id, view); }, OurViewChange(view) => { - handle_our_view_change(ctx, state, keystore, view).await?; + handle_our_view_change(ctx.sender(), state, keystore, view).await?; }, - PeerMessage(remote, Versioned::V1(msg)) => { + PeerMessage(remote, msg) => { process_incoming_peer_message(ctx, state, remote, msg).await; }, - PeerMessage(_, Versioned::VStaging(_msg)) => {}, } Ok(()) @@ -1114,7 +1321,7 @@ async fn process_msg( "CollateOn message is not expected on the validator side of the protocol", ); }, - DistributeCollation(_, _, _) => { + DistributeCollation(..) => { gum::warn!( target: LOG_TARGET, "DistributeCollation message is not expected on the validator side of the protocol", @@ -1133,15 +1340,50 @@ async fn process_msg( } }, Seconded(parent, stmt) => { - if let Some(collation_event) = state.pending_candidates.remove(&parent) { + let receipt = match stmt.payload() { + Statement::Seconded(receipt) => receipt, + Statement::Valid(_) => { + gum::warn!( + target: LOG_TARGET, + ?stmt, + relay_parent = %parent, + "Seconded message received with a `Valid` statement", + ); + return + }, + }; + let fetched_collation = FetchedCollation::from(&receipt.to_plain()); + if let Some(collation_event) = state.fetched_candidates.remove(&fetched_collation) { let (collator_id, pending_collation) = collation_event; - let PendingCollation { relay_parent, peer_id, .. } = pending_collation; - note_good_collation(ctx.sender(), &state.peer_data, collator_id).await; - notify_collation_seconded(ctx.sender(), peer_id, relay_parent, stmt).await; + let PendingCollation { + relay_parent, peer_id, para_id, prospective_candidate, .. + } = pending_collation; + note_good_collation(ctx.sender(), &state.peer_data, collator_id.clone()).await; + if let Some(peer_data) = state.peer_data.get(&peer_id) { + notify_collation_seconded( + ctx.sender(), + peer_id, + peer_data.version, + relay_parent, + stmt, + ) + .await; + } - if let Some(collations) = state.collations_per_relay_parent.get_mut(&parent) { - collations.status = CollationStatus::Seconded; + if let Some(state) = state.per_relay_parent.get_mut(&parent) { + state.collations.status = CollationStatus::Seconded; + state.collations.note_seconded(para_id); } + // If async backing is enabled, make an attempt to fetch next collation. + let maybe_candidate_hash = + prospective_candidate.as_ref().map(ProspectiveCandidate::candidate_hash); + dequeue_next_collation_and_fetch( + ctx, + state, + parent, + (collator_id, maybe_candidate_hash), + ) + .await; } else { gum::debug!( target: LOG_TARGET, @@ -1151,7 +1393,9 @@ async fn process_msg( } }, Invalid(parent, candidate_receipt) => { - let id = match state.pending_candidates.entry(parent) { + let fetched_collation = FetchedCollation::from(&candidate_receipt); + let candidate_hash = fetched_collation.candidate_hash; + let id = match state.fetched_candidates.entry(fetched_collation) { Entry::Occupied(entry) if entry.get().1.commitments_hash == Some(candidate_receipt.commitments_hash) => @@ -1170,7 +1414,7 @@ async fn process_msg( report_collator(ctx.sender(), &state.peer_data, id.clone()).await; - dequeue_next_collation_and_fetch(ctx, state, parent, id).await; + dequeue_next_collation_and_fetch(ctx, state, parent, (id, Some(candidate_hash))).await; }, } } @@ -1212,17 +1456,47 @@ pub(crate) async fn run( disconnect_inactive_peers(ctx.sender(), &eviction_policy, &state.peer_data).await; } res = state.collation_fetches.select_next_some() => { - handle_collation_fetched_result(&mut ctx, &mut state, res).await; + let (collator_id, pc) = res.0.clone(); + if let Err(err) = kick_off_seconding(&mut ctx, &mut state, res).await { + gum::warn!( + target: LOG_TARGET, + relay_parent = ?pc.relay_parent, + para_id = ?pc.para_id, + peer_id = ?pc.peer_id, + error = %err, + "Seconding aborted due to an error", + ); + + if err.is_malicious() { + // Report malicious peer. + modify_reputation(ctx.sender(), pc.peer_id, COST_REPORT_BAD).await; + } + let maybe_candidate_hash = + pc.prospective_candidate.as_ref().map(ProspectiveCandidate::candidate_hash); + dequeue_next_collation_and_fetch( + &mut ctx, + &mut state, + pc.relay_parent, + (collator_id, maybe_candidate_hash), + ) + .await; + } } res = state.collation_fetch_timeouts.select_next_some() => { - let (collator_id, relay_parent) = res; + let (collator_id, maybe_candidate_hash, relay_parent) = res; gum::debug!( target: LOG_TARGET, ?relay_parent, ?collator_id, "Timeout hit - already seconded?" ); - dequeue_next_collation_and_fetch(&mut ctx, &mut state, relay_parent, collator_id).await; + dequeue_next_collation_and_fetch( + &mut ctx, + &mut state, + relay_parent, + (collator_id, maybe_candidate_hash), + ) + .await; } _ = check_collations_stream.next() => { let reputation_changes = poll_requests( @@ -1271,142 +1545,151 @@ async fn dequeue_next_collation_and_fetch( ctx: &mut Context, state: &mut State, relay_parent: Hash, - // The collator we tried to fetch from last. - previous_fetch: CollatorId, + // The collator we tried to fetch from last, optionally which candidate. + previous_fetch: (CollatorId, Option), ) { - if let Some((next, id)) = state - .collations_per_relay_parent - .get_mut(&relay_parent) - .and_then(|c| c.get_next_collation_to_fetch(Some(&previous_fetch))) - { + while let Some((next, id)) = state.per_relay_parent.get_mut(&relay_parent).and_then(|state| { + state + .collations + .get_next_collation_to_fetch(&previous_fetch, state.prospective_parachains_mode) + }) { gum::debug!( target: LOG_TARGET, ?relay_parent, ?id, "Successfully dequeued next advertisement - fetching ..." ); - fetch_collation(ctx.sender(), state, next, id).await; - } else { - gum::debug!( - target: LOG_TARGET, - ?relay_parent, - previous_collator = ?previous_fetch, - "No collations are available to fetch" - ); + if let Err(err) = fetch_collation(ctx.sender(), state, next, id).await { + gum::debug!( + target: LOG_TARGET, + relay_parent = ?next.relay_parent, + para_id = ?next.para_id, + peer_id = ?next.peer_id, + error = %err, + "Failed to request a collation, dequeueing next one", + ); + } else { + break + } } } -#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] -async fn request_persisted_validation_data( - ctx: &mut Context, +async fn request_persisted_validation_data( + sender: &mut Sender, relay_parent: Hash, para_id: ParaId, -) -> Option { - // TODO [https://github.com/paritytech/polkadot/issues/5054] - // - // As of https://github.com/paritytech/polkadot/pull/5557 the - // `Second` message requires the `PersistedValidationData` to be - // supplied. - // - // Without asynchronous backing, this can be easily fetched from the - // chain state. - // - // This assumes the core is _scheduled_, in keeping with the effective - // current behavior. If the core is occupied, we simply don't return - // anything. Likewise with runtime API errors, which are rare. - let res = polkadot_node_subsystem_util::request_persisted_validation_data( +) -> std::result::Result, SecondingError> +where + Sender: CollatorProtocolSenderTrait, +{ + // The core is guaranteed to be scheduled since we accepted the advertisement. + polkadot_node_subsystem_util::request_persisted_validation_data( relay_parent, para_id, OccupiedCoreAssumption::Free, - ctx.sender(), + sender, ) .await - .await; + .await + .map_err(SecondingError::CancelledRuntimePersistedValidationData)? + .map_err(SecondingError::RuntimeApi) +} - match res { - Ok(Ok(Some(pvd))) => Some(pvd), - _ => None, - } +async fn request_prospective_validation_data( + sender: &mut Sender, + candidate_relay_parent: Hash, + parent_head_data_hash: Hash, + para_id: ParaId, +) -> std::result::Result, SecondingError> +where + Sender: CollatorProtocolSenderTrait, +{ + let (tx, rx) = oneshot::channel(); + + let request = + ProspectiveValidationDataRequest { para_id, candidate_relay_parent, parent_head_data_hash }; + + sender + .send_message(ProspectiveParachainsMessage::GetProspectiveValidationData(request, tx)) + .await; + + rx.await.map_err(SecondingError::CancelledProspectiveValidationData) } /// Handle a fetched collation result. #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)] -async fn handle_collation_fetched_result( +async fn kick_off_seconding( ctx: &mut Context, state: &mut State, (mut collation_event, res): PendingCollationFetch, -) { - // If no prior collation for this relay parent has been seconded, then - // memorize the `collation_event` for that `relay_parent`, such that we may - // notify the collator of their successful second backing +) -> std::result::Result<(), SecondingError> { let relay_parent = collation_event.1.relay_parent; + let para_id = collation_event.1.para_id; - let (candidate_receipt, pov) = match res { - Ok(res) => res, - Err(e) => { - gum::debug!( + let per_relay_parent = match state.per_relay_parent.get_mut(&relay_parent) { + Some(state) => state, + None => { + // Relay parent went out of view, not an error. + gum::trace!( target: LOG_TARGET, - relay_parent = ?collation_event.1.relay_parent, - para_id = ?collation_event.1.para_id, - peer_id = ?collation_event.1.peer_id, - collator_id = ?collation_event.0, - error = ?e, - "Failed to fetch collation.", + relay_parent = ?relay_parent, + "Fetched collation for a parent out of view", ); - - dequeue_next_collation_and_fetch(ctx, state, relay_parent, collation_event.0).await; - return + return Ok(()) }, }; + let collations = &mut per_relay_parent.collations; + let relay_parent_mode = per_relay_parent.prospective_parachains_mode; - if let Some(collations) = state.collations_per_relay_parent.get_mut(&relay_parent) { - if let CollationStatus::Seconded = collations.status { - gum::debug!( - target: LOG_TARGET, - ?relay_parent, - "Already seconded - no longer interested in collation fetch result." - ); - return - } - collations.status = CollationStatus::WaitingOnValidation; - } + let (candidate_receipt, pov) = res?; - if let Entry::Vacant(entry) = state.pending_candidates.entry(relay_parent) { + let fetched_collation = FetchedCollation::from(&candidate_receipt); + if let Entry::Vacant(entry) = state.fetched_candidates.entry(fetched_collation) { collation_event.1.commitments_hash = Some(candidate_receipt.commitments_hash); - if let Some(pvd) = request_persisted_validation_data( - ctx, - candidate_receipt.descriptor().relay_parent, - candidate_receipt.descriptor().para_id, - ) - .await - { - // TODO [https://github.com/paritytech/polkadot/issues/5054] - // - // If PVD isn't available (core occupied) then we'll silently - // just not second this. But prior to asynchronous backing - // we wouldn't second anyway because the core is occupied. - // - // The proper refactoring would be to accept declares from collators - // but not even fetch from them if the core is occupied. Given 5054, - // there's no reason to do this right now. - ctx.send_message(CandidateBackingMessage::Second( - relay_parent.clone(), - candidate_receipt, - pvd, - pov, - )) - .await; + let pvd = match (relay_parent_mode, collation_event.1.prospective_candidate) { + ( + ProspectiveParachainsMode::Enabled, + Some(ProspectiveCandidate { parent_head_data_hash, .. }), + ) => + request_prospective_validation_data( + ctx.sender(), + relay_parent, + parent_head_data_hash, + para_id, + ) + .await?, + (ProspectiveParachainsMode::Disabled, _) => + request_persisted_validation_data( + ctx.sender(), + candidate_receipt.descriptor().relay_parent, + candidate_receipt.descriptor().para_id, + ) + .await?, + _ => { + // `handle_advertisement` checks for protocol mismatch. + return Ok(()) + }, } + .ok_or(SecondingError::PersistedValidationDataNotFound)?; + + fetched_collation_sanity_check(&collation_event.1, &candidate_receipt, &pvd)?; + + ctx.send_message(CandidateBackingMessage::Second( + relay_parent, + candidate_receipt, + pvd, + pov, + )) + .await; + // There's always a single collation being fetched at any moment of time. + // In case of a failure, we reset the status back to waiting. + collations.status = CollationStatus::WaitingOnValidation; entry.insert(collation_event); + Ok(()) } else { - gum::trace!( - target: LOG_TARGET, - ?relay_parent, - candidate = ?candidate_receipt.hash(), - "Trying to insert a pending candidate failed, because there is already one.", - ) + Err(SecondingError::Duplicate) } } @@ -1519,7 +1802,7 @@ async fn poll_collation_response( ); CollationFetchResult::Error(None) }, - Ok(CollationFetchingResponse::Collation(receipt, _)) + Ok(request_v1::CollationFetchingResponse::Collation(receipt, _)) if receipt.descriptor().para_id != pending_collation.para_id => { gum::debug!( @@ -1532,7 +1815,7 @@ async fn poll_collation_response( CollationFetchResult::Error(Some(COST_WRONG_PARA)) }, - Ok(CollationFetchingResponse::Collation(receipt, pov)) => { + Ok(request_v1::CollationFetchingResponse::Collation(receipt, pov)) => { gum::debug!( target: LOG_TARGET, para_id = %pending_collation.para_id, diff --git a/node/network/collator-protocol/src/validator_side/tests.rs b/node/network/collator-protocol/src/validator_side/tests/mod.rs similarity index 73% rename from node/network/collator-protocol/src/validator_side/tests.rs rename to node/network/collator-protocol/src/validator_side/tests/mod.rs index 578100663904..ffc8796d3450 100644 --- a/node/network/collator-protocol/src/validator_side/tests.rs +++ b/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -40,9 +40,22 @@ use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt_bad_sig, dummy_hash, }; +mod prospective_parachains; + const ACTIVITY_TIMEOUT: Duration = Duration::from_millis(500); const DECLARE_TIMEOUT: Duration = Duration::from_millis(25); +const API_VERSION_PROSPECTIVE_DISABLED: u32 = 2; + +fn dummy_pvd() -> PersistedValidationData { + PersistedValidationData { + parent_head: HeadData(vec![7, 8, 9]), + relay_parent_number: 5, + max_pov_size: 1024, + relay_parent_storage_root: Default::default(), + } +} + #[derive(Clone)] struct TestState { chain_ids: Vec, @@ -117,6 +130,7 @@ type VirtualOverseer = test_helpers::TestSubsystemContextHandle>(test: impl FnOnce(TestHarness) -> T) { @@ -138,9 +152,10 @@ fn test_harness>(test: impl FnOnce(TestHarne ) .unwrap(); + let keystore: SyncCryptoStorePtr = Arc::new(keystore); let subsystem = run( context, - Arc::new(keystore), + keystore.clone(), crate::CollatorEvictionPolicy { inactive_collator: ACTIVITY_TIMEOUT, undeclared: DECLARE_TIMEOUT, @@ -148,7 +163,7 @@ fn test_harness>(test: impl FnOnce(TestHarne Metrics::default(), ); - let test_fut = test(TestHarness { virtual_overseer }); + let test_fut = test(TestHarness { virtual_overseer, keystore }); futures::pin_mut!(test_fut); futures::pin_mut!(subsystem); @@ -245,30 +260,37 @@ async fn assert_candidate_backing_second( expected_relay_parent: Hash, expected_para_id: ParaId, expected_pov: &PoV, + mode: ProspectiveParachainsMode, ) -> CandidateReceipt { - // TODO [https://github.com/paritytech/polkadot/issues/5054] - // - // While collator protocol isn't updated, it's expected to receive - // a Runtime API request for persisted validation data. - let pvd = PersistedValidationData { - parent_head: HeadData(vec![7, 8, 9]), - relay_parent_number: 5, - max_pov_size: 1024, - relay_parent_storage_root: Default::default(), - }; - - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - hash, - RuntimeApiRequest::PersistedValidationData(para_id, assumption, tx), - )) => { - assert_eq!(expected_relay_parent, hash); - assert_eq!(expected_para_id, para_id); - assert_eq!(OccupiedCoreAssumption::Free, assumption); - tx.send(Ok(Some(pvd.clone()))).unwrap(); - } - ); + let pvd = dummy_pvd(); + + // Depending on relay parent mode pvd will be either requested + // from the Runtime API or Prospective Parachains. + let msg = overseer_recv(virtual_overseer).await; + match mode { + ProspectiveParachainsMode::Disabled => assert_matches!( + msg, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + hash, + RuntimeApiRequest::PersistedValidationData(para_id, assumption, tx), + )) => { + assert_eq!(expected_relay_parent, hash); + assert_eq!(expected_para_id, para_id); + assert_eq!(OccupiedCoreAssumption::Free, assumption); + tx.send(Ok(Some(pvd.clone()))).unwrap(); + } + ), + ProspectiveParachainsMode::Enabled => assert_matches!( + msg, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::GetProspectiveValidationData(request, tx), + ) => { + assert_eq!(expected_relay_parent, request.candidate_relay_parent); + assert_eq!(expected_para_id, request.para_id); + tx.send(Some(pvd.clone())).unwrap(); + } + ), + } assert_matches!( overseer_recv(virtual_overseer).await, @@ -306,6 +328,7 @@ async fn assert_fetch_collation_request( virtual_overseer: &mut VirtualOverseer, relay_parent: Hash, para_id: ParaId, + candidate_hash: Option, ) -> ResponseSender { assert_matches!( overseer_recv(virtual_overseer).await, @@ -313,14 +336,26 @@ async fn assert_fetch_collation_request( ) => { let req = reqs.into_iter().next() .expect("There should be exactly one request"); - match req { - Requests::CollationFetchingV1(req) => { - let payload = req.payload; - assert_eq!(payload.relay_parent, relay_parent); - assert_eq!(payload.para_id, para_id); - req.pending_response - } - _ => panic!("Unexpected request"), + match candidate_hash { + None => assert_matches!( + req, + Requests::CollationFetchingV1(req) => { + let payload = req.payload; + assert_eq!(payload.relay_parent, relay_parent); + assert_eq!(payload.para_id, para_id); + req.pending_response + } + ), + Some(candidate_hash) => assert_matches!( + req, + Requests::CollationFetchingVStaging(req) => { + let payload = req.payload; + assert_eq!(payload.relay_parent, relay_parent); + assert_eq!(payload.para_id, para_id); + assert_eq!(payload.candidate_hash, candidate_hash); + req.pending_response + } + ), } }) } @@ -331,27 +366,38 @@ async fn connect_and_declare_collator( peer: PeerId, collator: CollatorPair, para_id: ParaId, + version: CollationVersion, ) { overseer_send( virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( peer.clone(), ObservedRole::Full, - CollationVersion::V1.into(), + version.into(), None, )), ) .await; - overseer_send( - virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( - peer.clone(), - Versioned::V1(protocol_v1::CollatorProtocolMessage::Declare( + let wire_message = match version { + CollationVersion::V1 => Versioned::V1(protocol_v1::CollatorProtocolMessage::Declare( + collator.public(), + para_id, + collator.sign(&protocol_v1::declare_signature_payload(&peer)), + )), + CollationVersion::VStaging => + Versioned::VStaging(protocol_vstaging::CollatorProtocolMessage::Declare( collator.public(), para_id, collator.sign(&protocol_v1::declare_signature_payload(&peer)), )), + }; + + overseer_send( + virtual_overseer, + CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( + peer, + wire_message, )), ) .await; @@ -362,24 +408,48 @@ async fn advertise_collation( virtual_overseer: &mut VirtualOverseer, peer: PeerId, relay_parent: Hash, + candidate: Option<(CandidateHash, Hash)>, // Candidate hash + parent head data hash. ) { + let wire_message = match candidate { + Some((candidate_hash, parent_head_data_hash)) => + Versioned::VStaging(protocol_vstaging::CollatorProtocolMessage::AdvertiseCollation { + relay_parent, + candidate_hash, + parent_head_data_hash, + }), + None => + Versioned::V1(protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent)), + }; overseer_send( virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( peer, - Versioned::V1(protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent)), + wire_message, )), ) .await; } +async fn assert_runtime_version_request(virtual_overseer: &mut VirtualOverseer, hash: Hash) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + relay_parent, + RuntimeApiRequest::Version(tx) + )) => { + assert_eq!(relay_parent, hash); + tx.send(Ok(API_VERSION_PROSPECTIVE_DISABLED)).unwrap(); + } + ); +} + // As we receive a relevant advertisement act on it and issue a collation request. #[test] fn act_on_advertisement() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let pair = CollatorPair::generate().0; gum::trace!("activating"); @@ -392,6 +462,7 @@ fn act_on_advertisement() { ) .await; + assert_runtime_version_request(&mut virtual_overseer, test_state.relay_parent).await; respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; let peer_b = PeerId::random(); @@ -401,15 +472,75 @@ fn act_on_advertisement() { peer_b.clone(), pair.clone(), test_state.chain_ids[0], + CollationVersion::V1, ) .await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent, None) + .await; assert_fetch_collation_request( &mut virtual_overseer, test_state.relay_parent, test_state.chain_ids[0], + None, + ) + .await; + + virtual_overseer + }); +} + +/// Tests that validator side works with vstaging network protocol +/// before async backing is enabled. +#[test] +fn act_on_advertisement_vstaging() { + let test_state = TestState::default(); + + test_harness(|test_harness| async move { + let TestHarness { mut virtual_overseer, .. } = test_harness; + + let pair = CollatorPair::generate().0; + gum::trace!("activating"); + + overseer_send( + &mut virtual_overseer, + CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( + our_view![test_state.relay_parent], + )), + ) + .await; + + assert_runtime_version_request(&mut virtual_overseer, test_state.relay_parent).await; + respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + + let peer_b = PeerId::random(); + + connect_and_declare_collator( + &mut virtual_overseer, + peer_b.clone(), + pair.clone(), + test_state.chain_ids[0], + CollationVersion::VStaging, + ) + .await; + + let candidate_hash = CandidateHash::default(); + let parent_head_data_hash = Hash::zero(); + // vstaging advertisement. + advertise_collation( + &mut virtual_overseer, + peer_b.clone(), + test_state.relay_parent, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + + assert_fetch_collation_request( + &mut virtual_overseer, + test_state.relay_parent, + test_state.chain_ids[0], + Some(candidate_hash), ) .await; @@ -423,7 +554,7 @@ fn collator_reporting_works() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; overseer_send( &mut virtual_overseer, @@ -433,6 +564,8 @@ fn collator_reporting_works() { ) .await; + assert_runtime_version_request(&mut virtual_overseer, test_state.relay_parent).await; + respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; let peer_b = PeerId::random(); @@ -443,6 +576,7 @@ fn collator_reporting_works() { peer_b.clone(), test_state.collators[0].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; @@ -451,6 +585,7 @@ fn collator_reporting_works() { peer_c.clone(), test_state.collators[1].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; @@ -480,7 +615,7 @@ fn collator_authentication_verification_works() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let peer_b = PeerId::random(); @@ -531,20 +666,25 @@ fn fetch_one_collation_at_a_time() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let second = Hash::random(); + let our_view = our_view![test_state.relay_parent, second]; + overseer_send( &mut virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent, second], + our_view.clone(), )), ) .await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + // Iter over view since the order may change due to sorted invariant. + for hash in our_view.iter() { + assert_runtime_version_request(&mut virtual_overseer, *hash).await; + respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + } let peer_b = PeerId::random(); let peer_c = PeerId::random(); @@ -554,6 +694,7 @@ fn fetch_one_collation_at_a_time() { peer_b.clone(), test_state.collators[0].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; @@ -562,16 +703,20 @@ fn fetch_one_collation_at_a_time() { peer_c.clone(), test_state.collators[1].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent).await; - advertise_collation(&mut virtual_overseer, peer_c.clone(), test_state.relay_parent).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent, None) + .await; + advertise_collation(&mut virtual_overseer, peer_c.clone(), test_state.relay_parent, None) + .await; let response_channel = assert_fetch_collation_request( &mut virtual_overseer, test_state.relay_parent, test_state.chain_ids[0], + None, ) .await; @@ -585,10 +730,13 @@ fn fetch_one_collation_at_a_time() { dummy_candidate_receipt_bad_sig(dummy_hash(), Some(Default::default())); candidate_a.descriptor.para_id = test_state.chain_ids[0]; candidate_a.descriptor.relay_parent = test_state.relay_parent; + candidate_a.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); response_channel - .send(Ok( - CollationFetchingResponse::Collation(candidate_a.clone(), pov.clone()).encode() - )) + .send(Ok(request_v1::CollationFetchingResponse::Collation( + candidate_a.clone(), + pov.clone(), + ) + .encode())) .expect("Sending response should succeed"); assert_candidate_backing_second( @@ -596,6 +744,7 @@ fn fetch_one_collation_at_a_time() { test_state.relay_parent, test_state.chain_ids[0], &pov, + ProspectiveParachainsMode::Disabled, ) .await; @@ -616,20 +765,24 @@ fn fetches_next_collation() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let second = Hash::random(); + let our_view = our_view![test_state.relay_parent, second]; + overseer_send( &mut virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent, second], + our_view.clone(), )), ) .await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + for hash in our_view.iter() { + assert_runtime_version_request(&mut virtual_overseer, *hash).await; + respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + } let peer_b = PeerId::random(); let peer_c = PeerId::random(); @@ -640,6 +793,7 @@ fn fetches_next_collation() { peer_b.clone(), test_state.collators[2].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; @@ -648,6 +802,7 @@ fn fetches_next_collation() { peer_c.clone(), test_state.collators[3].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; @@ -656,45 +811,64 @@ fn fetches_next_collation() { peer_d.clone(), test_state.collators[4].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), second).await; - advertise_collation(&mut virtual_overseer, peer_c.clone(), second).await; - advertise_collation(&mut virtual_overseer, peer_d.clone(), second).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), second, None).await; + advertise_collation(&mut virtual_overseer, peer_c.clone(), second, None).await; + advertise_collation(&mut virtual_overseer, peer_d.clone(), second, None).await; // Dropping the response channel should lead to fetching the second collation. - assert_fetch_collation_request(&mut virtual_overseer, second, test_state.chain_ids[0]) - .await; + assert_fetch_collation_request( + &mut virtual_overseer, + second, + test_state.chain_ids[0], + None, + ) + .await; - let response_channel_non_exclusive = - assert_fetch_collation_request(&mut virtual_overseer, second, test_state.chain_ids[0]) - .await; + let response_channel_non_exclusive = assert_fetch_collation_request( + &mut virtual_overseer, + second, + test_state.chain_ids[0], + None, + ) + .await; // Third collator should receive response after that timeout: Delay::new(MAX_UNSHARED_DOWNLOAD_TIME + Duration::from_millis(50)).await; - let response_channel = - assert_fetch_collation_request(&mut virtual_overseer, second, test_state.chain_ids[0]) - .await; + let response_channel = assert_fetch_collation_request( + &mut virtual_overseer, + second, + test_state.chain_ids[0], + None, + ) + .await; let pov = PoV { block_data: BlockData(vec![1]) }; let mut candidate_a = dummy_candidate_receipt_bad_sig(dummy_hash(), Some(Default::default())); candidate_a.descriptor.para_id = test_state.chain_ids[0]; candidate_a.descriptor.relay_parent = second; + candidate_a.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); // First request finishes now: response_channel_non_exclusive - .send(Ok( - CollationFetchingResponse::Collation(candidate_a.clone(), pov.clone()).encode() - )) + .send(Ok(request_v1::CollationFetchingResponse::Collation( + candidate_a.clone(), + pov.clone(), + ) + .encode())) .expect("Sending response should succeed"); response_channel - .send(Ok( - CollationFetchingResponse::Collation(candidate_a.clone(), pov.clone()).encode() - )) + .send(Ok(request_v1::CollationFetchingResponse::Collation( + candidate_a.clone(), + pov.clone(), + ) + .encode())) .expect("Sending response should succeed"); assert_candidate_backing_second( @@ -702,6 +876,7 @@ fn fetches_next_collation() { second, test_state.chain_ids[0], &pov, + ProspectiveParachainsMode::Disabled, ) .await; @@ -714,7 +889,7 @@ fn reject_connection_to_next_group() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; overseer_send( &mut virtual_overseer, @@ -724,6 +899,7 @@ fn reject_connection_to_next_group() { ) .await; + assert_runtime_version_request(&mut virtual_overseer, test_state.relay_parent).await; respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; let peer_b = PeerId::random(); @@ -733,6 +909,7 @@ fn reject_connection_to_next_group() { peer_b.clone(), test_state.collators[0].clone(), test_state.chain_ids[1].clone(), // next, not current `para_id` + CollationVersion::V1, ) .await; @@ -759,20 +936,24 @@ fn fetch_next_collation_on_invalid_collation() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let second = Hash::random(); + let our_view = our_view![test_state.relay_parent, second]; + overseer_send( &mut virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent, second], + our_view.clone(), )), ) .await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + for hash in our_view.iter() { + assert_runtime_version_request(&mut virtual_overseer, *hash).await; + respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + } let peer_b = PeerId::random(); let peer_c = PeerId::random(); @@ -782,6 +963,7 @@ fn fetch_next_collation_on_invalid_collation() { peer_b.clone(), test_state.collators[0].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; @@ -790,16 +972,20 @@ fn fetch_next_collation_on_invalid_collation() { peer_c.clone(), test_state.collators[1].clone(), test_state.chain_ids[0].clone(), + CollationVersion::V1, ) .await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent).await; - advertise_collation(&mut virtual_overseer, peer_c.clone(), test_state.relay_parent).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent, None) + .await; + advertise_collation(&mut virtual_overseer, peer_c.clone(), test_state.relay_parent, None) + .await; let response_channel = assert_fetch_collation_request( &mut virtual_overseer, test_state.relay_parent, test_state.chain_ids[0], + None, ) .await; @@ -808,10 +994,13 @@ fn fetch_next_collation_on_invalid_collation() { dummy_candidate_receipt_bad_sig(dummy_hash(), Some(Default::default())); candidate_a.descriptor.para_id = test_state.chain_ids[0]; candidate_a.descriptor.relay_parent = test_state.relay_parent; + candidate_a.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); response_channel - .send(Ok( - CollationFetchingResponse::Collation(candidate_a.clone(), pov.clone()).encode() - )) + .send(Ok(request_v1::CollationFetchingResponse::Collation( + candidate_a.clone(), + pov.clone(), + ) + .encode())) .expect("Sending response should succeed"); let receipt = assert_candidate_backing_second( @@ -819,6 +1008,7 @@ fn fetch_next_collation_on_invalid_collation() { test_state.relay_parent, test_state.chain_ids[0], &pov, + ProspectiveParachainsMode::Disabled, ) .await; @@ -845,6 +1035,7 @@ fn fetch_next_collation_on_invalid_collation() { &mut virtual_overseer, test_state.relay_parent, test_state.chain_ids[0], + None, ) .await; @@ -857,7 +1048,7 @@ fn inactive_disconnected() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let pair = CollatorPair::generate().0; @@ -871,6 +1062,7 @@ fn inactive_disconnected() { ) .await; + assert_runtime_version_request(&mut virtual_overseer, hash_a).await; respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; let peer_b = PeerId::random(); @@ -880,14 +1072,17 @@ fn inactive_disconnected() { peer_b.clone(), pair.clone(), test_state.chain_ids[0], + CollationVersion::V1, ) .await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), test_state.relay_parent, None) + .await; assert_fetch_collation_request( &mut virtual_overseer, test_state.relay_parent, test_state.chain_ids[0], + None, ) .await; @@ -903,7 +1098,7 @@ fn activity_extends_life() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let pair = CollatorPair::generate().0; @@ -911,18 +1106,20 @@ fn activity_extends_life() { let hash_b = Hash::repeat_byte(1); let hash_c = Hash::repeat_byte(2); + let our_view = our_view![hash_a, hash_b, hash_c]; + overseer_send( &mut virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![hash_a, hash_b, hash_c], + our_view.clone(), )), ) .await; - // 3 heads, 3 times. - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + for hash in our_view.iter() { + assert_runtime_version_request(&mut virtual_overseer, *hash).await; + respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; + } let peer_b = PeerId::random(); @@ -931,29 +1128,45 @@ fn activity_extends_life() { peer_b.clone(), pair.clone(), test_state.chain_ids[0], + CollationVersion::V1, ) .await; Delay::new(ACTIVITY_TIMEOUT * 2 / 3).await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), hash_a).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), hash_a, None).await; - assert_fetch_collation_request(&mut virtual_overseer, hash_a, test_state.chain_ids[0]) - .await; + assert_fetch_collation_request( + &mut virtual_overseer, + hash_a, + test_state.chain_ids[0], + None, + ) + .await; Delay::new(ACTIVITY_TIMEOUT * 2 / 3).await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), hash_b).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), hash_b, None).await; - assert_fetch_collation_request(&mut virtual_overseer, hash_b, test_state.chain_ids[0]) - .await; + assert_fetch_collation_request( + &mut virtual_overseer, + hash_b, + test_state.chain_ids[0], + None, + ) + .await; Delay::new(ACTIVITY_TIMEOUT * 2 / 3).await; - advertise_collation(&mut virtual_overseer, peer_b.clone(), hash_c).await; + advertise_collation(&mut virtual_overseer, peer_b.clone(), hash_c, None).await; - assert_fetch_collation_request(&mut virtual_overseer, hash_c, test_state.chain_ids[0]) - .await; + assert_fetch_collation_request( + &mut virtual_overseer, + hash_c, + test_state.chain_ids[0], + None, + ) + .await; Delay::new(ACTIVITY_TIMEOUT * 3 / 2).await; @@ -968,7 +1181,7 @@ fn disconnect_if_no_declare() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; overseer_send( &mut virtual_overseer, @@ -978,6 +1191,7 @@ fn disconnect_if_no_declare() { ) .await; + assert_runtime_version_request(&mut virtual_overseer, test_state.relay_parent).await; respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; let peer_b = PeerId::random(); @@ -1004,7 +1218,7 @@ fn disconnect_if_wrong_declare() { let test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let pair = CollatorPair::generate().0; @@ -1016,6 +1230,7 @@ fn disconnect_if_wrong_declare() { ) .await; + assert_runtime_version_request(&mut virtual_overseer, test_state.relay_parent).await; respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; let peer_b = PeerId::random(); @@ -1066,7 +1281,7 @@ fn view_change_clears_old_collators() { let mut test_state = TestState::default(); test_harness(|test_harness| async move { - let TestHarness { mut virtual_overseer } = test_harness; + let TestHarness { mut virtual_overseer, .. } = test_harness; let pair = CollatorPair::generate().0; @@ -1078,6 +1293,7 @@ fn view_change_clears_old_collators() { ) .await; + assert_runtime_version_request(&mut virtual_overseer, test_state.relay_parent).await; respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; let peer_b = PeerId::random(); @@ -1087,6 +1303,7 @@ fn view_change_clears_old_collators() { peer_b.clone(), pair.clone(), test_state.chain_ids[0], + CollationVersion::V1, ) .await; @@ -1101,6 +1318,7 @@ fn view_change_clears_old_collators() { .await; test_state.group_rotation_info = test_state.group_rotation_info.bump_rotation(); + assert_runtime_version_request(&mut virtual_overseer, hash_b).await; respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; assert_collator_disconnect(&mut virtual_overseer, peer_b.clone()).await; diff --git a/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs new file mode 100644 index 000000000000..1eccb97cbd67 --- /dev/null +++ b/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -0,0 +1,616 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the validator side with enabled prospective parachains. + +use super::*; + +use polkadot_node_subsystem::messages::ChainApiMessage; +use polkadot_primitives::v2::{ + BlockNumber, CandidateCommitments, CommittedCandidateReceipt, Header, SigningContext, + ValidatorId, +}; + +const ALLOWED_ANCESTRY: u32 = 3; + +fn get_parent_hash(hash: Hash) -> Hash { + Hash::from_low_u64_be(hash.to_low_u64_be() + 1) +} + +async fn assert_assign_incoming( + virtual_overseer: &mut VirtualOverseer, + test_state: &TestState, + hash: Hash, + number: BlockNumber, + next_msg: &mut Option, +) { + let msg = match next_msg.take() { + Some(msg) => msg, + None => overseer_recv(virtual_overseer).await, + }; + assert_matches!( + msg, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::Validators(tx)) + ) if parent == hash => { + tx.send(Ok(test_state.validator_public.clone())).unwrap(); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::ValidatorGroups(tx)) + ) if parent == hash => { + let validator_groups = test_state.validator_groups.clone(); + let mut group_rotation_info = test_state.group_rotation_info.clone(); + group_rotation_info.now = number; + tx.send(Ok((validator_groups, group_rotation_info))).unwrap(); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::AvailabilityCores(tx)) + ) if parent == hash => { + tx.send(Ok(test_state.cores.clone())).unwrap(); + } + ); +} + +/// Handle a view update. +async fn update_view( + virtual_overseer: &mut VirtualOverseer, + test_state: &TestState, + new_view: Vec<(Hash, u32)>, // Hash and block number. + activated: u8, // How many new heads does this update contain? +) { + let new_view: HashMap = HashMap::from_iter(new_view); + + let our_view = + OurView::new(new_view.keys().map(|hash| (*hash, Arc::new(jaeger::Span::Disabled))), 0); + + overseer_send( + virtual_overseer, + CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange(our_view)), + ) + .await; + + let mut next_overseer_message = None; + for _ in 0..activated { + let (leaf_hash, leaf_number) = assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + parent, + RuntimeApiRequest::Version(tx), + )) => { + tx.send(Ok(RuntimeApiRequest::VALIDITY_CONSTRAINTS)).unwrap(); + (parent, new_view.get(&parent).copied().expect("Unknown parent requested")) + } + ); + + assert_assign_incoming( + virtual_overseer, + test_state, + leaf_hash, + leaf_number, + &mut next_overseer_message, + ) + .await; + + let min_number = leaf_number.saturating_sub(ALLOWED_ANCESTRY); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::GetMinimumRelayParents(parent, tx), + ) if parent == leaf_hash => { + tx.send(test_state.chain_ids.iter().map(|para_id| (*para_id, min_number)).collect()).unwrap(); + } + ); + + let ancestry_len = leaf_number + 1 - min_number; + let ancestry_hashes = std::iter::successors(Some(leaf_hash), |h| Some(get_parent_hash(*h))) + .take(ancestry_len as usize); + let ancestry_numbers = (min_number..=leaf_number).rev(); + let ancestry_iter = ancestry_hashes.clone().zip(ancestry_numbers).peekable(); + + // How many blocks were actually requested. + let mut requested_len: usize = 0; + { + let mut ancestry_iter = ancestry_iter.clone(); + loop { + let (hash, number) = match ancestry_iter.next() { + Some((hash, number)) => (hash, number), + None => break, + }; + + // May be `None` for the last element. + let parent_hash = + ancestry_iter.peek().map(|(h, _)| *h).unwrap_or_else(|| get_parent_hash(hash)); + + let msg = match next_overseer_message.take() { + Some(msg) => msg, + None => overseer_recv(virtual_overseer).await, + }; + + if !matches!(&msg, AllMessages::ChainApi(ChainApiMessage::BlockHeader(..))) { + // Ancestry has already been cached for this leaf. + next_overseer_message.replace(msg); + break + } + + assert_matches!( + msg, + AllMessages::ChainApi(ChainApiMessage::BlockHeader(.., tx)) => { + let header = Header { + parent_hash, + number, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + }; + + tx.send(Ok(Some(header))).unwrap(); + } + ); + + requested_len += 1; + } + } + + // Skip the leaf. + for (hash, number) in ancestry_iter.skip(1).take(requested_len.saturating_sub(1)) { + assert_assign_incoming( + virtual_overseer, + test_state, + hash, + number, + &mut next_overseer_message, + ) + .await; + } + } +} + +async fn send_seconded_statement( + virtual_overseer: &mut VirtualOverseer, + keystore: SyncCryptoStorePtr, + candidate: &CommittedCandidateReceipt, +) { + let signing_context = SigningContext { session_index: 0, parent_hash: Hash::zero() }; + let stmt = SignedFullStatement::sign( + &keystore, + Statement::Seconded(candidate.clone()), + &signing_context, + ValidatorIndex(0), + &ValidatorId::from(Sr25519Keyring::Alice.public()), + ) + .await + .ok() + .flatten() + .expect("should be signed"); + + overseer_send( + virtual_overseer, + CollatorProtocolMessage::Seconded(candidate.descriptor.relay_parent, stmt), + ) + .await; +} + +async fn assert_collation_seconded( + virtual_overseer: &mut VirtualOverseer, + relay_parent: Hash, + peer_id: PeerId, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer( + peer, + rep, + )) => { + assert_eq!(peer_id, peer); + assert_eq!(rep, BENEFIT_NOTIFY_GOOD); + } + ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendCollationMessage( + peers, + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + protocol_vstaging::CollatorProtocolMessage::CollationSeconded( + _relay_parent, + .., + ), + )), + )) => { + assert_eq!(peers, vec![peer_id]); + assert_eq!(relay_parent, _relay_parent); + } + ); +} + +#[test] +fn v1_advertisement_rejected() { + let test_state = TestState::default(); + + test_harness(|test_harness| async move { + let TestHarness { mut virtual_overseer, .. } = test_harness; + + let pair_a = CollatorPair::generate().0; + + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 0; + + update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + + let peer_a = PeerId::random(); + + // Accept both collators from the implicit view. + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair_a.clone(), + test_state.chain_ids[0], + CollationVersion::V1, + ) + .await; + + advertise_collation(&mut virtual_overseer, peer_a, head_b, None).await; + + // Not reported. + assert!(overseer_recv_with_timeout(&mut virtual_overseer, Duration::from_millis(50)) + .await + .is_none()); + + virtual_overseer + }); +} + +#[test] +fn accept_advertisements_from_implicit_view() { + let test_state = TestState::default(); + + test_harness(|test_harness| async move { + let TestHarness { mut virtual_overseer, .. } = test_harness; + + let pair_a = CollatorPair::generate().0; + let pair_b = CollatorPair::generate().0; + + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 2; + + // Grandparent of head `b`. + // Group rotation frequency is 1 by default, at `c` we're assigned + // to the first para. + let head_c = Hash::from_low_u64_be(130); + + // Activated leaf is `b`, but the collation will be based on `c`. + update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + // Accept both collators from the implicit view. + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair_a.clone(), + test_state.chain_ids[0], + CollationVersion::VStaging, + ) + .await; + connect_and_declare_collator( + &mut virtual_overseer, + peer_b, + pair_b.clone(), + test_state.chain_ids[1], + CollationVersion::VStaging, + ) + .await; + + let candidate_hash = CandidateHash::default(); + let parent_head_data_hash = Hash::zero(); + advertise_collation( + &mut virtual_overseer, + peer_b, + head_c, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + // Advertise with different para. + advertise_collation( + &mut virtual_overseer, + peer_a, + head_c, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + + let response_channel = assert_fetch_collation_request( + &mut virtual_overseer, + head_c, + test_state.chain_ids[1], + Some(candidate_hash), + ) + .await; + + // Respond with an error to abort seconding. + response_channel + .send(Err(sc_network::RequestFailure::NotConnected)) + .expect("Sending response should succeed"); + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(..),) + ); + + assert_fetch_collation_request( + &mut virtual_overseer, + head_c, + test_state.chain_ids[0], + Some(candidate_hash), + ) + .await; + + virtual_overseer + }); +} + +#[test] +fn second_multiple_candidates_per_relay_parent() { + let test_state = TestState::default(); + + test_harness(|test_harness| async move { + let TestHarness { mut virtual_overseer, keystore } = test_harness; + + let pair = CollatorPair::generate().0; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 2; + + // Grandparent of head `b`. + // Group rotation frequency is 1 by default, at `c` we're assigned + // to the first para. + let head_c = Hash::from_low_u64_be(130); + + // Activated leaf is `b`, but the collation will be based on `c`. + update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + + let peer_a = PeerId::random(); + + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair.clone(), + test_state.chain_ids[0], + CollationVersion::VStaging, + ) + .await; + + for i in 0..(MAX_CANDIDATE_DEPTH + 1) { + let mut candidate = dummy_candidate_receipt_bad_sig(head_c, Some(Default::default())); + candidate.descriptor.para_id = test_state.chain_ids[0]; + candidate.descriptor.relay_parent = head_c; + candidate.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); + let commitments = CandidateCommitments { + head_data: HeadData(vec![i as u8]), + horizontal_messages: Vec::new(), + upward_messages: Vec::new(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }; + candidate.commitments_hash = commitments.hash(); + + let candidate_hash = candidate.hash(); + let parent_head_data_hash = Hash::zero(); + + advertise_collation( + &mut virtual_overseer, + peer_a, + head_c, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + + let response_channel = assert_fetch_collation_request( + &mut virtual_overseer, + head_c, + test_state.chain_ids[0], + Some(candidate_hash), + ) + .await; + + let pov = PoV { block_data: BlockData(vec![1]) }; + + response_channel + .send(Ok(request_vstaging::CollationFetchingResponse::Collation( + candidate.clone(), + pov.clone(), + ) + .encode())) + .expect("Sending response should succeed"); + + assert_candidate_backing_second( + &mut virtual_overseer, + head_c, + test_state.chain_ids[0], + &pov, + ProspectiveParachainsMode::Enabled, + ) + .await; + + let candidate = + CommittedCandidateReceipt { descriptor: candidate.descriptor, commitments }; + + send_seconded_statement(&mut virtual_overseer, keystore.clone(), &candidate).await; + + assert_collation_seconded(&mut virtual_overseer, head_c, peer_a).await; + } + + // No more advertisements can be made for this relay parent. + let candidate_hash = CandidateHash(Hash::repeat_byte(0xAA)); + advertise_collation( + &mut virtual_overseer, + peer_a, + head_c, + Some((candidate_hash, Hash::zero())), + ) + .await; + + // Reported because reached the limit of advertisements per relay parent. + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::ReportPeer(peer_id, rep), + ) => { + assert_eq!(peer_a, peer_id); + assert_eq!(rep, COST_UNEXPECTED_MESSAGE); + } + ); + + // By different peer too (not reported). + let pair_b = CollatorPair::generate().0; + let peer_b = PeerId::random(); + + connect_and_declare_collator( + &mut virtual_overseer, + peer_b, + pair_b.clone(), + test_state.chain_ids[0], + CollationVersion::VStaging, + ) + .await; + + let candidate_hash = CandidateHash(Hash::repeat_byte(0xFF)); + advertise_collation( + &mut virtual_overseer, + peer_b, + head_c, + Some((candidate_hash, Hash::zero())), + ) + .await; + + assert!(overseer_recv_with_timeout(&mut virtual_overseer, Duration::from_millis(50)) + .await + .is_none()); + + virtual_overseer + }); +} + +#[test] +fn fetched_collation_sanity_check() { + let test_state = TestState::default(); + + test_harness(|test_harness| async move { + let TestHarness { mut virtual_overseer, .. } = test_harness; + + let pair = CollatorPair::generate().0; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 2; + + // Grandparent of head `b`. + // Group rotation frequency is 1 by default, at `c` we're assigned + // to the first para. + let head_c = Hash::from_low_u64_be(130); + + // Activated leaf is `b`, but the collation will be based on `c`. + update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + + let peer_a = PeerId::random(); + + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair.clone(), + test_state.chain_ids[0], + CollationVersion::VStaging, + ) + .await; + + let mut candidate = dummy_candidate_receipt_bad_sig(head_c, Some(Default::default())); + candidate.descriptor.para_id = test_state.chain_ids[0]; + candidate.descriptor.relay_parent = head_c; + let commitments = CandidateCommitments { + head_data: HeadData(vec![1, 2, 3]), + horizontal_messages: Vec::new(), + upward_messages: Vec::new(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }; + candidate.commitments_hash = commitments.hash(); + + let candidate_hash = CandidateHash(Hash::zero()); + let parent_head_data_hash = Hash::zero(); + + advertise_collation( + &mut virtual_overseer, + peer_a, + head_c, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + + let response_channel = assert_fetch_collation_request( + &mut virtual_overseer, + head_c, + test_state.chain_ids[0], + Some(candidate_hash), + ) + .await; + + let pov = PoV { block_data: BlockData(vec![1]) }; + + response_channel + .send(Ok(request_vstaging::CollationFetchingResponse::Collation( + candidate.clone(), + pov.clone(), + ) + .encode())) + .expect("Sending response should succeed"); + + // PVD request. + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ProspectiveParachains( + ProspectiveParachainsMessage::GetProspectiveValidationData(request, tx), + ) => { + assert_eq!(head_c, request.candidate_relay_parent); + assert_eq!(test_state.chain_ids[0], request.para_id); + tx.send(Some(dummy_pvd())).unwrap(); + } + ); + + // Reported malicious. + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::ReportPeer(peer_id, rep), + ) => { + assert_eq!(peer_a, peer_id); + assert_eq!(rep, COST_REPORT_BAD); + } + ); + + virtual_overseer + }); +} diff --git a/node/network/protocol/src/lib.rs b/node/network/protocol/src/lib.rs index a45bca82df49..7e08d22697f5 100644 --- a/node/network/protocol/src/lib.rs +++ b/node/network/protocol/src/lib.rs @@ -592,7 +592,7 @@ pub mod vstaging { use parity_scale_codec::{Decode, Encode}; use polkadot_primitives::vstaging::{ - CandidateIndex, CollatorId, CollatorSignature, Hash, Id as ParaId, + CandidateHash, CandidateIndex, CollatorId, CollatorSignature, Hash, Id as ParaId, UncheckedSignedAvailabilityBitfield, }; @@ -651,7 +651,14 @@ pub mod vstaging { /// Advertise a collation to a validator. Can only be sent once the peer has /// declared that they are a collator with given ID. #[codec(index = 1)] - AdvertiseCollation(Hash), + AdvertiseCollation { + /// Hash of the relay parent advertised collation is based on. + relay_parent: Hash, + /// Candidate hash. + candidate_hash: CandidateHash, + /// Parachain head data hash before candidate execution. + parent_head_data_hash: Hash, + }, /// A collation sent to a validator was seconded. #[codec(index = 4)] CollationSeconded(Hash, UncheckedSignedFullStatement), diff --git a/node/network/protocol/src/request_response/mod.rs b/node/network/protocol/src/request_response/mod.rs index d24537e219c7..13541f810f91 100644 --- a/node/network/protocol/src/request_response/mod.rs +++ b/node/network/protocol/src/request_response/mod.rs @@ -55,6 +55,9 @@ pub use outgoing::{OutgoingRequest, OutgoingResult, Recipient, Requests, Respons /// Actual versioned requests and responses, that are sent over the wire. pub mod v1; +/// Staging requests to be sent over the wire. +pub mod vstaging; + /// A protocol per subsystem seems to make the most sense, this way we don't need any dispatching /// within protocols. #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, EnumIter)] @@ -63,6 +66,8 @@ pub enum Protocol { ChunkFetchingV1, /// Protocol for fetching collations from collators. CollationFetchingV1, + /// Protocol for fetching collations from collators when async backing is enabled. + CollationFetchingVStaging, /// Protocol for fetching seconded PoVs from validators of the same group. PoVFetchingV1, /// Protocol for fetching available data. @@ -147,15 +152,16 @@ impl Protocol { request_timeout: CHUNK_REQUEST_TIMEOUT, inbound_queue: Some(tx), }, - Protocol::CollationFetchingV1 => RequestResponseConfig { - name, - fallback_names, - max_request_size: 1_000, - max_response_size: POV_RESPONSE_SIZE, - // Taken from initial implementation in collator protocol: - request_timeout: POV_REQUEST_TIMEOUT_CONNECTED, - inbound_queue: Some(tx), - }, + Protocol::CollationFetchingV1 | Protocol::CollationFetchingVStaging => + RequestResponseConfig { + name, + fallback_names, + max_request_size: 1_000, + max_response_size: POV_RESPONSE_SIZE, + // Taken from initial implementation in collator protocol: + request_timeout: POV_REQUEST_TIMEOUT_CONNECTED, + inbound_queue: Some(tx), + }, Protocol::PoVFetchingV1 => RequestResponseConfig { name, fallback_names, @@ -215,7 +221,7 @@ impl Protocol { // as well. Protocol::ChunkFetchingV1 => 100, // 10 seems reasonable, considering group sizes of max 10 validators. - Protocol::CollationFetchingV1 => 10, + Protocol::CollationFetchingV1 | Protocol::CollationFetchingVStaging => 10, // 10 seems reasonable, considering group sizes of max 10 validators. Protocol::PoVFetchingV1 => 10, // Validators are constantly self-selecting to request available data which may lead @@ -259,6 +265,7 @@ impl Protocol { match self { Protocol::ChunkFetchingV1 => "/polkadot/req_chunk/1", Protocol::CollationFetchingV1 => "/polkadot/req_collation/1", + Protocol::CollationFetchingVStaging => "/polkadot/req_collation/2", Protocol::PoVFetchingV1 => "/polkadot/req_pov/1", Protocol::AvailableDataFetchingV1 => "/polkadot/req_available_data/1", Protocol::StatementFetchingV1 => "/polkadot/req_statement/1", @@ -314,6 +321,7 @@ impl ReqProtocolNames { let short_name = match protocol { Protocol::ChunkFetchingV1 => "/req_chunk/1", Protocol::CollationFetchingV1 => "/req_collation/1", + Protocol::CollationFetchingVStaging => "/req_collation/2", Protocol::PoVFetchingV1 => "/req_pov/1", Protocol::AvailableDataFetchingV1 => "/req_available_data/1", Protocol::StatementFetchingV1 => "/req_statement/1", diff --git a/node/network/protocol/src/request_response/outgoing.rs b/node/network/protocol/src/request_response/outgoing.rs index b93c4e93cd31..e01111d1cf12 100644 --- a/node/network/protocol/src/request_response/outgoing.rs +++ b/node/network/protocol/src/request_response/outgoing.rs @@ -23,7 +23,7 @@ use sc_network::PeerId; use polkadot_primitives::v2::AuthorityDiscoveryId; -use super::{v1, IsRequest, Protocol}; +use super::{v1, vstaging, IsRequest, Protocol}; /// All requests that can be sent to the network bridge via `NetworkBridgeTxMessage::SendRequest`. #[derive(Debug)] @@ -40,6 +40,10 @@ pub enum Requests { StatementFetchingV1(OutgoingRequest), /// Requests for notifying about an ongoing dispute. DisputeSendingV1(OutgoingRequest), + + /// Fetch a collation from a collator which previously announced it. + /// Compared to V1 it requires specifying which candidate is requested by its hash. + CollationFetchingVStaging(OutgoingRequest), } impl Requests { @@ -48,6 +52,7 @@ impl Requests { match self { Self::ChunkFetchingV1(_) => Protocol::ChunkFetchingV1, Self::CollationFetchingV1(_) => Protocol::CollationFetchingV1, + Self::CollationFetchingVStaging(_) => Protocol::CollationFetchingVStaging, Self::PoVFetchingV1(_) => Protocol::PoVFetchingV1, Self::AvailableDataFetchingV1(_) => Protocol::AvailableDataFetchingV1, Self::StatementFetchingV1(_) => Protocol::StatementFetchingV1, @@ -66,6 +71,7 @@ impl Requests { match self { Self::ChunkFetchingV1(r) => r.encode_request(), Self::CollationFetchingV1(r) => r.encode_request(), + Self::CollationFetchingVStaging(r) => r.encode_request(), Self::PoVFetchingV1(r) => r.encode_request(), Self::AvailableDataFetchingV1(r) => r.encode_request(), Self::StatementFetchingV1(r) => r.encode_request(), diff --git a/node/network/protocol/src/request_response/vstaging.rs b/node/network/protocol/src/request_response/vstaging.rs new file mode 100644 index 000000000000..0b8d223e3aee --- /dev/null +++ b/node/network/protocol/src/request_response/vstaging.rs @@ -0,0 +1,40 @@ +// Copyright 2022 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use parity_scale_codec::{Decode, Encode}; +use polkadot_primitives::v2::{CandidateHash, Hash, Id as ParaId}; + +use super::{IsRequest, Protocol}; + +/// Responses as sent by collators. +pub type CollationFetchingResponse = super::v1::CollationFetchingResponse; + +/// Request the advertised collation at that relay-parent. +#[derive(Debug, Clone, Encode, Decode)] +pub struct CollationFetchingRequest { + /// Relay parent collation is built on top of. + pub relay_parent: Hash, + /// The `ParaId` of the collation. + pub para_id: ParaId, + /// Candidate hash. + pub candidate_hash: CandidateHash, +} + +impl IsRequest for CollationFetchingRequest { + // The response is the same as for V1. + type Response = CollationFetchingResponse; + const PROTOCOL: Protocol = Protocol::CollationFetchingVStaging; +} diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 464e47d60111..3546161c0706 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -558,6 +558,8 @@ pub struct Overseer { NetworkBridgeTxMessage, RuntimeApiMessage, CandidateBackingMessage, + ChainApiMessage, + ProspectiveParachainsMessage, ])] collator_protocol: CollatorProtocol, diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 3619d05c7592..2552f9c44330 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -887,7 +887,11 @@ where config.network.request_response_protocols.push(cfg); let (chunk_req_receiver, cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); config.network.request_response_protocols.push(cfg); - let (collation_req_receiver, cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); + let (collation_req_v1_receiver, cfg) = + IncomingRequest::get_config_receiver(&req_protocol_names); + config.network.request_response_protocols.push(cfg); + let (collation_req_vstaging_receiver, cfg) = + IncomingRequest::get_config_receiver(&req_protocol_names); config.network.request_response_protocols.push(cfg); let (available_data_req_receiver, cfg) = IncomingRequest::get_config_receiver(&req_protocol_names); @@ -1067,7 +1071,8 @@ where authority_discovery_service, pov_req_receiver, chunk_req_receiver, - collation_req_receiver, + collation_req_v1_receiver, + collation_req_vstaging_receiver, available_data_req_receiver, statement_req_receiver, dispute_req_receiver, diff --git a/node/service/src/overseer.rs b/node/service/src/overseer.rs index af8ca6aea54b..ef7c8de74e9c 100644 --- a/node/service/src/overseer.rs +++ b/node/service/src/overseer.rs @@ -26,7 +26,9 @@ use polkadot_node_core_chain_selection::Config as ChainSelectionConfig; use polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig; use polkadot_node_network_protocol::{ peer_set::PeerSetProtocolNames, - request_response::{v1 as request_v1, IncomingRequestReceiver, ReqProtocolNames}, + request_response::{ + v1 as request_v1, vstaging as request_vstaging, IncomingRequestReceiver, ReqProtocolNames, + }, }; #[cfg(any(feature = "malus", test))] pub use polkadot_overseer::{ @@ -93,13 +95,21 @@ where pub network_service: Arc>, /// Underlying authority discovery service. pub authority_discovery_service: AuthorityDiscoveryService, - /// POV request receiver + /// POV request receiver. pub pov_req_receiver: IncomingRequestReceiver, + /// Erasure chunks request receiver. pub chunk_req_receiver: IncomingRequestReceiver, - pub collation_req_receiver: IncomingRequestReceiver, + /// Collations request receiver for network protocol v1. + pub collation_req_v1_receiver: IncomingRequestReceiver, + /// Collations request receiver for network protocol vstaging. + pub collation_req_vstaging_receiver: + IncomingRequestReceiver, + /// Receiver for available data requests. pub available_data_req_receiver: IncomingRequestReceiver, + /// Receiver for incoming large statement requests. pub statement_req_receiver: IncomingRequestReceiver, + /// Receiver for incoming disputes. pub dispute_req_receiver: IncomingRequestReceiver, /// Prometheus registry, commonly used for production systems, less so for test. pub registry: Option<&'a Registry>, @@ -139,7 +149,8 @@ pub fn prepared_overseer_builder<'a, Spawner, RuntimeClient>( authority_discovery_service, pov_req_receiver, chunk_req_receiver, - collation_req_receiver, + collation_req_v1_receiver, + collation_req_vstaging_receiver, available_data_req_receiver, statement_req_receiver, dispute_req_receiver, @@ -257,12 +268,13 @@ where .collation_generation(CollationGenerationSubsystem::new(Metrics::register(registry)?)) .collator_protocol({ let side = match is_collator { - IsCollator::Yes(collator_pair) => ProtocolSide::Collator( - network_service.local_peer_id().clone(), + IsCollator::Yes(collator_pair) => ProtocolSide::Collator { + peer_id: network_service.local_peer_id().clone(), collator_pair, - collation_req_receiver, - Metrics::register(registry)?, - ), + request_receiver_v1: collation_req_v1_receiver, + request_receiver_vstaging: collation_req_vstaging_receiver, + metrics: Metrics::register(registry)?, + }, IsCollator::No => ProtocolSide::Validator { keystore: keystore.clone(), eviction_policy: Default::default(), diff --git a/node/subsystem-types/src/messages.rs b/node/subsystem-types/src/messages.rs index 84a38d4a7be6..e3c03ee11c05 100644 --- a/node/subsystem-types/src/messages.rs +++ b/node/subsystem-types/src/messages.rs @@ -195,10 +195,16 @@ pub enum CollatorProtocolMessage { /// This should be sent before any `DistributeCollation` message. CollateOn(ParaId), /// Provide a collation to distribute to validators with an optional result sender. + /// The second argument is the parent head-data hash. /// /// The result sender should be informed when at least one parachain validator seconded the collation. It is also /// completely okay to just drop the sender. - DistributeCollation(CandidateReceipt, PoV, Option>), + DistributeCollation( + CandidateReceipt, + Hash, + PoV, + Option>, + ), /// Report a collator as having provided an invalid collation. This should lead to disconnect /// and blacklist of the collator. ReportCollator(CollatorId), @@ -719,7 +725,7 @@ impl RuntimeApiRequest { /// `Disputes` pub const DISPUTES_RUNTIME_REQUIREMENT: u32 = 3; - /// Minimum version for valididty constraints, required for async backing. + /// Minimum version for validity constraints, required for async backing. /// /// 99 for now, should be adjusted to VSTAGING/actual runtime version once released. pub const VALIDITY_CONSTRAINTS: u32 = 99; diff --git a/node/subsystem-util/src/backing_implicit_view.rs b/node/subsystem-util/src/backing_implicit_view.rs index dc10efe519fe..d3734e73c14f 100644 --- a/node/subsystem-util/src/backing_implicit_view.rs +++ b/node/subsystem-util/src/backing_implicit_view.rs @@ -100,6 +100,11 @@ struct BlockInfo { } impl View { + /// Get an iterator over active leaves in the view. + pub fn leaves<'a>(&'a self) -> impl Iterator + 'a { + self.leaves.keys() + } + /// Activate a leaf in the view. /// This will request the minimum relay parents from the /// Prospective Parachains subsystem for each leaf and will load headers in the ancestry of each @@ -152,9 +157,13 @@ impl View { } /// Deactivate a leaf in the view. This prunes any outdated implicit ancestors as well. - pub fn deactivate_leaf(&mut self, leaf_hash: Hash) { + /// + /// Returns hashes of blocks pruned from storage. + pub fn deactivate_leaf(&mut self, leaf_hash: Hash) -> Vec { + let mut removed = Vec::new(); + if self.leaves.remove(&leaf_hash).is_none() { - return + return removed } // Prune everything before the minimum out of all leaves, @@ -165,8 +174,15 @@ impl View { { let minimum = self.leaves.values().map(|l| l.retain_minimum).min(); - self.block_info_storage - .retain(|_, i| minimum.map_or(false, |m| i.block_number >= m)); + self.block_info_storage.retain(|hash, i| { + let keep = minimum.map_or(false, |m| i.block_number >= m); + if !keep { + removed.push(*hash); + } + keep + }); + + removed } } @@ -212,17 +228,26 @@ impl View { } /// Errors when fetching a leaf and associated ancestry. -#[derive(Debug)] +#[fatality::fatality] pub enum FetchError { - /// Leaf was already known. + /// Activated leaf is already present in view. + #[error("Leaf was already known")] AlreadyKnown, - /// The prospective parachains subsystem was unavailable. + + /// Request to the prospective parachains subsystem failed. + #[error("The prospective parachains subsystem was unavailable")] ProspectiveParachainsUnavailable, - /// A block header was unavailable. + + /// Failed to fetch the block header. + #[error("A block header was unavailable")] BlockHeaderUnavailable(Hash, BlockHeaderUnavailableReason), + /// A block header was unavailable due to a chain API error. + #[error("A block header was unavailable due to a chain API error")] ChainApiError(Hash, ChainApiError), - /// The chain API subsystem was unavailable. + + /// Request to the Chain API subsystem failed. + #[error("The chain API subsystem was unavailable")] ChainApiUnavailable, } diff --git a/scripts/ci/gitlab/lingua.dic b/scripts/ci/gitlab/lingua.dic index 3add6a276cf0..b68a8c20600a 100644 --- a/scripts/ci/gitlab/lingua.dic +++ b/scripts/ci/gitlab/lingua.dic @@ -298,6 +298,7 @@ unreserve unreserving unroutable unservable/B +unshare/D untrusted untyped unvested @@ -314,10 +315,11 @@ verify/R versa Versi version/DMSG -versioned VMP/SM VPS VRF/SM +vstaging +VStaging w3f/MS wakeup wakeups