collator-protocol-revamp: CollationManager and subsystem impl

cumulus/client/relay-chain-inprocess-interface/src/lib.rs

@@ -419,6 +419,7 @@ fn build_polkadot_full_node(
 		keep_finalized_for: None,
 		invulnerable_ah_collators: HashSet::new(),
 		collator_protocol_hold_off: None,
+		experimental_collator_protocol: false,
 	};
 
 	let (relay_chain_full_node, paranode_req_receiver) = match config.network.network_backend {

cumulus/zombienet/zombienet-sdk-helpers/src/lib.rs

@@ -116,7 +116,7 @@ pub async fn assert_para_throughput(
 	for (para_id, expected_candidate_range) in expected_candidate_ranges {
 		let actual = candidate_count
 			.get(&para_id)
-			.ok_or_else(|| anyhow!("ParaId did not have any backed candidates"))?;
+			.ok_or_else(|| anyhow!("ParaId {} did not have any backed candidates", para_id))?;
 
 		if !expected_candidate_range.contains(actual) {
 			return Err(anyhow!(

cumulus/zombienet/zombienet-sdk/tests/zombie_ci/elastic_scaling/upgrade_to_3_cores.rs

@@ -134,27 +134,16 @@ async fn build_network_config(async_backing: bool) -> Result<NetworkConfig, anyh
 		.with_parachain(|p| {
 			p.with_id(PARA_ID)
 				.with_default_command("test-parachain")
+				.with_default_image(images.cumulus.as_str())
+				.with_default_args(vec![
+					"--authoring=slot-based".into(),
+					"-lparachain=debug,aura=debug".into(),
+				])
 				.onboard_as_parachain(false)
 				.with_chain(chain)
-				.with_default_image(images.cumulus.as_str())
-				.with_collator(|n| {
-					n.with_name("collator0").validator(true).with_args(vec![
-						"--authoring=slot-based".into(),
-						("-lparachain=debug,aura=debug").into(),
-					])
-				})
-				.with_collator(|n| {
-					n.with_name("collator1").validator(true).with_args(vec![
-						"--authoring=slot-based".into(),
-						("-lparachain=debug,aura=debug").into(),
-					])
-				})
-				.with_collator(|n| {
-					n.with_name("collator2").validator(true).with_args(vec![
-						"--authoring=slot-based".into(),
-						("-lparachain=debug,aura=debug").into(),
-					])
-				})
+				.with_collator(|n| n.with_name("collator0").validator(true))
+				.with_collator(|n| n.with_name("collator1").validator(true))
+				.with_collator(|n| n.with_name("collator2").validator(true))
 		})
 		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
 			Ok(val) => global_settings.with_base_dir(val),

polkadot/cli/src/cli.rs

@@ -166,6 +166,10 @@ pub struct RunCmd {
 	///  **Dangerous!** Do not touch unless explicitly advised to.
 	#[arg(long, hide = true)]
 	pub collator_protocol_hold_off: Option<u64>,
+
+	/// Enable experimental collator protocol. TESTING ONLY! Don't use on production
+	#[arg(long, hide = true, default_value = "false")]
+	pub experimental_collator_protocol: bool,
 }
 
 #[allow(missing_docs)]

polkadot/cli/src/command.rs

@@ -257,6 +257,7 @@ where
 	// Parse collator protocol hold off value and get the list of the invlunerable collators.
 	let collator_protocol_hold_off = cli.run.collator_protocol_hold_off.map(Duration::from_millis);
 	let invulnerable_ah_collators = get_invulnerable_ah_collators(&chain_spec);
+	let experimental_collator_protocol = cli.run.experimental_collator_protocol;
 
 	runner.run_node_until_exit(move |config| async move {
 		let hwbench = (!cli.run.no_hardware_benchmarks)
@@ -292,6 +293,7 @@ where
 				keep_finalized_for: cli.run.keep_finalized_for,
 				invulnerable_ah_collators,
 				collator_protocol_hold_off,
+				experimental_collator_protocol,
 			},
 		)
 		.map(|full| full.task_manager)?;

polkadot/node/core/prospective-parachains/src/lib.rs

@@ -627,6 +627,7 @@ async fn handle_candidate_backed(
 				?relay_parent,
 				?para,
 				?is_active_leaf,
+				?candidate_hash,
 				"Candidate backed. Candidate chain for para: {:?}",
 				chain.best_chain_vec()
 			);

polkadot/node/network/collator-protocol/Cargo.toml

@@ -12,7 +12,7 @@ repository.workspace = true
 workspace = true
 
 [dependencies]
-async-trait = { workspace = true, optional = true }
+async-trait = { workspace = true }
 bitvec = { features = ["alloc"], workspace = true }
 futures = { workspace = true }
 futures-timer = { workspace = true }
@@ -31,15 +31,13 @@ polkadot-node-subsystem-util = { workspace = true, default-features = true }
 polkadot-primitives = { workspace = true, default-features = true }
 thiserror = { workspace = true }
 tokio-util = { workspace = true }
-# This should have really been a dev-dependency but clippy is complaining that it's not used with
-# experimental-collator-protocol disabled, while the rust compiler claims that having optional
-# dev-dependencies is not possible.
-tokio = { features = ["macros"], workspace = true, default-features = true, optional = true }
 
 [dev-dependencies]
 assert_matches = { workspace = true }
 rstest = { workspace = true }
+sc-network-types = { workspace = true, default-features = true }
 sp-tracing = { workspace = true }
+tokio = { features = ["macros"], workspace = true, default-features = true }
 
 codec = { features = ["std"], workspace = true, default-features = true }
 sc-keystore = { workspace = true, default-features = true }
@@ -53,4 +51,3 @@ polkadot-primitives-test-helpers = { workspace = true }
 
 [features]
 default = []
-experimental-collator-protocol = ["async-trait", "tokio"]

polkadot/node/network/collator-protocol/src/lib.rs

@@ -44,9 +44,11 @@ use polkadot_node_subsystem::{errors::SubsystemError, overseer, DummySubsystem,
 
 mod collator_side;
 mod validator_side;
-#[cfg(feature = "experimental-collator-protocol")]
 mod validator_side_experimental;
 
+// TODO: move into validator_side_experimental once `validator_side` is retired
+mod validator_side_metrics;
+
 const LOG_TARGET: &'static str = "parachain::collator-protocol";
 const LOG_TARGET_STATS: &'static str = "parachain::collator-protocol::stats";
 
@@ -84,7 +86,6 @@ pub enum ProtocolSide {
 		collator_protocol_hold_off: Option<Duration>,
 	},
 	/// Experimental variant of the validator side. Do not use in production.
-	#[cfg(feature = "experimental-collator-protocol")]
 	ValidatorExperimental {
 		/// The keystore holding validator keys.
 		keystore: KeystorePtr,
@@ -114,9 +115,6 @@ pub struct CollatorProtocolSubsystem {
 #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)]
 impl CollatorProtocolSubsystem {
 	/// Start the collator protocol.
-	/// If `id` is `Some` this is a collator side of the protocol.
-	/// If `id` is `None` this is a validator side of the protocol.
-	/// Caller must provide a registry for prometheus metrics.
 	pub fn new(protocol_side: ProtocolSide) -> Self {
 		Self { protocol_side }
 	}
@@ -150,7 +148,6 @@ impl<Context> CollatorProtocolSubsystem {
 				.map_err(|e| SubsystemError::with_origin("collator-protocol", e))
 				.boxed()
 			},
-			#[cfg(feature = "experimental-collator-protocol")]
 			ProtocolSide::ValidatorExperimental { keystore, metrics } =>
 				validator_side_experimental::run(ctx, keystore, metrics)
 					.map_err(|e| SubsystemError::with_origin("collator-protocol", e))