98.6% OF DEVELOPERS CANNOT REVIEW THIS PR! [read more...]

Cargo.toml

@@ -2,6 +2,14 @@
 name = "polkadot"
 path = "src/main.rs"
 
+[[bin]]
+name = "polkadot-execute-worker"
+path = "node/core/pvf/execute-worker/src/main.rs"
+
+[[bin]]
+name = "polkadot-prepare-worker"
+path = "node/core/pvf/prepare-worker/src/main.rs"
+
 [package]
 name = "polkadot"
 description = "Implementation of a `https://polkadot.network` node in Rust based on the Substrate framework."
@@ -22,10 +30,15 @@ version = "0.9.43"
 color-eyre = { version = "0.6.1", default-features = false }
 tikv-jemallocator = "0.5.0"
 
+# Needed for worker binaries.
+polkadot-node-core-pvf-common = { path = "node/core/pvf/common" }
+polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" }
+polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" }
+sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
+
 # Crates in our workspace, defined as dependencies so we can pass them feature flags.
 polkadot-cli = { path = "cli", features = [ "polkadot-native", "kusama-native", "westend-native", "rococo-native" ]  }
 polkadot-node-core-pvf = { path = "node/core/pvf" }
-polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" }
 polkadot-overseer = { path = "node/overseer" }
 
 [dev-dependencies]
@@ -226,6 +239,8 @@ license-file = ["LICENSE", "0"]
 maintainer-scripts = "scripts/packaging/deb-maintainer-scripts"
 assets = [
 	["target/release/polkadot", "/usr/bin/", "755"],
+	["target/release/polkadot-prepare-worker", "/usr/libexec/", "755"],
+	["target/release/polkadot-execute-worker", "/usr/libexec/", "755"],
 	["scripts/packaging/polkadot.service", "/lib/systemd/system/", "644"]
 ]
 conf-files = [

cli/src/cli.rs

@@ -17,6 +17,7 @@
 //! Polkadot CLI library.
 
 use clap::Parser;
+use std::path::PathBuf;
 
 #[allow(missing_docs)]
 #[derive(Debug, Parser)]
@@ -42,14 +43,6 @@ pub enum Subcommand {
 	/// Revert the chain to a previous state.
 	Revert(sc_cli::RevertCmd),
 
-	#[allow(missing_docs)]
-	#[command(name = "prepare-worker", hide = true)]
-	PvfPrepareWorker(ValidationWorkerCommand),
-
-	#[allow(missing_docs)]
-	#[command(name = "execute-worker", hide = true)]
-	PvfExecuteWorker(ValidationWorkerCommand),
-
 	/// Sub-commands concerned with benchmarking.
 	/// The pallet benchmarking moved to the `pallet` sub-command.
 	#[command(subcommand)]
@@ -75,17 +68,6 @@ pub enum Subcommand {
 	ChainInfo(sc_cli::ChainInfoCmd),
 }
 
-#[allow(missing_docs)]
-#[derive(Debug, Parser)]
-pub struct ValidationWorkerCommand {
-	/// The path to the validation host's socket.
-	#[arg(long)]
-	pub socket_path: String,
-	/// Calling node implementation version
-	#[arg(long)]
-	pub node_impl_version: String,
-}
-
 #[allow(missing_docs)]
 #[derive(Debug, Parser)]
 #[group(skip)]
@@ -148,6 +130,13 @@ pub struct RunCmd {
 	/// **Dangerous!** Do not touch unless explicitly adviced to.
 	#[arg(long)]
 	pub overseer_channel_capacity_override: Option<usize>,
+
+	/// Path to the directory where auxiliary worker binaries reside. If not specified, the main
+	/// binary's directory is searched first, then `/usr/libexec` is searched. If the path points to
+	/// an executable rather then directory, that executable is used both as preparation and
+	/// execution worker (supposed to be used for tests only).
+	#[arg(long, value_name = "PATH")]
+	pub workers_path: Option<PathBuf>,
 }
 
 #[allow(missing_docs)]

cli/src/command.rs

@@ -289,6 +289,7 @@ where
 			grandpa_pause,
 			enable_beefy,
 			jaeger_agent,
+			cli.run.workers_path,
 			None,
 			false,
 			overseer_gen,
@@ -421,50 +422,6 @@ pub fn run() -> Result<()> {
 				))
 			})?)
 		},
-		Some(Subcommand::PvfPrepareWorker(cmd)) => {
-			let mut builder = sc_cli::LoggerBuilder::new("");
-			builder.with_colors(false);
-			let _ = builder.init();
-
-			#[cfg(target_os = "android")]
-			{
-				return Err(sc_cli::Error::Input(
-					"PVF preparation workers are not supported under this platform".into(),
-				)
-				.into())
-			}
-
-			#[cfg(not(target_os = "android"))]
-			{
-				polkadot_node_core_pvf_prepare_worker::worker_entrypoint(
-					&cmd.socket_path,
-					Some(&cmd.node_impl_version),
-				);
-				Ok(())
-			}
-		},
-		Some(Subcommand::PvfExecuteWorker(cmd)) => {
-			let mut builder = sc_cli::LoggerBuilder::new("");
-			builder.with_colors(false);
-			let _ = builder.init();
-
-			#[cfg(target_os = "android")]
-			{
-				return Err(sc_cli::Error::Input(
-					"PVF execution workers are not supported under this platform".into(),
-				)
-				.into())
-			}
-
-			#[cfg(not(target_os = "android"))]
-			{
-				polkadot_node_core_pvf_execute_worker::worker_entrypoint(
-					&cmd.socket_path,
-					Some(&cmd.node_impl_version),
-				);
-				Ok(())
-			}
-		},
 		Some(Subcommand::Benchmark(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			let chain_spec = &runner.config().chain_spec;

node/core/candidate-validation/src/lib.rs

@@ -93,9 +93,10 @@ const DEFAULT_APPROVAL_EXECUTION_TIMEOUT: Duration = Duration::from_secs(12);
 pub struct Config {
 	/// The path where candidate validation can store compiled artifacts for PVFs.
 	pub artifacts_cache_path: PathBuf,
-	/// The path to the executable which can be used for spawning PVF compilation & validation
-	/// workers.
-	pub program_path: PathBuf,
+	/// Path to the preparation worker binary
+	pub prep_worker_path: PathBuf,
+	/// Path to the execution worker binary
+	pub exec_worker_path: PathBuf,
 }
 
 /// The candidate validation subsystem.
@@ -129,7 +130,8 @@ impl<Context> CandidateValidationSubsystem {
 			self.metrics,
 			self.pvf_metrics,
 			self.config.artifacts_cache_path,
-			self.config.program_path,
+			self.config.prep_worker_path,
+			self.config.exec_worker_path,
 		)
 		.map_err(|e| SubsystemError::with_origin("candidate-validation", e))
 		.boxed();
@@ -143,10 +145,11 @@ async fn run<Context>(
 	metrics: Metrics,
 	pvf_metrics: polkadot_node_core_pvf::Metrics,
 	cache_path: PathBuf,
-	program_path: PathBuf,
+	prep_worker_path: PathBuf,
+	exec_worker_path: PathBuf,
 ) -> SubsystemResult<()> {
 	let (validation_host, task) = polkadot_node_core_pvf::start(
-		polkadot_node_core_pvf::Config::new(cache_path, program_path),
+		polkadot_node_core_pvf::Config::new(cache_path, prep_worker_path, exec_worker_path),
 		pvf_metrics,
 	);
 	ctx.spawn_blocking("pvf-validation-host", task.boxed())?;

node/core/pvf/common/src/worker/mod.rs

@@ -34,37 +34,63 @@ use tokio::{io, net::UnixStream, runtime::Runtime};
 #[macro_export]
 macro_rules! decl_worker_main {
 	($expected_command:expr, $entrypoint:expr) => {
+		fn print_help(expected_command: &str) {
+			let worker_version = $crate::worker::worker_version();
+
+			println!("{} {}", expected_command, worker_version);
+			println!();
+			println!("PVF worker that is called by polkadot.");
+		}
+
 		fn main() {
 			::sp_tracing::try_init_simple();
 
 			let args = std::env::args().collect::<Vec<_>>();
-			if args.len() < 3 {
-				panic!("wrong number of arguments");
+			if args.len() == 1 {
+				print_help($expected_command);
+				return
+			}
+
+			match args[1].as_ref() {
+				"--help" => {
+					print_help($expected_command);
+					return
+				},
+				"--version" => {
+					println!("{}", $crate::worker::worker_version());
+					return
+				},
+				subcommand => {
+					// Must be passed for compatibility with the single-binary test workers.
+					if subcommand != $expected_command {
+						panic!(
+							"trying to run {} binary with the {} subcommand",
+							$expected_command, subcommand
+						)
+					}
+				},
 			}
 
-			let mut version = None;
+			let mut node_version = None;
 			let mut socket_path: &str = "";
 
-			for i in 2..args.len() {
+			for i in (2..args.len()).step_by(2) {
 				match args[i].as_ref() {
 					"--socket-path" => socket_path = args[i + 1].as_str(),
-					"--node-version" => version = Some(args[i + 1].as_str()),
-					_ => (),
+					"--node-impl-version" => node_version = Some(args[i + 1].as_str()),
+					arg => panic!("Unexpected argument found: {}", arg),
 				}
 			}
 
-			let subcommand = &args[1];
-			if subcommand != $expected_command {
-				panic!(
-					"trying to run {} binary with the {} subcommand",
-					$expected_command, subcommand
-				)
-			}
-			$entrypoint(&socket_path, version);
+			$entrypoint(&socket_path, node_version);
 		}
 	};
 }
 
+pub fn worker_version() -> &'static str {
+	env!("SUBSTRATE_CLI_IMPL_VERSION")
+}
+
 /// Some allowed overhead that we account for in the "CPU time monitor" thread's sleeps, on the
 /// child process.
 pub const JOB_TIMEOUT_OVERHEAD: Duration = Duration::from_millis(50);
@@ -88,11 +114,14 @@ pub fn worker_event_loop<F, Fut>(
 	gum::debug!(target: LOG_TARGET, %worker_pid, "starting pvf worker ({})", debug_id);
 
 	// Check for a mismatch between the node and worker versions.
-	if let Some(version) = node_version {
-		if version != env!("SUBSTRATE_CLI_IMPL_VERSION") {
+	if let Some(node_version) = node_version {
+		let worker_version = env!("SUBSTRATE_CLI_IMPL_VERSION");
+		if node_version != worker_version {
 			gum::error!(
 				target: LOG_TARGET,
 				%worker_pid,
+				%node_version,
+				%worker_version,
 				"Node and worker version mismatch, node needs restarting, forcing shutdown",
 			);
 			kill_parent_node_in_emergency();

node/core/pvf/execute-worker/src/main.rs

@@ -0,0 +1,22 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Execute worker.
+
+polkadot_node_core_pvf_common::decl_worker_main!(
+	"execute-worker",
+	polkadot_node_core_pvf_execute_worker::worker_entrypoint
+);