Frontier Upgrade to polkadot-v0.9.43

Cargo.toml

@@ -86,10 +86,9 @@ pallet-bridge-parachains = { git = "https://github.com/darwinia-network/darwinia
 pallet-fee-market        = { git = "https://github.com/darwinia-network/darwinia-messages-substrate", branch = "polkadot-v0.9.43", default-features = false }
 
 # frontier
-fc-cli                         = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43" }
 fc-consensus                   = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43" }
 fc-db                          = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43" }
-fc-mapping-sync                = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43" }
+fc-mapping-sync                = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43", features = ["sql"] }
 fc-rpc                         = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43", features = ["rpc-binary-search-estimate"] }
 fc-rpc-core                    = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43" }
 fc-storage                     = { git = "https://github.com/paritytech/frontier", branch = "polkadot-v0.9.43" }
@@ -111,9 +110,7 @@ moonbeam-evm-tracer            = { git = "https://github.com/darwinia-network/mo
 moonbeam-primitives-ext        = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev", default-features = false }
 moonbeam-rpc-debug             = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev" }
 moonbeam-rpc-primitives-debug  = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev", default-features = false }
-moonbeam-rpc-primitives-txpool = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev", default-features = false }
 moonbeam-rpc-trace             = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev" }
-moonbeam-rpc-txpool            = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev" }
 pallet-ethereum-xcm            = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev", default-features = false }
 precompile-utils               = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev", default-features = false }
 xcm-primitives                 = { git = "https://github.com/darwinia-network/moonbeam", branch = "dev", default-features = false }

node/Cargo.toml

@@ -39,7 +39,6 @@ pangolin-runtime = { workspace = true, optional = true }
 pangoro-runtime  = { workspace = true, optional = true }
 
 # frontier
-fc-cli          = { workspace = true }
 fc-db           = { workspace = true }
 fc-mapping-sync = { workspace = true }
 fc-rpc          = { workspace = true }
@@ -52,9 +51,7 @@ fp-rpc          = { workspace = true, features = ["std"] }
 moonbeam-primitives-ext        = { workspace = true, optional = true, features = ["std"] }
 moonbeam-rpc-debug             = { workspace = true }
 moonbeam-rpc-primitives-debug  = { workspace = true, features = ["std"] }
-moonbeam-rpc-primitives-txpool = { workspace = true, features = ["std"] }
 moonbeam-rpc-trace             = { workspace = true }
-moonbeam-rpc-txpool            = { workspace = true }
 
 # polkadot
 polkadot-cli        = { workspace = true }

node/src/cli.rs

@@ -49,9 +49,6 @@ pub enum Subcommand {
 	/// Export the genesis wasm of the parachain.
 	ExportGenesisWasm(cumulus_client_cli::ExportGenesisWasmCommand),
 
-	/// Db meta columns information.
-	FrontierDb(fc_cli::FrontierDbCmd),
-
 	/// Sub-commands concerned with benchmarking.
 	/// The pallet benchmarking moved to the `pallet` sub-command.
 	#[cfg(feature = "runtime-benchmarks")]
@@ -131,6 +128,16 @@ impl RelayChainCli {
 	}
 }
 
+/// Available frontier backend types.
+#[derive(Debug, Copy, Clone, Default, clap::ValueEnum)]
+pub enum FrontierBackendType {
+	/// Either RocksDb or ParityDb as per inherited from the global backend settings.
+	#[default]
+	KeyValue,
+	/// Sql database with custom log indexing.
+	Sql,
+}
+
 #[derive(Debug, clap::Parser)]
 pub struct EthArgs {
 	/// Enable EVM tracing functionalities.
@@ -172,6 +179,27 @@ pub struct EthArgs {
 	/// Maximum fee history cache size.
 	#[arg(long, default_value = "2048")]
 	pub fee_history_limit: u64,
+
+	/// Sets the frontier backend type (KeyValue or Sql)
+	#[arg(long, value_enum, ignore_case = true, default_value_t = FrontierBackendType::default())]
+	pub frontier_backend_type: FrontierBackendType,
+
+	// Sets the SQL backend's pool size.
+	#[arg(long, default_value = "100")]
+	pub frontier_sql_backend_pool_size: u32,
+
+	/// Sets the SQL backend's query timeout in number of VM ops.
+	#[arg(long, default_value = "10000000")]
+	pub frontier_sql_backend_num_ops_timeout: u32,
+
+	/// Sets the SQL backend's auxiliary thread limit.
+	#[arg(long, default_value = "4")]
+	pub frontier_sql_backend_thread_count: u32,
+
+	/// Sets the SQL backend's query timeout in number of VM ops.
+	/// Default value is 200MB.
+	#[arg(long, default_value = "209715200")]
+	pub frontier_sql_backend_cache_size: u64,
 }
 impl EthArgs {
 	pub fn build_eth_rpc_config(&self) -> EthRpcConfig {
@@ -185,6 +213,11 @@ impl EthArgs {
 			eth_log_block_cache: self.eth_log_block_cache,
 			max_past_logs: self.max_past_logs,
 			fee_history_limit: self.fee_history_limit,
+			frontier_backend_type: self.frontier_backend_type,
+			frontier_sql_backend_pool_size: self.frontier_sql_backend_pool_size,
+			frontier_sql_backend_num_ops_timeout: self.frontier_sql_backend_num_ops_timeout,
+			frontier_sql_backend_thread_count: self.frontier_sql_backend_thread_count,
+			frontier_sql_backend_cache_size: self.frontier_sql_backend_cache_size,
 		}
 	}
 }
@@ -217,4 +250,9 @@ pub struct EthRpcConfig {
 	pub eth_statuses_cache: usize,
 	pub fee_history_limit: u64,
 	pub max_past_logs: u32,
+	pub frontier_backend_type: FrontierBackendType,
+	pub frontier_sql_backend_pool_size: u32,
+	pub frontier_sql_backend_num_ops_timeout: u32,
+	pub frontier_sql_backend_thread_count: u32,
+	pub frontier_sql_backend_cache_size: u64,
 }

node/src/command.rs

@@ -25,21 +25,19 @@ use cumulus_primitives_core::ParaId;
 // darwinia
 use crate::{
 	chain_spec::*,
-	cli::{Cli, RelayChainCli, Subcommand},
+	cli::{Cli, FrontierBackendType, RelayChainCli, Subcommand},
 	frontier_service,
 	service::{self, *},
 };
 use dc_primitives::Block;
-// frontier
-use fc_db::frontier_database_dir;
 // substrate
 use sc_cli::{
 	CliConfiguration, DefaultConfigurationValues, ImportParams, KeystoreParams, NetworkParams,
 	Result, RuntimeVersion, SharedParams, SubstrateCli,
 };
 use sc_service::{
 	config::{BasePath, PrometheusConfig},
-	ChainSpec, DatabaseSource, PartialComponents,
+	ChainSpec, DatabaseSource,
 };
 use sp_core::{
 	crypto::{self, Ss58AddressFormatRegistry},
@@ -435,22 +433,48 @@ pub fn run() -> Result<()> {
 
 			set_default_ss58_version(chain_spec);
 			runner.sync_run(|config| {
-				// Remove Frontier DB.
+				// Remove Frontier off-chain db
 				let db_config_dir = frontier_service::db_config_dir(&config);
-				let frontier_database_config = match config.database {
-					DatabaseSource::RocksDb { .. } => DatabaseSource::RocksDb {
-						path: frontier_database_dir(&db_config_dir, "db"),
-						cache_size: 0,
-					},
-					DatabaseSource::ParityDb { .. } => DatabaseSource::ParityDb {
-						path: frontier_database_dir(&db_config_dir, "paritydb"),
-					},
-					_ =>
-						return Err(format!("Cannot purge `{:?}` database", config.database).into()),
+				match cli.eth_args.frontier_backend_type {
+					FrontierBackendType::KeyValue => {
+						let frontier_database_config = match config.database {
+							DatabaseSource::RocksDb { .. } => DatabaseSource::RocksDb {
+								path: fc_db::kv::frontier_database_dir(&db_config_dir, "db"),
+								cache_size: 0,
+							},
+							DatabaseSource::ParityDb { .. } => DatabaseSource::ParityDb {
+								path: fc_db::kv::frontier_database_dir(&db_config_dir, "paritydb"),
+							},
+							_ => {
+								return Err(format!(
+									"Cannot purge `{:?}` database",
+									config.database
+								)
+								.into())
+							}
+						};
+						cmd.base.run(frontier_database_config)?;
+					}
+					FrontierBackendType::Sql => {
+						let db_path = db_config_dir.join("sql");
+						match std::fs::remove_dir_all(&db_path) {
+							Ok(_) => {
+								println!("{:?} removed.", &db_path);
+							}
+							Err(ref err) if err.kind() == std::io::ErrorKind::NotFound => {
+								eprintln!("{:?} did not exist.", &db_path);
+							}
+							Err(err) => {
+								return Err(format!(
+									"Cannot purge `{:?}` database: {:?}",
+									db_path, err,
+								)
+								.into())
+							}
+						};
+					}
 				};
 
-				cmd.base.run(frontier_database_config)?;
-
 				let polkadot_cli = RelayChainCli::new(
 					&config,
 					[RelayChainCli::executable_name()].iter().chain(cli.relay_chain_args.iter()),
@@ -486,61 +510,6 @@ pub fn run() -> Result<()> {
 				cmd.run(&*spec)
 			})
 		},
-		Some(Subcommand::FrontierDb(cmd)) => {
-			let runner = cli.create_runner(cmd)?;
-
-			runner.sync_run(|config| {
-				let chain_spec = &config.chain_spec;
-
-				set_default_ss58_version(chain_spec);
-
-				#[cfg(feature = "crab-native")]
-				if chain_spec.is_crab() {
-					let PartialComponents { client, other: (frontier_backend, ..), .. } =
-						service::new_partial::<CrabRuntimeApi, CrabRuntimeExecutor>(
-							&config,
-							&cli.eth_args.build_eth_rpc_config(),
-						)?;
-
-					return cmd.run::<_, dc_primitives::Block>(client, frontier_backend);
-				}
-
-				#[cfg(feature = "darwinia-native")]
-				if chain_spec.is_darwinia() {
-					let PartialComponents { client, other: (frontier_backend, ..), .. } =
-						service::new_partial::<DarwiniaRuntimeApi, DarwiniaRuntimeExecutor>(
-							&config,
-							&cli.eth_args.build_eth_rpc_config(),
-						)?;
-
-					return cmd.run::<_, dc_primitives::Block>(client, frontier_backend);
-				}
-
-				#[cfg(feature = "pangolin-native")]
-				if chain_spec.is_pangolin() {
-					let PartialComponents { client, other: (frontier_backend, ..), .. } =
-						service::new_partial::<PangolinRuntimeApi, PangolinRuntimeExecutor>(
-							&config,
-							&cli.eth_args.build_eth_rpc_config(),
-						)?;
-
-					return cmd.run::<_, dc_primitives::Block>(client, frontier_backend);
-				}
-
-				#[cfg(feature = "pangoro-native")]
-				if chain_spec.is_pangoro() {
-					let PartialComponents { client, other: (frontier_backend, ..), .. } =
-						service::new_partial::<PangoroRuntimeApi, PangoroRuntimeExecutor>(
-							&config,
-							&cli.eth_args.build_eth_rpc_config(),
-						)?;
-
-					return cmd.run::<_, dc_primitives::Block>(client, frontier_backend);
-				}
-
-				panic!("No feature(crab-native, darwinia-native, pangolin-native, pangoro-native) is enabled!");
-			})
-		},
 		#[cfg(feature = "runtime-benchmarks")]
 		Some(Subcommand::Benchmark(cmd)) => {
 			// substrate

node/src/frontier_service.rs

@@ -24,22 +24,18 @@ use std::{path::PathBuf, sync::Arc, time::Duration};
 use futures::{future, StreamExt};
 use tokio::sync::Semaphore;
 // darwinia
-use crate::cli::{Cli, EthRpcConfig, TracingApi};
+use crate::cli::{EthRpcConfig, TracingApi};
 use dc_primitives::{BlockNumber, Hash, Hashing};
 // frontier
-use fc_db::Backend as FrontierBackend;
-use fc_mapping_sync::{
-	EthereumBlockNotification, EthereumBlockNotificationSinks, MappingSyncWorker, SyncStrategy,
-};
+use fc_mapping_sync::{EthereumBlockNotification, EthereumBlockNotificationSinks};
 use fc_rpc::{EthTask, OverrideHandle};
 use fc_rpc_core::types::{FeeHistoryCache, FeeHistoryCacheLimit, FilterPool};
 //  moonbeam
 use moonbeam_rpc_debug::{DebugHandler, DebugRequester};
 use moonbeam_rpc_trace::{CacheRequester as TraceFilterCacheRequester, CacheTask};
 // substrate
-use sc_cli::SubstrateCli;
 use sc_network_sync::SyncingService;
-use sc_service::{BasePath, Configuration, TaskManager};
+use sc_service::{Configuration, TaskManager};
 
 #[derive(Clone)]
 pub struct RpcRequesters {
@@ -52,7 +48,7 @@ pub fn spawn_frontier_tasks<B, BE, C>(
 	task_manager: &TaskManager,
 	client: Arc<C>,
 	backend: Arc<BE>,
-	frontier_backend: Arc<FrontierBackend<B>>,
+	frontier_backend: fc_db::Backend<B>,
 	filter_pool: Option<FilterPool>,
 	overrides: Arc<OverrideHandle<B>>,
 	fee_history_cache: FeeHistoryCache,
@@ -77,24 +73,47 @@ where
 	BE: 'static + sc_client_api::backend::Backend<B>,
 	BE::State: sc_client_api::backend::StateBackend<Hashing>,
 {
-	task_manager.spawn_essential_handle().spawn(
-		"frontier-mapping-sync-worker",
-		Some("frontier"),
-		MappingSyncWorker::new(
-			client.import_notification_stream(),
-			Duration::new(6, 0),
-			client.clone(),
-			backend.clone(),
-			overrides.clone(),
-			frontier_backend.clone(),
-			3,
-			0,
-			SyncStrategy::Parachain,
-			sync,
-			pubsub_notification_sinks,
-		)
-		.for_each(|()| future::ready(())),
-	);
+	match frontier_backend.clone() {
+		fc_db::Backend::KeyValue(bd) => {
+			task_manager.spawn_essential_handle().spawn(
+				"frontier-mapping-sync-worker",
+				Some("frontier"),
+				fc_mapping_sync::kv::MappingSyncWorker::new(
+					client.import_notification_stream(),
+					Duration::new(6, 0),
+					client.clone(),
+					backend.clone(),
+					overrides.clone(),
+					Arc::new(bd),
+					3,
+					0,
+					fc_mapping_sync::SyncStrategy::Parachain,
+					sync,
+					pubsub_notification_sinks,
+				)
+				.for_each(|()| future::ready(())),
+			);
+		},
+		fc_db::Backend::Sql(bd) => {
+			task_manager.spawn_essential_handle().spawn_blocking(
+				"frontier-mapping-sync-worker",
+				Some("frontier"),
+				fc_mapping_sync::sql::SyncWorker::run(
+					client.clone(),
+					backend.clone(),
+					Arc::new(bd),
+					client.import_notification_stream(),
+					fc_mapping_sync::sql::SyncWorkerConfig {
+						read_notification_timeout: Duration::from_secs(10),
+						check_indexed_blocks_interval: Duration::from_secs(60),
+					},
+					fc_mapping_sync::SyncStrategy::Parachain,
+					sync,
+					pubsub_notification_sinks,
+				),
+			);
+		},
+	}
 
 	// Spawn Frontier EthFilterApi maintenance task.
 	if let Some(filter_pool) = filter_pool {
@@ -142,7 +161,10 @@ where
 				let (debug_task, debug_requester) = DebugHandler::task(
 					Arc::clone(&client),
 					Arc::clone(&backend),
-					Arc::clone(&frontier_backend),
+					match frontier_backend {
+						fc_db::Backend::KeyValue(bd) => Arc::new(bd),
+						fc_db::Backend::Sql(bd) => Arc::new(bd),
+					},
 					Arc::clone(&permit_pool),
 					Arc::clone(&overrides),
 					eth_rpc_config.tracing_raw_max_memory_usage,
@@ -178,13 +200,7 @@ where
 	}
 }
 
+// TODO: CHECK THIS
 pub(crate) fn db_config_dir(config: &Configuration) -> PathBuf {
-	config
-		.base_path
-		.as_ref()
-		.map(|base_path| base_path.config_dir(config.chain_spec.id()))
-		.unwrap_or_else(|| {
-			BasePath::from_project("", "", &Cli::executable_name())
-				.config_dir(config.chain_spec.id())
-		})
+	config.base_path.config_dir(config.chain_spec.id())
 }