diff --git a/Cargo.lock b/Cargo.lock index dac2c8de..1e8d029d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1318,7 +1318,7 @@ dependencies = [ [[package]] name = "bls" version = "0.2.0" -source = "git+https://github.com/sigp/lighthouse?rev=8d058e4040b765a96aa4968f4167af7571292be2#8d058e4040b765a96aa4968f4167af7571292be2" +source = "git+https://github.com/sigp/lighthouse?tag=v7.0.1#e42406d7b79a85ad4622f3a7440ff6468ac4c9e1" dependencies = [ "alloy-primitives", "arbitrary", @@ -1496,6 +1496,7 @@ dependencies = [ "jsonwebtoken", "pbkdf2 0.12.2", "rand 0.9.0", + "rayon", "reqwest", "serde", "serde_json", @@ -1845,6 +1846,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -2243,11 +2263,11 @@ dependencies = [ [[package]] name = "eth2_key_derivation" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?rev=8d058e4040b765a96aa4968f4167af7571292be2#8d058e4040b765a96aa4968f4167af7571292be2" +source = "git+https://github.com/sigp/lighthouse?tag=v7.0.1#e42406d7b79a85ad4622f3a7440ff6468ac4c9e1" dependencies = [ "bls", "num-bigint-dig", - "ring 0.16.20", + "ring", "sha2 0.9.9", "zeroize", ] @@ -2255,7 +2275,7 @@ dependencies = [ [[package]] name = "eth2_keystore" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?rev=8d058e4040b765a96aa4968f4167af7571292be2#8d058e4040b765a96aa4968f4167af7571292be2" +source = "git+https://github.com/sigp/lighthouse?tag=v7.0.1#e42406d7b79a85ad4622f3a7440ff6468ac4c9e1" dependencies = [ "aes 0.7.5", "bls", @@ -2281,7 +2301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c853bd72c9e5787f8aafc3df2907c2ed03cff3150c3acd94e2e53a98ab70a8ab" dependencies = [ "cpufeatures", - "ring 0.17.14", + "ring", "sha2 0.10.8", ] @@ -2400,7 +2420,7 @@ dependencies = [ [[package]] name = "fixed_bytes" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?rev=8d058e4040b765a96aa4968f4167af7571292be2#8d058e4040b765a96aa4968f4167af7571292be2" +source = "git+https://github.com/sigp/lighthouse?tag=v7.0.1#e42406d7b79a85ad4622f3a7440ff6468ac4c9e1" dependencies = [ "alloy-primitives", "safe_arith", @@ -3179,7 +3199,7 @@ checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ "base64 0.22.1", "js-sys", - "ring 0.17.14", + "ring", "serde", "serde_json", ] @@ -3223,7 +3243,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin", ] [[package]] @@ -4004,6 +4024,26 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "recvmsg" version = "1.0.0" @@ -4119,21 +4159,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.14" @@ -4144,7 +4169,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.15", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] @@ -4260,7 +4285,7 @@ checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "log", "once_cell", - "ring 0.17.14", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -4288,9 +4313,9 @@ version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ - "ring 0.17.14", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -4320,7 +4345,7 @@ checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "safe_arith" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?rev=8d058e4040b765a96aa4968f4167af7571292be2#8d058e4040b765a96aa4968f4167af7571292be2" +source = "git+https://github.com/sigp/lighthouse?tag=v7.0.1#e42406d7b79a85ad4622f3a7440ff6468ac4c9e1" [[package]] name = "salsa20" @@ -4679,12 +4704,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -5472,12 +5491,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 283caf0d..651b4bd4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ color-eyre = "0.6.3" ctr = "0.9.2" derive_more = { version = "2.0.1", features = ["deref", "display", "from", "into"] } docker-compose-types = "0.16.0" -eth2_keystore = { git = "https://github.com/sigp/lighthouse", rev = "8d058e4040b765a96aa4968f4167af7571292be2" } +eth2_keystore = { git = "https://github.com/sigp/lighthouse", tag = "v7.0.1" } ethereum_serde_utils = "0.7.0" ethereum_ssz = "0.8" ethereum_ssz_derive = "0.8" @@ -51,6 +51,7 @@ pbkdf2 = "0.12.2" prometheus = "0.13.4" prost = "0.13.4" rand = { version = "0.9", features = ["os_rng"] } +rayon = "1.10.0" reqwest = { version = "0.12.4", features = ["json", "stream"] } serde = { version = "1.0.202", features = ["derive"] } serde_json = "1.0.117" diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index df78b046..5d71e52a 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -21,8 +21,10 @@ ethereum_serde_utils.workspace = true ethereum_ssz.workspace = true ethereum_ssz_derive.workspace = true eyre.workspace = true +jsonwebtoken.workspace = true pbkdf2.workspace = true rand.workspace = true +rayon.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true @@ -40,4 +42,3 @@ tree_hash.workspace = true tree_hash_derive.workspace = true unicode-normalization.workspace = true url.workspace = true -jsonwebtoken.workspace = true diff --git a/crates/common/src/signer/loader.rs b/crates/common/src/signer/loader.rs index 9e60d85b..d678795e 100644 --- a/crates/common/src/signer/loader.rs +++ b/crates/common/src/signer/loader.rs @@ -1,5 +1,4 @@ use std::{ - ffi::OsStr, fs::{self, File}, io::BufReader, path::PathBuf, @@ -11,8 +10,9 @@ use aes::{ }; use alloy::{primitives::hex::FromHex, rpc::types::beacon::BlsPublicKey}; use eth2_keystore::{json_keystore::JsonKeystore, Keystore}; -use eyre::{eyre, Context, OptionExt}; +use eyre::{eyre, Context}; use pbkdf2::{hmac, pbkdf2}; +use rayon::prelude::*; use serde::{de, Deserialize, Deserializer, Serialize}; use tracing::warn; use unicode_normalization::UnicodeNormalization; @@ -113,31 +113,34 @@ fn load_from_lighthouse_format( keys_path: PathBuf, secrets_path: PathBuf, ) -> eyre::Result> { - let entries = fs::read_dir(keys_path.clone())?; - - let mut signers = Vec::new(); + let paths: Vec<_> = + fs::read_dir(&keys_path)?.map(|res| res.map(|e| e.path())).collect::>()?; + + let signers = paths + .into_par_iter() + .filter_map(|path| { + if !path.is_dir() { + return None + } - for entry in entries { - let entry = entry?; - let path = entry.path(); + let maybe_pubkey = path.file_name().and_then(|d| d.to_str())?; + let Ok(pubkey) = BlsPublicKey::from_hex(maybe_pubkey) else { + warn!("Invalid pubkey: {}", maybe_pubkey); + return None + }; - // Check if file name is a pubkey - if path.is_dir() { - if let Some(maybe_pubkey) = path.file_name().and_then(|d| d.to_str()) { - if let Ok(pubkey) = BlsPublicKey::from_hex(maybe_pubkey) { - let ks_path = keys_path.join(maybe_pubkey).join("voting-keystore.json"); - let pw_path = secrets_path.join(pubkey.to_string()); + let ks_path = keys_path.join(maybe_pubkey).join("voting-keystore.json"); + let pw_path = secrets_path.join(pubkey.to_string()); - match load_one(ks_path, pw_path) { - Ok(signer) => signers.push(signer), - Err(e) => warn!("Failed to load signer for pubkey: {}, err: {}", pubkey, e), - } - } else { - warn!("Invalid pubkey: {}", maybe_pubkey); + match load_one(ks_path, pw_path) { + Ok(signer) => Some(signer), + Err(e) => { + warn!("Failed to load signer for pubkey: {}, err: {}", pubkey, e); + None } } - } - } + }) + .collect(); Ok(signers) } @@ -146,34 +149,31 @@ fn load_from_teku_format( keys_path: PathBuf, secrets_path: PathBuf, ) -> eyre::Result> { - let entries = fs::read_dir(keys_path.clone())?; - let mut signers = Vec::new(); - - for entry in entries { - let entry = entry?; - let path = entry.path(); - - if path.is_dir() { - warn!("Path {path:?} is a dir"); - continue; - } - - let file_name = path - .file_name() - .and_then(OsStr::to_str) - .ok_or_eyre("File name not valid")? - .rsplit_once(".") - .ok_or_eyre("File doesn't have extension")? - .0; - - match load_one( - keys_path.join(format!("{file_name}.json")), - secrets_path.join(format!("{file_name}.txt")), - ) { - Ok(signer) => signers.push(signer), - Err(e) => warn!("Sign load error: {e}"), - } - } + let paths: Vec<_> = + fs::read_dir(&keys_path)?.map(|res| res.map(|e| e.path())).collect::>()?; + + let signers = paths + .into_par_iter() + .filter_map(|path| { + if path.is_dir() { + warn!("Path {path:?} is a dir"); + return None; + } + + let file_name = path.file_name()?.to_str()?.rsplit_once(".")?.0; + + match load_one( + keys_path.join(format!("{file_name}.json")), + secrets_path.join(format!("{file_name}.txt")), + ) { + Ok(signer) => Some(signer), + Err(e) => { + warn!("Sign load error: {e}"); + None + } + } + }) + .collect(); Ok(signers) } @@ -182,23 +182,26 @@ fn load_from_lodestar_format( keys_path: PathBuf, password_path: PathBuf, ) -> eyre::Result> { - let entries = fs::read_dir(keys_path)?; - let mut signers = Vec::new(); - - for entry in entries { - let entry = entry?; - let path = entry.path(); - - if path.is_dir() { - warn!("Path {path:?} is a dir"); - continue; - } - - match load_one(path, password_path.clone()) { - Ok(signer) => signers.push(signer), - Err(e) => warn!("Sign load error: {e}"), - } - } + let paths: Vec<_> = + fs::read_dir(&keys_path)?.map(|res| res.map(|e| e.path())).collect::>()?; + + let signers = paths + .into_par_iter() + .filter_map(|path| { + if path.is_dir() { + warn!("Path {path:?} is a dir"); + return None; + } + + match load_one(path, password_path.clone()) { + Ok(signer) => Some(signer), + Err(e) => { + warn!("Sign load error: {e}"); + None + } + } + }) + .collect(); Ok(signers) } @@ -252,19 +255,19 @@ fn load_from_prysm_format( let mut cipher = ctr::Ctr128BE::::new_from_slices(&decryption_key[..16], &keystore.iv) .map_err(|_| eyre!("Invalid key or nonce"))?; - let mut buf = vec![0u8; ciphertext.len()].into_boxed_slice(); + let mut buf = vec![0u8; ciphertext.len()]; cipher .apply_keystream_b2b(&ciphertext, &mut buf) .map_err(|_| eyre!("Failed decrypting accounts"))?; let decrypted_keystore: PrysmDecryptedKeystore = serde_json::from_slice(&buf).map_err(|e| eyre!("Failed reading json: {e}"))?; - let mut signers = Vec::with_capacity(decrypted_keystore.private_keys.len()); - - for key in decrypted_keystore.private_keys { - let signer = ConsensusSigner::new_from_bytes(&key)?; - signers.push(signer); - } + let signers = decrypted_keystore + .private_keys + .into_par_iter() + .map(|pk| ConsensusSigner::new_from_bytes(&pk)) + .collect::, _>>() + .context("failed to load signers")?; Ok(signers) } diff --git a/crates/signer/Cargo.toml b/crates/signer/Cargo.toml index 69f92886..4e38da88 100644 --- a/crates/signer/Cargo.toml +++ b/crates/signer/Cargo.toml @@ -16,6 +16,7 @@ cb-metrics.workspace = true eyre.workspace = true futures.workspace = true headers.workspace = true +jsonwebtoken.workspace = true lazy_static.workspace = true prometheus.workspace = true prost.workspace = true @@ -26,7 +27,6 @@ tonic.workspace = true tracing.workspace = true tree_hash.workspace = true uuid.workspace = true -jsonwebtoken.workspace = true [build-dependencies] tonic-build.workspace = true