[Merged by Bors] - Post merge local testnets

.github/workflows/local-testnet.yml

@@ -25,9 +25,23 @@ jobs:
         uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
         with:
             repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install anvil
-        run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
-
+      - name: Install geth (ubuntu)
+        if: matrix.os == 'ubuntu-22.04'
+        run: |
+          sudo add-apt-repository -y ppa:ethereum/ethereum
+          sudo apt-get update
+          sudo apt-get install ethereum
+      - name: Install geth (mac)
+        if: matrix.os == 'macos-12'
+        run: |
+          brew tap ethereum/ethereum
+          brew install ethereum
+      - name: Install GNU sed & GNU grep
+        if: matrix.os == 'macos-12'
+        run: |
+          brew install gnu-sed grep
+          echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> $GITHUB_PATH
+          echo "$(brew --prefix)/opt/grep/libexec/gnubin" >> $GITHUB_PATH
       # https://github.com/actions/cache/blob/main/examples.md#rust---cargo
       - uses: actions/cache@v3
         id: cache-cargo
@@ -44,7 +58,7 @@ jobs:
         run: make && make install-lcli
 
       - name: Start local testnet
-        run: ./start_local_testnet.sh && sleep 60
+        run: ./start_local_testnet.sh genesis.json && sleep 60
         working-directory: scripts/local_testnet
 
       - name: Print logs
@@ -60,7 +74,7 @@ jobs:
         working-directory: scripts/local_testnet
 
       - name: Start local testnet with blinded block production
-        run: ./start_local_testnet.sh -p && sleep 60
+        run: ./start_local_testnet.sh -p genesis.json && sleep 60
         working-directory: scripts/local_testnet
 
       - name: Print logs for blinded block testnet
@@ -69,4 +83,4 @@ jobs:
 
       - name: Stop local testnet with blinded block production
         run: ./stop_local_testnet.sh
-        working-directory: scripts/local_testnet
+        working-directory: scripts/local_testnet

.github/workflows/test-suite.yml

@@ -228,8 +228,6 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install anvil
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
     - name: Run the beacon chain sim without an eth1 connection
       run: cargo run --release --bin simulator no-eth1-sim
   syncing-simulator-ubuntu:
@@ -260,20 +258,23 @@ jobs:
       uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-    - name: Install anvil
-      run: cargo install --git https://github.com/foundry-rs/foundry --locked anvil
+    - name: Install geth
+      run: |
+          sudo add-apt-repository -y ppa:ethereum/ethereum
+          sudo apt-get update
+          sudo apt-get install ethereum
     - name: Install lighthouse and lcli
       run: |
           make
           make install-lcli
-    - name: Run the doppelganger protection success test script
+    - name: Run the doppelganger protection failure test script
       run: |
           cd scripts/tests
-          ./doppelganger_protection.sh success
-    - name: Run the doppelganger protection failure test script
+          ./doppelganger_protection.sh failure genesis.json
+    - name: Run the doppelganger protection success test script
       run: |
           cd scripts/tests
-          ./doppelganger_protection.sh failure
+          ./doppelganger_protection.sh success genesis.json
   execution-engine-integration-ubuntu:
     name: execution-engine-integration-ubuntu
     runs-on: ubuntu-latest

lcli/Cargo.toml

@@ -21,6 +21,7 @@ env_logger = "0.9.0"
 types = { path = "../consensus/types" }
 state_processing = { path = "../consensus/state_processing" }
 int_to_bytes = { path = "../consensus/int_to_bytes" }
+ethereum_hashing = "1.0.0-beta.2"
 ethereum_ssz = "0.5.0"
 environment = { path = "../lighthouse/environment" }
 eth2_network_config = { path = "../common/eth2_network_config" }
@@ -41,6 +42,7 @@ snap = "1.0.1"
 beacon_chain = { path = "../beacon_node/beacon_chain" }
 store = { path = "../beacon_node/store" }
 malloc_utils = { path = "../common/malloc_utils" }
+rayon = "1.7.0"
 
 [package.metadata.cargo-udeps.ignore]
 normal = ["malloc_utils"]

lcli/src/main.rs

@@ -10,6 +10,7 @@ mod generate_bootnode_enr;
 mod indexed_attestations;
 mod insecure_validators;
 mod interop_genesis;
+mod mnemonic_validators;
 mod new_testnet;
 mod parse_ssz;
 mod replace_state_pubkeys;
@@ -449,6 +450,22 @@ fn main() {
                             "If present, a interop-style genesis.ssz file will be generated.",
                         ),
                 )
+                .arg(
+                    Arg::with_name("derived-genesis-state")
+                        .long("derived-genesis-state")
+                        .takes_value(false)
+                        .help(
+                            "If present, a genesis.ssz file will be generated with keys generated from a given mnemonic.",
+                        ),
+                )
+                .arg(
+                    Arg::with_name("mnemonic-phrase")
+                        .long("mnemonic-phrase")
+                        .value_name("MNEMONIC_PHRASE")
+                        .takes_value(true)
+                        .requires("derived-genesis-state")
+                        .help("The mnemonic with which we generate the validator keys for a derived genesis state"),
+                )
                 .arg(
                     Arg::with_name("min-genesis-time")
                         .long("min-genesis-time")
@@ -568,14 +585,32 @@ fn main() {
                         ),
                 )
                 .arg(
-                    Arg::with_name("merge-fork-epoch")
-                        .long("merge-fork-epoch")
+                    Arg::with_name("bellatrix-fork-epoch")
+                        .long("bellatrix-fork-epoch")
                         .value_name("EPOCH")
                         .takes_value(true)
                         .help(
                             "The epoch at which to enable the Merge hard fork",
                         ),
                 )
+                .arg(
+                    Arg::with_name("capella-fork-epoch")
+                        .long("capella-fork-epoch")
+                        .value_name("EPOCH")
+                        .takes_value(true)
+                        .help(
+                            "The epoch at which to enable the Capella hard fork",
+                        ),
+                )
+                .arg(
+                    Arg::with_name("ttd")
+                        .long("ttd")
+                        .value_name("TTD")
+                        .takes_value(true)
+                        .help(
+                            "The terminal total difficulty",
+                        ),
+                )
                 .arg(
                     Arg::with_name("eth1-block-hash")
                         .long("eth1-block-hash")
@@ -695,13 +730,44 @@ fn main() {
                         .long("count")
                         .value_name("COUNT")
                         .takes_value(true)
+                        .required(true)
+                        .help("Produces validators in the range of 0..count."),
+                )
+                .arg(
+                    Arg::with_name("base-dir")
+                        .long("base-dir")
+                        .value_name("BASE_DIR")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The base directory where validator keypairs and secrets are stored"),
+                )
+                .arg(
+                    Arg::with_name("node-count")
+                        .long("node-count")
+                        .value_name("NODE_COUNT")
+                        .takes_value(true)
+                        .help("The number of nodes to divide the validator keys to"),
+                )
+        )
+        .subcommand(
+            SubCommand::with_name("mnemonic-validators")
+                .about("Produces validator directories by deriving the keys from \
+                        a mnemonic. For testing purposes only, DO NOT USE IN \
+                        PRODUCTION!")
+                .arg(
+                    Arg::with_name("count")
+                        .long("count")
+                        .value_name("COUNT")
+                        .takes_value(true)
+                        .required(true)
                         .help("Produces validators in the range of 0..count."),
                 )
                 .arg(
                     Arg::with_name("base-dir")
                         .long("base-dir")
                         .value_name("BASE_DIR")
                         .takes_value(true)
+                        .required(true)
                         .help("The base directory where validator keypairs and secrets are stored"),
                 )
                 .arg(
@@ -711,6 +777,14 @@ fn main() {
                         .takes_value(true)
                         .help("The number of nodes to divide the validator keys to"),
                 )
+                .arg(
+                    Arg::with_name("mnemonic-phrase")
+                        .long("mnemonic-phrase")
+                        .value_name("MNEMONIC_PHRASE")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The mnemonic with which we generate the validator keys"),
+                )
         )
         .subcommand(
             SubCommand::with_name("indexed-attestations")
@@ -853,6 +927,8 @@ fn run<T: EthSpec>(
             .map_err(|e| format!("Failed to run generate-bootnode-enr command: {}", e)),
         ("insecure-validators", Some(matches)) => insecure_validators::run(matches)
             .map_err(|e| format!("Failed to run insecure-validators command: {}", e)),
+        ("mnemonic-validators", Some(matches)) => mnemonic_validators::run(matches)
+            .map_err(|e| format!("Failed to run mnemonic-validators command: {}", e)),
         ("indexed-attestations", Some(matches)) => indexed_attestations::run::<T>(matches)
             .map_err(|e| format!("Failed to run indexed-attestations command: {}", e)),
         ("block-root", Some(matches)) => block_root::run::<T>(env, matches)

lcli/src/mnemonic_validators.rs

@@ -0,0 +1,104 @@
+use account_utils::eth2_keystore::{keypair_from_secret, Keystore, KeystoreBuilder};
+use account_utils::random_password;
+use clap::ArgMatches;
+use eth2_wallet::bip39::Seed;
+use eth2_wallet::bip39::{Language, Mnemonic};
+use eth2_wallet::{recover_validator_secret_from_mnemonic, KeyType};
+use rayon::prelude::*;
+use std::fs;
+use std::path::PathBuf;
+use validator_dir::Builder as ValidatorBuilder;
+
+/// Generates validator directories with keys derived from the given mnemonic.
+pub fn generate_validator_dirs(
+    indices: &[usize],
+    mnemonic_phrase: &str,
+    validators_dir: PathBuf,
+    secrets_dir: PathBuf,
+) -> Result<(), String> {
+    if !validators_dir.exists() {
+        fs::create_dir_all(&validators_dir)
+            .map_err(|e| format!("Unable to create validators dir: {:?}", e))?;
+    }
+
+    if !secrets_dir.exists() {
+        fs::create_dir_all(&secrets_dir)
+            .map_err(|e| format!("Unable to create secrets dir: {:?}", e))?;
+    }
+    let mnemonic = Mnemonic::from_phrase(mnemonic_phrase, Language::English).map_err(|e| {
+        format!(
+            "Unable to derive mnemonic from string {:?}: {:?}",
+            mnemonic_phrase, e
+        )
+    })?;
+
+    let seed = Seed::new(&mnemonic, "");
+
+    let _: Vec<_> = indices
+        .par_iter()
+        .map(|index| {
+            let voting_password = random_password();
+
+            let derive = |key_type: KeyType, password: &[u8]| -> Result<Keystore, String> {
+                let (secret, path) = recover_validator_secret_from_mnemonic(
+                    seed.as_bytes(),
+                    *index as u32,
+                    key_type,
+                )
+                .map_err(|e| format!("Unable to recover validator keys: {:?}", e))?;
+
+                let keypair = keypair_from_secret(secret.as_bytes())
+                    .map_err(|e| format!("Unable build keystore: {:?}", e))?;
+
+                KeystoreBuilder::new(&keypair, password, format!("{}", path))
+                    .map_err(|e| format!("Unable build keystore: {:?}", e))?
+                    .build()
+                    .map_err(|e| format!("Unable build keystore: {:?}", e))
+            };
+
+            let voting_keystore = derive(KeyType::Voting, voting_password.as_bytes()).unwrap();
+
+            println!("Validator {}", index + 1);
+
+            ValidatorBuilder::new(validators_dir.clone())
+                .password_dir(secrets_dir.clone())
+                .store_withdrawal_keystore(false)
+                .voting_keystore(voting_keystore, voting_password.as_bytes())
+                .build()
+                .map_err(|e| format!("Unable to build validator: {:?}", e))
+                .unwrap()
+        })
+        .collect();
+
+    Ok(())
+}
+
+pub fn run(matches: &ArgMatches) -> Result<(), String> {
+    let validator_count: usize = clap_utils::parse_required(matches, "count")?;
+    let base_dir: PathBuf = clap_utils::parse_required(matches, "base-dir")?;
+    let node_count: Option<usize> = clap_utils::parse_optional(matches, "node-count")?;
+    let mnemonic_phrase: String = clap_utils::parse_required(matches, "mnemonic-phrase")?;
+    if let Some(node_count) = node_count {
+        let validators_per_node = validator_count / node_count;
+        let validator_range = (0..validator_count).collect::<Vec<_>>();
+        let indices_range = validator_range
+            .chunks(validators_per_node)
+            .collect::<Vec<_>>();
+
+        for (i, indices) in indices_range.iter().enumerate() {
+            let validators_dir = base_dir.join(format!("node_{}", i + 1)).join("validators");
+            let secrets_dir = base_dir.join(format!("node_{}", i + 1)).join("secrets");
+            generate_validator_dirs(indices, &mnemonic_phrase, validators_dir, secrets_dir)?;
+        }
+    } else {
+        let validators_dir = base_dir.join("validators");
+        let secrets_dir = base_dir.join("secrets");
+        generate_validator_dirs(
+            (0..validator_count).collect::<Vec<_>>().as_slice(),
+            &mnemonic_phrase,
+            validators_dir,
+            secrets_dir,
+        )?;
+    }
+    Ok(())
+}