feat: Prototype native merkle trees

barretenberg/cpp/scripts/stdlib-tests

@@ -3,7 +3,6 @@ stdlib_aes128_tests
 stdlib_blake2s_tests
 stdlib_blake3s_tests
 stdlib_ecdsa_tests
-stdlib_merkle_tree_tests
 stdlib_pedersen_commitment_tests
 stdlib_pedersen_hash_tests
 stdlib_poseidon2_tests

barretenberg/cpp/src/CMakeLists.txt

@@ -127,7 +127,7 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:stdlib_blake2s_objects>
     $<TARGET_OBJECTS:stdlib_blake3s_objects>
     $<TARGET_OBJECTS:stdlib_keccak_objects>
-    $<TARGET_OBJECTS:stdlib_merkle_tree_objects>
+    $<TARGET_OBJECTS:crypto_merkle_tree_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
     $<TARGET_OBJECTS:stdlib_poseidon2_objects>

barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt

@@ -7,5 +7,9 @@ add_subdirectory(pippenger_bench)
 add_subdirectory(plonk_bench)
 add_subdirectory(protogalaxy_bench)
 add_subdirectory(relations_bench)
+add_subdirectory(widgets_bench)
+add_subdirectory(poseidon2_bench)
+add_subdirectory(merkle_tree_bench)
+add_subdirectory(indexed_tree_bench)
+add_subdirectory(append_only_tree_bench)
 add_subdirectory(ultra_bench)
-add_subdirectory(widgets_bench)

barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp

@@ -0,0 +1,54 @@
+#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/numeric/random/engine.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace benchmark;
+using namespace bb::crypto::merkle_tree;
+
+using Pedersen = AppendOnlyTree<ArrayStore, PedersenHashPolicy>;
+using Poseidon2 = AppendOnlyTree<ArrayStore, Poseidon2HashPolicy>;
+
+const size_t TREE_DEPTH = 32;
+const size_t MAX_BATCH_SIZE = 128;
+
+namespace {
+auto& random_engine = bb::numeric::get_randomness();
+} // namespace
+
+template <typename TreeType> void perform_batch_insert(TreeType& tree, const std::vector<fr>& values)
+{
+    tree.add_values(values);
+}
+
+template <typename TreeType> void append_only_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values);
+    }
+}
+BENCHMARK(append_only_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+BENCHMARK(append_only_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives)
+barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp

@@ -0,0 +1,87 @@
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp"
+#include "barretenberg/numeric/random/engine.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace benchmark;
+using namespace bb::crypto::merkle_tree;
+
+using Poseidon2 = IndexedTree<ArrayStore, LeavesCache, Poseidon2HashPolicy>;
+using Pedersen = IndexedTree<ArrayStore, LeavesCache, PedersenHashPolicy>;
+
+const size_t TREE_DEPTH = 32;
+const size_t MAX_BATCH_SIZE = 128;
+
+namespace {
+auto& random_engine = bb::numeric::get_randomness();
+} // namespace
+
+template <typename TreeType>
+void perform_batch_insert(TreeType& tree, const std::vector<fr>& values, bool single_threaded)
+{
+    tree.add_or_update_values(values, single_threaded);
+}
+
+template <typename TreeType> void multi_thread_indexed_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth, batch_size);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values, false);
+    }
+}
+
+template <typename TreeType> void single_thread_indexed_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth, batch_size);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values, true);
+    }
+}
+BENCHMARK(single_thread_indexed_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+
+BENCHMARK(multi_thread_indexed_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+
+BENCHMARK(single_thread_indexed_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives)
+barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)

barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp

@@ -1,15 +1,17 @@
-#include "merkle_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_store.hpp"
 #include "barretenberg/numeric/random/engine.hpp"
-#include "hash.hpp"
-#include "memory_store.hpp"
 #include <benchmark/benchmark.h>
 
 using namespace benchmark;
-using namespace bb::stdlib::merkle_tree;
+using namespace bb::crypto::merkle_tree;
+
+using TreeType = MerkleTree<MemoryStore, PedersenHashPolicy>;
 
 namespace {
 auto& engine = bb::numeric::get_debug_randomness();
-}
+} // namespace
 
 constexpr size_t DEPTH = 256;
 constexpr size_t MAX = 4096;
@@ -33,7 +35,7 @@ BENCHMARK(hash)->MinTime(5);
 void update_first_element(State& state) noexcept
 {
     MemoryStore store;
-    MerkleTree<MemoryStore> db(store, DEPTH);
+    TreeType db(store, DEPTH);
 
     for (auto _ : state) {
         db.update_element(0, VALUES[1]);
@@ -46,7 +48,7 @@ void update_elements(State& state) noexcept
     for (auto _ : state) {
         state.PauseTiming();
         MemoryStore store;
-        MerkleTree<MemoryStore> db(store, DEPTH);
+        TreeType db(store, DEPTH);
         state.ResumeTiming();
         for (size_t i = 0; i < (size_t)state.range(0); ++i) {
             db.update_element(i, VALUES[i]);
@@ -60,10 +62,10 @@ void update_random_elements(State& state) noexcept
     for (auto _ : state) {
         state.PauseTiming();
         MemoryStore store;
-        MerkleTree db(store, DEPTH);
+        TreeType db(store, DEPTH);
         for (size_t i = 0; i < (size_t)state.range(0); i++) {
             state.PauseTiming();
-            auto index = MerkleTree<MemoryStore>::index_t(engine.get_random_uint256());
+            auto index = TreeType::index_t(engine.get_random_uint256());
             state.ResumeTiming();
             db.update_element(index, VALUES[i]);
         }

barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(poseidon2_bench crypto_poseidon2)

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp

@@ -1,4 +1,4 @@
-#include "./poseidon2.hpp"
+#include "barretenberg/crypto/poseidon2/poseidon2.hpp"
 #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 #include <benchmark/benchmark.h>
 
@@ -24,4 +24,20 @@ void native_poseidon2_commitment_bench(State& state) noexcept
 }
 BENCHMARK(native_poseidon2_commitment_bench)->Arg(10)->Arg(1000)->Arg(10000);
 
+grumpkin::fq poseiden_hash_impl(const grumpkin::fq& x, const grumpkin::fq& y)
+{
+    std::vector<grumpkin::fq> to_hash{ x, y };
+    return bb::crypto::Poseidon2<bb::crypto::Poseidon2Bn254ScalarFieldParams>::hash(to_hash);
+}
+
+void poseiden_hash_bench(State& state) noexcept
+{
+    grumpkin::fq x = grumpkin::fq::random_element();
+    grumpkin::fq y = grumpkin::fq::random_element();
+    for (auto _ : state) {
+        DoNotOptimize(poseiden_hash_impl(x, y));
+    }
+}
+BENCHMARK(poseiden_hash_bench)->Unit(benchmark::kMillisecond);
+
 BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt

@@ -2,5 +2,5 @@ barretenberg_module(ultra_bench
   ultra_honk
   stdlib_sha256
   stdlib_keccak
-  stdlib_merkle_tree
+  crypto_merkle_tree
   stdlib_recursion)

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp

@@ -2,17 +2,17 @@
 #include <benchmark/benchmark.h>
 #include <cstddef>
 
+#include "barretenberg/crypto/merkle_tree/membership.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_store.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/plonk/composer/standard_composer.hpp"
 #include "barretenberg/plonk/composer/ultra_composer.hpp"
 #include "barretenberg/proof_system/types/circuit_type.hpp"
 #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp"
 #include "barretenberg/stdlib/hash/keccak/keccak.hpp"
 #include "barretenberg/stdlib/hash/sha256/sha256.hpp"
-#include "barretenberg/stdlib/merkle_tree/membership.hpp"
-#include "barretenberg/stdlib/merkle_tree/memory_store.hpp"
-#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp"
-#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp"
 #include "barretenberg/stdlib/primitives/bool/bool.hpp"
 #include "barretenberg/stdlib/primitives/curves/secp256k1.hpp"
 #include "barretenberg/stdlib/primitives/field/field.hpp"

barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt

@@ -9,4 +9,5 @@ add_subdirectory(schnorr)
 add_subdirectory(sha256)
 add_subdirectory(ecdsa)
 add_subdirectory(aes128)
-add_subdirectory(poseidon2)
+add_subdirectory(poseidon2)
+add_subdirectory(merkle_tree)

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(crypto_merkle_tree stdlib_primitives stdlib_blake3s stdlib_pedersen_hash)