Skip to content

Commit 1aa0ebd

Browse files
fcarreirofcarreiro
committed
feat(avm): merkle db hints (part 3)
1 parent d1df374 commit 1aa0ebd

File tree

15 files changed

+366
-23
lines changed

15 files changed

+366
-23
lines changed

‎barretenberg/cpp/src/barretenberg/common/utils.hpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,19 @@ template <typename... Ts> struct HashableTuple : public std::tuple<Ts...> {
3535

3636
} // namespace bb::utils
3737

38-
// Define hash function so that they can be used as keys in maps.
39-
// See https://en.cppreference.com/w/cpp/utility/hash.
40-
template <typename... Ts> struct std::hash<bb::utils::HashableTuple<Ts...>> {
41-
std::size_t operator()(const bb::utils::HashableTuple<Ts...>& st) const noexcept { return st.hash(); }
42-
};
43-
4438
// Needed for HashableTuple to work as a tuple.
4539
template <typename... Ts> struct std::tuple_size<bb::utils::HashableTuple<Ts...>> {
4640
static constexpr size_t value = sizeof...(Ts);
4741
};
42+
43+
// Define std::hash for any type that has a hash() method. This includes HashableTuple.
44+
template <typename T>
45+
concept Hashable = requires(const T& t) {
46+
{
47+
t.hash()
48+
} -> std::same_as<std::size_t>;
49+
};
50+
51+
template <Hashable T> struct std::hash<T> {
52+
std::size_t operator()(const T& t) const noexcept { return t.hash(); }
53+
};

‎barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include "barretenberg/common/utils.hpp"
34
#include "barretenberg/crypto/merkle_tree/types.hpp"
45
#include "barretenberg/ecc/curves/bn254/fr.hpp"
56
#include "barretenberg/serialize/msgpack.hpp"
@@ -61,6 +62,8 @@ struct NullifierLeafValue {
6162
static NullifierLeafValue padding(index_t i) { return { i }; }
6263

6364
static std::string name() { return "NullifierLeafValue"; };
65+
66+
size_t hash() const noexcept { return std::hash<fr>{}(nullifier); }
6467
};
6568

6669
struct PublicDataLeafValue {
@@ -121,6 +124,8 @@ struct PublicDataLeafValue {
121124
static PublicDataLeafValue padding(index_t i) { return { i, fr::zero() }; }
122125

123126
static std::string name() { return "PublicDataLeafValue"; };
127+
128+
size_t hash() const noexcept { return utils::hash_as_tuple(value, slot); }
124129
};
125130

126131
template <typename LeafType> struct IndexedLeaf {

‎barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,6 @@
2020
#include <utility>
2121
#include <vector>
2222

23-
template <> struct std::hash<uint256_t> {
24-
std::size_t operator()(const uint256_t& k) const { return k.data[0]; }
25-
};
26-
template <> struct std::hash<bb::fr> {
27-
std::size_t operator()(const bb::fr& k) const
28-
{
29-
bb::numeric::uint256_t val(k);
30-
return val.data[0];
31-
}
32-
};
33-
3423
namespace bb::crypto::merkle_tree {
3524

3625
// Stores all of the penidng updates to a mekle tree indexed for optimal retrieval
@@ -457,4 +446,4 @@ void ContentAddressedCache<LeafValueType>::put_node_by_index(uint32_t level, con
457446
}
458447
nodes_by_index_[level][index] = node;
459448
}
460-
} // namespace bb::crypto::merkle_tree
449+
} // namespace bb::crypto::merkle_tree

‎barretenberg/cpp/src/barretenberg/numeric/uint256/uint256.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "../uint128/uint128.hpp"
1515
#include "barretenberg/common/serialize.hpp"
1616
#include "barretenberg/common/throw_or_abort.hpp"
17+
#include "barretenberg/common/utils.hpp"
1718
#include <concepts>
1819
#include <cstdint>
1920
#include <iomanip>
@@ -202,6 +203,8 @@ class alignas(32) uint256_t {
202203

203204
[[nodiscard]] constexpr std::pair<uint256_t, uint256_t> divmod(const uint256_t& b) const;
204205

206+
size_t hash() const noexcept { return utils::hash_as_tuple(data[0], data[1], data[2], data[3]); }
207+
205208
private:
206209
[[nodiscard]] static constexpr std::pair<uint64_t, uint64_t> mul_wide(uint64_t a, uint64_t b);
207210
[[nodiscard]] static constexpr std::pair<uint64_t, uint64_t> addc(uint64_t a, uint64_t b, uint64_t carry_in);

‎barretenberg/cpp/src/barretenberg/vm2/common/avm_inputs.hpp‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77

88
#include "barretenberg/common/utils.hpp"
99
#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
10+
#include "barretenberg/crypto/merkle_tree/response.hpp"
1011
#include "barretenberg/serialize/msgpack.hpp"
12+
#include "barretenberg/world_state/world_state.hpp"
13+
1114
#include "barretenberg/vm2/common/aztec_types.hpp"
1215
#include "barretenberg/vm2/common/field.hpp"
1316
#include "barretenberg/world_state/types.hpp"
@@ -158,6 +161,22 @@ struct GetLeafValueHint {
158161
MSGPACK_FIELDS(hintKey, treeId, index, value);
159162
};
160163

164+
template <typename Leaf> struct SequentialInsertHint {
165+
AppendOnlyTreeSnapshot hintKey;
166+
// params
167+
world_state::MerkleTreeId treeId;
168+
Leaf leaf;
169+
// return
170+
crypto::merkle_tree::LeafUpdateWitnessData<Leaf> lowLeavesWitnessData;
171+
crypto::merkle_tree::LeafUpdateWitnessData<Leaf> insertionWitnessData;
172+
// evolved state
173+
AppendOnlyTreeSnapshot stateAfter;
174+
175+
bool operator==(const SequentialInsertHint<Leaf>& other) const = default;
176+
177+
MSGPACK_FIELDS(hintKey, treeId, leaf, lowLeavesWitnessData, insertionWitnessData, stateAfter);
178+
};
179+
161180
////////////////////////////////////////////////////////////////////////////
162181
// Hints (other)
163182
////////////////////////////////////////////////////////////////////////////
@@ -189,6 +208,8 @@ struct ExecutionHints {
189208
std::vector<GetLeafPreimageHint<crypto::merkle_tree::IndexedLeaf<crypto::merkle_tree::NullifierLeafValue>>>
190209
getLeafPreimageHintsNullifierTree;
191210
std::vector<GetLeafValueHint> getLeafValueHints;
211+
std::vector<SequentialInsertHint<crypto::merkle_tree::PublicDataLeafValue>> sequentialInsertHintsPublicDataTree;
212+
std::vector<SequentialInsertHint<crypto::merkle_tree::NullifierLeafValue>> sequentialInsertHintsNullifierTree;
192213

193214
bool operator==(const ExecutionHints& other) const = default;
194215

@@ -200,7 +221,9 @@ struct ExecutionHints {
200221
getPreviousValueIndexHints,
201222
getLeafPreimageHintsPublicDataTree,
202223
getLeafPreimageHintsNullifierTree,
203-
getLeafValueHints);
224+
getLeafValueHints,
225+
sequentialInsertHintsPublicDataTree,
226+
sequentialInsertHintsNullifierTree);
204227
};
205228

206229
////////////////////////////////////////////////////////////////////////////
@@ -218,9 +241,6 @@ struct AvmProvingInputs {
218241

219242
} // namespace bb::avm2
220243

221-
// This has to be done outside of the namespace.
222-
MSGPACK_ADD_ENUM(bb::world_state::MerkleTreeId);
223-
224244
// Define hash function so that they can be used as keys in maps.
225245
// See https://en.cppreference.com/w/cpp/utility/hash.
226246
template <> struct std::hash<bb::avm2::AppendOnlyTreeSnapshot> {

‎barretenberg/cpp/src/barretenberg/vm2/common/avm_inputs.testdata.bin‎

126 KB
Binary file not shown.

‎barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_interfaces.hpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "barretenberg/vm2/common/avm_inputs.hpp"
1010
#include "barretenberg/vm2/common/aztec_types.hpp"
1111
#include "barretenberg/world_state/types.hpp"
12+
#include "barretenberg/world_state/world_state.hpp"
1213

1314
namespace bb::avm2::simulation {
1415

@@ -38,6 +39,16 @@ class LowLevelMerkleDBInterface {
3839
get_leaf_preimage_public_data_tree(crypto::merkle_tree::index_t leaf_index) const = 0;
3940
virtual crypto::merkle_tree::IndexedLeaf<crypto::merkle_tree::NullifierLeafValue> get_leaf_preimage_nullifier_tree(
4041
crypto::merkle_tree::index_t leaf_index) const = 0;
42+
43+
// Inserts a leaf into the public data tree sequentially, getting witnesses at every step.
44+
// Note: This method doesn't support inserting empty leaves.
45+
virtual world_state::SequentialInsertionResult<crypto::merkle_tree::PublicDataLeafValue>
46+
insert_indexed_leaves_public_data_tree(const crypto::merkle_tree::PublicDataLeafValue& leaf_value) = 0;
47+
48+
// Inserts a leaf into the nullifier tree sequentially, getting witnesses at every step.
49+
// Note: This method doesn't support inserting empty leaves.
50+
virtual world_state::SequentialInsertionResult<crypto::merkle_tree::NullifierLeafValue>
51+
insert_indexed_leaves_nullifier_tree(const crypto::merkle_tree::NullifierLeafValue& leaf_value) = 0;
4152
};
4253

4354
// High level access to a merkle db. In general these will be constrained.

‎barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,11 @@ HintedRawMerkleDB::HintedRawMerkleDB(const ExecutionHints& hints, const TreeSnap
106106
"\n * get_leaf_preimage hints_nullifier_tree: ",
107107
hints.getLeafPreimageHintsNullifierTree.size(),
108108
"\n * get_leaf_value_hints: ",
109-
hints.getLeafValueHints.size());
109+
hints.getLeafValueHints.size(),
110+
"\n * sequential_insert_hints_public_data_tree: ",
111+
hints.sequentialInsertHintsPublicDataTree.size(),
112+
"\n * sequential_insert_hints_nullifier_tree: ",
113+
hints.sequentialInsertHintsNullifierTree.size());
110114
debug("Initializing HintedRawMerkleDB with snapshots...",
111115
"\n * nullifierTree: ",
112116
tree_roots.nullifierTree.root,
@@ -160,6 +164,20 @@ HintedRawMerkleDB::HintedRawMerkleDB(const ExecutionHints& hints, const TreeSnap
160164
GetLeafValueKey key = { get_leaf_value_hint.hintKey, get_leaf_value_hint.treeId, get_leaf_value_hint.index };
161165
get_leaf_value_hints[key] = get_leaf_value_hint.value;
162166
}
167+
168+
for (const auto& sequential_insert_hint : hints.sequentialInsertHintsPublicDataTree) {
169+
SequentialInsertHintPublicDataTreeKey key = { sequential_insert_hint.hintKey,
170+
sequential_insert_hint.treeId,
171+
sequential_insert_hint.leaf };
172+
sequential_insert_hints_public_data_tree[key] = sequential_insert_hint;
173+
}
174+
175+
for (const auto& sequential_insert_hint : hints.sequentialInsertHintsNullifierTree) {
176+
SequentialInsertHintNullifierTreeKey key = { sequential_insert_hint.hintKey,
177+
sequential_insert_hint.treeId,
178+
sequential_insert_hint.leaf };
179+
sequential_insert_hints_nullifier_tree[key] = sequential_insert_hint;
180+
}
163181
}
164182

165183
const AppendOnlyTreeSnapshot& HintedRawMerkleDB::get_tree_info(world_state::MerkleTreeId tree_id) const
@@ -262,4 +280,84 @@ crypto::merkle_tree::IndexedLeaf<crypto::merkle_tree::NullifierLeafValue> Hinted
262280
return it->second;
263281
}
264282

283+
world_state::SequentialInsertionResult<crypto::merkle_tree::PublicDataLeafValue> HintedRawMerkleDB::
284+
insert_indexed_leaves_public_data_tree(const crypto::merkle_tree::PublicDataLeafValue& leaf_value)
285+
{
286+
auto tree_info = get_tree_info(world_state::MerkleTreeId::PUBLIC_DATA_TREE);
287+
SequentialInsertHintPublicDataTreeKey key = { tree_info, world_state::MerkleTreeId::PUBLIC_DATA_TREE, leaf_value };
288+
auto it = sequential_insert_hints_public_data_tree.find(key);
289+
if (it == sequential_insert_hints_public_data_tree.end()) {
290+
throw std::runtime_error(format("Sequential insert hint (PUBLIC_DATA_TREE) not found for key (root: ",
291+
tree_info.root,
292+
", size: ",
293+
tree_info.nextAvailableLeafIndex,
294+
", leaf_value: ",
295+
leaf_value,
296+
")"));
297+
}
298+
const auto& hint = it->second;
299+
300+
world_state::SequentialInsertionResult<crypto::merkle_tree::PublicDataLeafValue> result;
301+
302+
// Convert low leaves witness data
303+
result.low_leaf_witness_data.emplace_back(
304+
hint.lowLeavesWitnessData.leaf, hint.lowLeavesWitnessData.index, hint.lowLeavesWitnessData.path);
305+
306+
// Convert insertion witness data
307+
result.insertion_witness_data.emplace_back(
308+
hint.insertionWitnessData.leaf, hint.insertionWitnessData.index, hint.insertionWitnessData.path);
309+
310+
// Evolve state.
311+
tree_roots.publicDataTree.root = hint.stateAfter.root;
312+
tree_roots.publicDataTree.nextAvailableLeafIndex = hint.stateAfter.nextAvailableLeafIndex;
313+
314+
debug("Evolved state of PUBLIC_DATA_TREE: ",
315+
tree_roots.publicDataTree.root,
316+
" (size: ",
317+
tree_roots.publicDataTree.nextAvailableLeafIndex,
318+
")");
319+
320+
return result;
321+
}
322+
323+
world_state::SequentialInsertionResult<crypto::merkle_tree::NullifierLeafValue> HintedRawMerkleDB::
324+
insert_indexed_leaves_nullifier_tree(const crypto::merkle_tree::NullifierLeafValue& leaf_value)
325+
{
326+
auto tree_info = get_tree_info(world_state::MerkleTreeId::NULLIFIER_TREE);
327+
SequentialInsertHintNullifierTreeKey key = { tree_info, world_state::MerkleTreeId::NULLIFIER_TREE, leaf_value };
328+
auto it = sequential_insert_hints_nullifier_tree.find(key);
329+
if (it == sequential_insert_hints_nullifier_tree.end()) {
330+
throw std::runtime_error(format("Sequential insert hint (NULLIFIER_TREE) not found for key (root: ",
331+
tree_info.root,
332+
", size: ",
333+
tree_info.nextAvailableLeafIndex,
334+
", leaf_value: ",
335+
leaf_value,
336+
")"));
337+
}
338+
const auto& hint = it->second;
339+
340+
world_state::SequentialInsertionResult<crypto::merkle_tree::NullifierLeafValue> result;
341+
342+
// Convert low leaves witness data
343+
result.low_leaf_witness_data.emplace_back(
344+
hint.lowLeavesWitnessData.leaf, hint.lowLeavesWitnessData.index, hint.lowLeavesWitnessData.path);
345+
346+
// Convert insertion witness data
347+
result.insertion_witness_data.emplace_back(
348+
hint.insertionWitnessData.leaf, hint.insertionWitnessData.index, hint.insertionWitnessData.path);
349+
350+
// Evolve state.
351+
tree_roots.nullifierTree.root = hint.stateAfter.root;
352+
tree_roots.nullifierTree.nextAvailableLeafIndex = hint.stateAfter.nextAvailableLeafIndex;
353+
354+
debug("Evolved state of NULLIFIER_TREE: ",
355+
tree_roots.nullifierTree.root,
356+
" (size: ",
357+
tree_roots.nullifierTree.nextAvailableLeafIndex,
358+
")");
359+
360+
return result;
361+
}
362+
265363
} // namespace bb::avm2::simulation

‎barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@
33
#include "barretenberg/common/utils.hpp"
44
#include "barretenberg/crypto/merkle_tree/hash_path.hpp"
55
#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
6+
#include "barretenberg/crypto/merkle_tree/response.hpp"
67
#include "barretenberg/vm2/common/avm_inputs.hpp"
78
#include "barretenberg/vm2/common/aztec_types.hpp"
89
#include "barretenberg/vm2/common/field.hpp"
910
#include "barretenberg/vm2/common/map.hpp"
1011
#include "barretenberg/vm2/simulation/lib/db_interfaces.hpp"
1112
#include "barretenberg/world_state/types.hpp"
13+
#include "barretenberg/world_state/world_state.hpp"
1214

1315
namespace bb::avm2::simulation {
1416

@@ -37,6 +39,7 @@ class HintedRawMerkleDB final : public LowLevelMerkleDBInterface {
3739

3840
const TreeSnapshots& get_tree_roots() const override { return tree_roots; }
3941

42+
// Query methods.
4043
crypto::merkle_tree::fr_sibling_path get_sibling_path(world_state::MerkleTreeId tree_id,
4144
crypto::merkle_tree::index_t leaf_index) const override;
4245
crypto::merkle_tree::GetLowIndexedLeafResponse get_low_indexed_leaf(world_state::MerkleTreeId tree_id,
@@ -47,9 +50,16 @@ class HintedRawMerkleDB final : public LowLevelMerkleDBInterface {
4750
crypto::merkle_tree::IndexedLeaf<crypto::merkle_tree::NullifierLeafValue> get_leaf_preimage_nullifier_tree(
4851
crypto::merkle_tree::index_t leaf_index) const override;
4952

53+
// State modification methods.
54+
world_state::SequentialInsertionResult<crypto::merkle_tree::PublicDataLeafValue>
55+
insert_indexed_leaves_public_data_tree(const crypto::merkle_tree::PublicDataLeafValue& leaf_value) override;
56+
world_state::SequentialInsertionResult<crypto::merkle_tree::NullifierLeafValue>
57+
insert_indexed_leaves_nullifier_tree(const crypto::merkle_tree::NullifierLeafValue& leaf_value) override;
58+
5059
private:
5160
TreeSnapshots tree_roots;
5261

62+
// Query hints.
5363
using GetSiblingPathKey =
5464
utils::HashableTuple<AppendOnlyTreeSnapshot, world_state::MerkleTreeId, crypto::merkle_tree::index_t>;
5565
unordered_flat_map<GetSiblingPathKey, crypto::merkle_tree::fr_sibling_path> get_sibling_path_hints;
@@ -64,6 +74,17 @@ class HintedRawMerkleDB final : public LowLevelMerkleDBInterface {
6474
using GetLeafValueKey =
6575
utils::HashableTuple<AppendOnlyTreeSnapshot, world_state::MerkleTreeId, crypto::merkle_tree::index_t>;
6676
unordered_flat_map<GetLeafValueKey, FF> get_leaf_value_hints;
77+
// State modification hints.
78+
using SequentialInsertHintPublicDataTreeKey = utils::
79+
HashableTuple<AppendOnlyTreeSnapshot, world_state::MerkleTreeId, crypto::merkle_tree::PublicDataLeafValue>;
80+
unordered_flat_map<SequentialInsertHintPublicDataTreeKey,
81+
SequentialInsertHint<crypto::merkle_tree::PublicDataLeafValue>>
82+
sequential_insert_hints_public_data_tree;
83+
using SequentialInsertHintNullifierTreeKey = utils::
84+
HashableTuple<AppendOnlyTreeSnapshot, world_state::MerkleTreeId, crypto::merkle_tree::NullifierLeafValue>;
85+
unordered_flat_map<SequentialInsertHintNullifierTreeKey,
86+
SequentialInsertHint<crypto::merkle_tree::NullifierLeafValue>>
87+
sequential_insert_hints_nullifier_tree;
6788

6889
const AppendOnlyTreeSnapshot& get_tree_info(world_state::MerkleTreeId tree_id) const;
6990
};

‎barretenberg/cpp/src/barretenberg/vm2/simulation/testing/mock_dbs.hpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ class MockLowLevelMerkleDB : public LowLevelMerkleDBInterface {
4444
get_leaf_preimage_nullifier_tree,
4545
(crypto::merkle_tree::index_t leaf_index),
4646
(const, override));
47+
MOCK_METHOD(world_state::SequentialInsertionResult<crypto::merkle_tree::PublicDataLeafValue>,
48+
insert_indexed_leaves_public_data_tree,
49+
(const crypto::merkle_tree::PublicDataLeafValue& leaf_value),
50+
(override));
51+
MOCK_METHOD(world_state::SequentialInsertionResult<crypto::merkle_tree::NullifierLeafValue>,
52+
insert_indexed_leaves_nullifier_tree,
53+
(const crypto::merkle_tree::NullifierLeafValue& leaf_value),
54+
(override));
4755
};
4856

4957
class MockHighLevelMerkleDB : public HighLevelMerkleDBInterface {

0 commit comments

Comments
 (0)