1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
use super::{
    mmr::Mmr, BTreeMap, EmptySubtreeRoots, InnerNodeInfo, KvMap, MerkleError, MerklePath,
    MerklePathSet, MerkleTree, NodeIndex, RecordingMap, RootPath, Rpo256, RpoDigest, SimpleSmt,
    TieredSmt, ValuePath, Vec,
};
use crate::utils::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable};
use core::borrow::Borrow;

#[cfg(test)]
mod tests;

// MERKLE STORE
// ================================================================================================

/// A default [MerkleStore] which uses a simple [BTreeMap] as the backing storage.
pub type DefaultMerkleStore = MerkleStore<BTreeMap<RpoDigest, StoreNode>>;

/// A [MerkleStore] with recording capabilities which uses [RecordingMap] as the backing storage.
pub type RecordingMerkleStore = MerkleStore<RecordingMap<RpoDigest, StoreNode>>;

#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
pub struct StoreNode {
    left: RpoDigest,
    right: RpoDigest,
}

/// An in-memory data store for Merkelized data.
///
/// This is a in memory data store for Merkle trees, this store allows all the nodes of multiple
/// trees to live as long as necessary and without duplication, this allows the implementation of
/// space efficient persistent data structures.
///
/// Example usage:
///
/// ```rust
/// # use miden_crypto::{ZERO, Felt, Word};
/// # use miden_crypto::merkle::{NodeIndex, MerkleStore, MerkleTree};
/// # use miden_crypto::hash::rpo::Rpo256;
/// # const fn int_to_node(value: u64) -> Word {
/// #     [Felt::new(value), ZERO, ZERO, ZERO]
/// # }
/// # let A = int_to_node(1);
/// # let B = int_to_node(2);
/// # let C = int_to_node(3);
/// # let D = int_to_node(4);
/// # let E = int_to_node(5);
/// # let F = int_to_node(6);
/// # let G = int_to_node(7);
/// # let H0 = int_to_node(8);
/// # let H1 = int_to_node(9);
/// # let T0 = MerkleTree::new([A, B, C, D, E, F, G, H0].to_vec()).expect("even number of leaves provided");
/// # let T1 = MerkleTree::new([A, B, C, D, E, F, G, H1].to_vec()).expect("even number of leaves provided");
/// # let ROOT0 = T0.root();
/// # let ROOT1 = T1.root();
/// let mut store: MerkleStore = MerkleStore::new();
///
/// // the store is initialized with the SMT empty nodes
/// assert_eq!(store.num_internal_nodes(), 255);
///
/// let tree1 = MerkleTree::new(vec![A, B, C, D, E, F, G, H0]).unwrap();
/// let tree2 = MerkleTree::new(vec![A, B, C, D, E, F, G, H1]).unwrap();
///
/// // populates the store with two merkle trees, common nodes are shared
/// store.extend(tree1.inner_nodes());
/// store.extend(tree2.inner_nodes());
///
/// // every leaf except the last are the same
/// for i in 0..7 {
///     let idx0 = NodeIndex::new(3, i).unwrap();
///     let d0 = store.get_node(ROOT0, idx0).unwrap();
///     let idx1 = NodeIndex::new(3, i).unwrap();
///     let d1 = store.get_node(ROOT1, idx1).unwrap();
///     assert_eq!(d0, d1, "Both trees have the same leaf at pos {i}");
/// }
///
/// // The leafs A-B-C-D are the same for both trees, so are their 2 immediate parents
/// for i in 0..4 {
///     let idx0 = NodeIndex::new(3, i).unwrap();
///     let d0 = store.get_path(ROOT0, idx0).unwrap();
///     let idx1 = NodeIndex::new(3, i).unwrap();
///     let d1 = store.get_path(ROOT1, idx1).unwrap();
///     assert_eq!(d0.path[0..2], d1.path[0..2], "Both sub-trees are equal up to two levels");
/// }
///
/// // Common internal nodes are shared, the two added trees have a total of 30, but the store has
/// // only 10 new entries, corresponding to the 10 unique internal nodes of these trees.
/// assert_eq!(store.num_internal_nodes() - 255, 10);
/// ```
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct MerkleStore<T: KvMap<RpoDigest, StoreNode> = BTreeMap<RpoDigest, StoreNode>> {
    nodes: T,
}

impl<T: KvMap<RpoDigest, StoreNode>> Default for MerkleStore<T> {
    fn default() -> Self {
        Self::new()
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> MerkleStore<T> {
    // CONSTRUCTORS
    // --------------------------------------------------------------------------------------------

    /// Creates an empty `MerkleStore` instance.
    pub fn new() -> MerkleStore<T> {
        // pre-populate the store with the empty hashes
        let nodes = empty_hashes().into_iter().collect();
        MerkleStore { nodes }
    }

    // PUBLIC ACCESSORS
    // --------------------------------------------------------------------------------------------

    /// Return a count of the non-leaf nodes in the store.
    pub fn num_internal_nodes(&self) -> usize {
        self.nodes.len()
    }

    /// Returns the node at `index` rooted on the tree `root`.
    ///
    /// # Errors
    /// This method can return the following errors:
    /// - `RootNotInStore` if the `root` is not present in the store.
    /// - `NodeNotInStore` if a node needed to traverse from `root` to `index` is not present in
    ///   the store.
    pub fn get_node(&self, root: RpoDigest, index: NodeIndex) -> Result<RpoDigest, MerkleError> {
        let mut hash = root;

        // corner case: check the root is in the store when called with index `NodeIndex::root()`
        self.nodes.get(&hash).ok_or(MerkleError::RootNotInStore(hash))?;

        for i in (0..index.depth()).rev() {
            let node = self.nodes.get(&hash).ok_or(MerkleError::NodeNotInStore(hash, index))?;

            let bit = (index.value() >> i) & 1;
            hash = if bit == 0 { node.left } else { node.right }
        }

        Ok(hash)
    }

    /// Returns the node at the specified `index` and its opening to the `root`.
    ///
    /// The path starts at the sibling of the target leaf.
    ///
    /// # Errors
    /// This method can return the following errors:
    /// - `RootNotInStore` if the `root` is not present in the store.
    /// - `NodeNotInStore` if a node needed to traverse from `root` to `index` is not present in
    ///   the store.
    pub fn get_path(&self, root: RpoDigest, index: NodeIndex) -> Result<ValuePath, MerkleError> {
        let mut hash = root;
        let mut path = Vec::with_capacity(index.depth().into());

        // corner case: check the root is in the store when called with index `NodeIndex::root()`
        self.nodes.get(&hash).ok_or(MerkleError::RootNotInStore(hash))?;

        for i in (0..index.depth()).rev() {
            let node = self.nodes.get(&hash).ok_or(MerkleError::NodeNotInStore(hash, index))?;

            let bit = (index.value() >> i) & 1;
            hash = if bit == 0 {
                path.push(node.right);
                node.left
            } else {
                path.push(node.left);
                node.right
            }
        }

        // the path is computed from root to leaf, so it must be reversed
        path.reverse();

        Ok(ValuePath {
            value: hash,
            path: MerklePath::new(path),
        })
    }

    /// Reconstructs a path from the root until a leaf or empty node and returns its depth.
    ///
    /// The `tree_depth` parameter defines up to which depth the tree will be traversed, starting
    /// from `root`. The maximum value the argument accepts is [u64::BITS].
    ///
    /// The traversed path from leaf to root will start at the least significant bit of `index`,
    /// and will be executed for `tree_depth` bits.
    ///
    /// # Errors
    /// Will return an error if:
    /// - The provided root is not found.
    /// - The path from the root continues to a depth greater than `tree_depth`.
    /// - The provided `tree_depth` is greater than `64.
    /// - The provided `index` is not valid for a depth equivalent to `tree_depth`. For more
    ///   information, check [NodeIndex::new].
    pub fn get_leaf_depth(
        &self,
        root: RpoDigest,
        tree_depth: u8,
        index: u64,
    ) -> Result<u8, MerkleError> {
        // validate depth and index
        if tree_depth > 64 {
            return Err(MerkleError::DepthTooBig(tree_depth as u64));
        }
        NodeIndex::new(tree_depth, index)?;

        // it's not illegal to have a maximum depth of `0`; we should just return the root in that
        // case. this check will simplify the implementation as we could overflow bits for depth
        // `0`.
        if tree_depth == 0 {
            return Ok(0);
        }

        // check if the root exists, providing the proper error report if it doesn't
        let empty = EmptySubtreeRoots::empty_hashes(tree_depth);
        let mut hash = root;
        if !self.nodes.contains_key(&hash) {
            return Err(MerkleError::RootNotInStore(hash));
        }

        // we traverse from root to leaf, so the path is reversed
        let mut path = (index << (64 - tree_depth)).reverse_bits();

        // iterate every depth and reconstruct the path from root to leaf
        for depth in 0..tree_depth {
            // we short-circuit if an empty node has been found
            if hash == empty[depth as usize] {
                return Ok(depth);
            }

            // fetch the children pair, mapped by its parent hash
            let children = match self.nodes.get(&hash) {
                Some(node) => node,
                None => return Ok(depth),
            };

            // traverse down
            hash = if path & 1 == 0 { children.left } else { children.right };
            path >>= 1;
        }

        // at max depth assert it doesn't have sub-trees
        if self.nodes.contains_key(&hash) {
            return Err(MerkleError::DepthTooBig(tree_depth as u64 + 1));
        }

        // depleted bits; return max depth
        Ok(tree_depth)
    }

    // DATA EXTRACTORS
    // --------------------------------------------------------------------------------------------

    /// Returns a subset of this Merkle store such that the returned Merkle store contains all
    /// nodes which are descendants of the specified roots.
    ///
    /// The roots for which no descendants exist in this Merkle store are ignored.
    pub fn subset<I, R>(&self, roots: I) -> MerkleStore<T>
    where
        I: Iterator<Item = R>,
        R: Borrow<RpoDigest>,
    {
        let mut store = MerkleStore::new();
        for root in roots {
            let root = *root.borrow();
            store.clone_tree_from(root, self);
        }
        store
    }

    /// Iterator over the inner nodes of the [MerkleStore].
    pub fn inner_nodes(&self) -> impl Iterator<Item = InnerNodeInfo> + '_ {
        self.nodes.iter().map(|(r, n)| InnerNodeInfo {
            value: *r,
            left: n.left,
            right: n.right,
        })
    }

    // STATE MUTATORS
    // --------------------------------------------------------------------------------------------

    /// Adds all the nodes of a Merkle path represented by `path`, opening to `node`. Returns the
    /// new root.
    ///
    /// This will compute the sibling elements determined by the Merkle `path` and `node`, and
    /// include all the nodes into the store.
    pub fn add_merkle_path(
        &mut self,
        index: u64,
        node: RpoDigest,
        path: MerklePath,
    ) -> Result<RpoDigest, MerkleError> {
        let root = path.inner_nodes(index, node)?.fold(RpoDigest::default(), |_, node| {
            let value: RpoDigest = node.value;
            let left: RpoDigest = node.left;
            let right: RpoDigest = node.right;

            debug_assert_eq!(Rpo256::merge(&[left, right]), value);
            self.nodes.insert(value, StoreNode { left, right });

            node.value
        });
        Ok(root)
    }

    /// Adds all the nodes of multiple Merkle paths into the store.
    ///
    /// This will compute the sibling elements for each Merkle `path` and include all the nodes
    /// into the store.
    ///
    /// For further reference, check [MerkleStore::add_merkle_path].
    pub fn add_merkle_paths<I>(&mut self, paths: I) -> Result<(), MerkleError>
    where
        I: IntoIterator<Item = (u64, RpoDigest, MerklePath)>,
    {
        for (index_value, node, path) in paths.into_iter() {
            self.add_merkle_path(index_value, node, path)?;
        }
        Ok(())
    }

    /// Appends the provided [MerklePathSet] into the store.
    ///
    /// For further reference, check [MerkleStore::add_merkle_path].
    pub fn add_merkle_path_set(
        &mut self,
        path_set: &MerklePathSet,
    ) -> Result<RpoDigest, MerkleError> {
        let root = path_set.root();
        for (index, path) in path_set.to_paths() {
            self.add_merkle_path(index, path.value, path.path)?;
        }
        Ok(root)
    }

    /// Sets a node to `value`.
    ///
    /// # Errors
    /// This method can return the following errors:
    /// - `RootNotInStore` if the `root` is not present in the store.
    /// - `NodeNotInStore` if a node needed to traverse from `root` to `index` is not present in
    ///   the store.
    pub fn set_node(
        &mut self,
        mut root: RpoDigest,
        index: NodeIndex,
        value: RpoDigest,
    ) -> Result<RootPath, MerkleError> {
        let node = value;
        let ValuePath { value, path } = self.get_path(root, index)?;

        // performs the update only if the node value differs from the opening
        if node != value {
            root = self.add_merkle_path(index.value(), node, path.clone())?;
        }

        Ok(RootPath { root, path })
    }

    /// Merges two elements and adds the resulting node into the store.
    ///
    /// Merges arbitrary values. They may be leafs, nodes, or a mixture of both.
    pub fn merge_roots(
        &mut self,
        left_root: RpoDigest,
        right_root: RpoDigest,
    ) -> Result<RpoDigest, MerkleError> {
        let parent = Rpo256::merge(&[left_root, right_root]);
        self.nodes.insert(
            parent,
            StoreNode {
                left: left_root,
                right: right_root,
            },
        );

        Ok(parent)
    }

    // DESTRUCTURING
    // --------------------------------------------------------------------------------------------

    /// Returns the inner storage of this MerkleStore while consuming `self`.
    pub fn into_inner(self) -> T {
        self.nodes
    }

    // HELPER METHODS
    // --------------------------------------------------------------------------------------------

    /// Recursively clones a tree with the specified root from the specified source into self.
    ///
    /// If the source store does not contain a tree with the specified root, this is a noop.
    fn clone_tree_from(&mut self, root: RpoDigest, source: &Self) {
        // process the node only if it is in the source
        if let Some(node) = source.nodes.get(&root) {
            // if the node has already been inserted, no need to process it further as all of its
            // descendants should be already cloned from the source store
            if self.nodes.insert(root, *node).is_none() {
                self.clone_tree_from(node.left, source);
                self.clone_tree_from(node.right, source);
            }
        }
    }
}

// CONVERSIONS
// ================================================================================================

impl<T: KvMap<RpoDigest, StoreNode>> From<&MerkleTree> for MerkleStore<T> {
    fn from(value: &MerkleTree) -> Self {
        let nodes = combine_nodes_with_empty_hashes(value.inner_nodes()).collect();
        Self { nodes }
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> From<&SimpleSmt> for MerkleStore<T> {
    fn from(value: &SimpleSmt) -> Self {
        let nodes = combine_nodes_with_empty_hashes(value.inner_nodes()).collect();
        Self { nodes }
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> From<&Mmr> for MerkleStore<T> {
    fn from(value: &Mmr) -> Self {
        let nodes = combine_nodes_with_empty_hashes(value.inner_nodes()).collect();
        Self { nodes }
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> From<&TieredSmt> for MerkleStore<T> {
    fn from(value: &TieredSmt) -> Self {
        let nodes = combine_nodes_with_empty_hashes(value.inner_nodes()).collect();
        Self { nodes }
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> From<T> for MerkleStore<T> {
    fn from(values: T) -> Self {
        let nodes = values.into_iter().chain(empty_hashes().into_iter()).collect();
        Self { nodes }
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> FromIterator<InnerNodeInfo> for MerkleStore<T> {
    fn from_iter<I: IntoIterator<Item = InnerNodeInfo>>(iter: I) -> Self {
        let nodes = combine_nodes_with_empty_hashes(iter.into_iter()).collect();
        Self { nodes }
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> FromIterator<(RpoDigest, StoreNode)> for MerkleStore<T> {
    fn from_iter<I: IntoIterator<Item = (RpoDigest, StoreNode)>>(iter: I) -> Self {
        let nodes = iter.into_iter().chain(empty_hashes().into_iter()).collect();
        Self { nodes }
    }
}

// ITERATORS
// ================================================================================================

impl<T: KvMap<RpoDigest, StoreNode>> Extend<InnerNodeInfo> for MerkleStore<T> {
    fn extend<I: IntoIterator<Item = InnerNodeInfo>>(&mut self, iter: I) {
        self.nodes.extend(iter.into_iter().map(|info| {
            (
                info.value,
                StoreNode {
                    left: info.left,
                    right: info.right,
                },
            )
        }));
    }
}

// SERIALIZATION
// ================================================================================================

impl Serializable for StoreNode {
    fn write_into<W: ByteWriter>(&self, target: &mut W) {
        self.left.write_into(target);
        self.right.write_into(target);
    }
}

impl Deserializable for StoreNode {
    fn read_from<R: ByteReader>(source: &mut R) -> Result<Self, DeserializationError> {
        let left = RpoDigest::read_from(source)?;
        let right = RpoDigest::read_from(source)?;
        Ok(StoreNode { left, right })
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> Serializable for MerkleStore<T> {
    fn write_into<W: ByteWriter>(&self, target: &mut W) {
        target.write_u64(self.nodes.len() as u64);

        for (k, v) in self.nodes.iter() {
            k.write_into(target);
            v.write_into(target);
        }
    }
}

impl<T: KvMap<RpoDigest, StoreNode>> Deserializable for MerkleStore<T> {
    fn read_from<R: ByteReader>(source: &mut R) -> Result<Self, DeserializationError> {
        let len = source.read_u64()?;
        let mut nodes: Vec<(RpoDigest, StoreNode)> = Vec::with_capacity(len as usize);

        for _ in 0..len {
            let key = RpoDigest::read_from(source)?;
            let value = StoreNode::read_from(source)?;
            nodes.push((key, value));
        }

        Ok(nodes.into_iter().collect())
    }
}

// HELPER FUNCTIONS
// ================================================================================================

/// Creates empty hashes for all the subtrees of a tree with a max depth of 255.
fn empty_hashes() -> impl IntoIterator<Item = (RpoDigest, StoreNode)> {
    let subtrees = EmptySubtreeRoots::empty_hashes(255);
    subtrees.iter().rev().copied().zip(subtrees.iter().rev().skip(1).copied()).map(
        |(child, parent)| {
            (
                parent,
                StoreNode {
                    left: child,
                    right: child,
                },
            )
        },
    )
}

/// Consumes an iterator of [InnerNodeInfo] and returns an iterator of `(value, node)` tuples
/// which includes the nodes associate with roots of empty subtrees up to a depth of 255.
fn combine_nodes_with_empty_hashes(
    nodes: impl IntoIterator<Item = InnerNodeInfo>,
) -> impl Iterator<Item = (RpoDigest, StoreNode)> {
    nodes
        .into_iter()
        .map(|info| {
            (
                info.value,
                StoreNode {
                    left: info.left,
                    right: info.right,
                },
            )
        })
        .chain(empty_hashes().into_iter())
}