Skip to main content

commonware_storage/qmdb/keyless/
mod.rs

1//! The [Keyless] qmdb allows for append-only storage of data that can later be retrieved by its
2//! location. Both fixed-size and variable-size values are supported via the [fixed] and [variable]
3//! submodules.
4//!
5//! # Examples
6//!
7//! ```ignore
8//! // Simple mode: apply a batch, then durably commit it.
9//! let batch = db.new_batch().append(value);
10//! let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
11//! let (db, _) = db.apply_batch(merkleized).await?;
12//! let db = db.commit().await?;
13//! ```
14//!
15//! ```ignore
16//! // Batches can still fork before you apply them.
17//! let floor = db.inactivity_floor_loc();
18//! let parent = db.new_batch().append(value_a);
19//! let parent = parent.merkleize(&db, None, floor).await;
20//!
21//! let child_a = parent.new_batch();
22//! let child_a = child_a.append(value_b);
23//! let child_a = child_a.merkleize(&db, None, floor).await;
24//!
25//! let child_b = parent.new_batch();
26//! let child_b = child_b.append(value_c);
27//! let child_b = child_b.merkleize(&db, None, floor).await;
28//!
29//! let (db, _) = db.apply_batch(child_a).await?;
30//! let db = db.commit().await?;
31//! ```
32//!
33//! ```ignore
34//! // Sequential commit: apply parent then child.
35//! let floor = db.inactivity_floor_loc();
36//! let parent = db.new_batch().append(value_a);
37//! let parent_m = parent.merkleize(&db, None, floor).await;
38//! let child = parent_m.new_batch().append(value_b);
39//! let child_m = child.merkleize(&db, None, floor).await;
40//!
41//! let (db, _) = db.apply_batch(parent_m).await?;
42//! let (db, _) = db.apply_batch(child_m).await?;
43//! let db = db.commit().await?;
44//! ```
45
46use crate::{
47    Context,
48    journal::{
49        authenticated,
50        contiguous::{Contiguous, Mutable},
51    },
52    merkle::{Family, Location, Proof, full::Config as MerkleConfig},
53    qmdb::{
54        Error, any::value::ValueEncoding, batch_chain, metrics::Metrics, single_operation_root,
55    },
56};
57use commonware_codec::EncodeShared;
58use commonware_cryptography::Hasher;
59use commonware_macros::boxed;
60use commonware_parallel::Strategy;
61use commonware_runtime::Handle;
62use std::{num::NonZeroU64, sync::Arc};
63use tracing::{debug, warn};
64
65pub mod batch;
66mod compact;
67pub mod fixed;
68mod operation;
69pub(crate) mod sync;
70pub mod variable;
71pub use compact::{
72    Config as CompactConfig, Db as CompactDb, MerkleizedBatch as CompactMerkleizedBatch,
73    UnmerkleizedBatch as CompactUnmerkleizedBatch,
74};
75pub use operation::Operation;
76
77/// Compute the authenticated root of a newly initialized database without opening storage.
78///
79/// The initial commit never carries metadata, so this root always represents `Commit(None, 0)`.
80pub fn initial_root<F, V, H>() -> H::Digest
81where
82    F: Family,
83    V: ValueEncoding,
84    H: Hasher,
85    Operation<F, V>: EncodeShared,
86{
87    single_operation_root::<F, H>(&Operation::<F, V>::Commit(None, Location::new(0)))
88}
89
90/// Configuration for a [Keyless] authenticated db.
91#[derive(Clone)]
92pub struct Config<J, S: Strategy> {
93    /// Configuration for the Merkle structure backing the authenticated journal.
94    pub merkle: MerkleConfig<S>,
95
96    /// Configuration for the operations log journal.
97    pub log: J,
98}
99
100/// A keyless authenticated database.
101pub struct Keyless<F, E, V, C, H, S>
102where
103    F: Family,
104    E: Context,
105    V: ValueEncoding,
106    C: Contiguous<Item = Operation<F, V>>,
107    H: Hasher,
108    S: Strategy,
109    Operation<F, V>: EncodeShared,
110{
111    /// Authenticated journal of operations.
112    journal: authenticated::Journal<F, E, C, H, S>,
113
114    /// Cached canonical operations root.
115    root: H::Digest,
116
117    /// The location of the last commit, if any.
118    last_commit_loc: Location<F>,
119
120    /// The inactivity floor declared by the last committed batch. Operations at locations below
121    /// this value are considered inactive by the application and may be pruned.
122    inactivity_floor_loc: Location<F>,
123
124    /// Metrics for this database.
125    metrics: Metrics<E>,
126}
127
128impl<F, E, V, C, H, S> std::fmt::Debug for Keyless<F, E, V, C, H, S>
129where
130    F: Family,
131    E: Context,
132    V: ValueEncoding,
133    C: Mutable<Item = Operation<F, V>>,
134    H: Hasher,
135    S: Strategy,
136    Operation<F, V>: EncodeShared,
137{
138    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
139        f.debug_struct("Keyless")
140            .field("bounds", &self.bounds())
141            .field("inactivity_floor_loc", &self.inactivity_floor_loc())
142            .finish_non_exhaustive()
143    }
144}
145
146impl<F, E, V, C, H, S> Keyless<F, E, V, C, H, S>
147where
148    F: Family,
149    E: Context,
150    V: ValueEncoding,
151    C: Mutable<Item = Operation<F, V>>,
152    H: Hasher,
153    S: Strategy,
154    Operation<F, V>: EncodeShared,
155{
156    #[boxed]
157    pub(crate) async fn init_from_journal(
158        mut journal: authenticated::Journal<F, E, C, H, S>,
159        context: E,
160    ) -> Result<Self, Error<F>> {
161        let metrics = Metrics::new(context);
162        if journal.size() == 0 {
163            warn!("no operations found in log, creating initial commit");
164            (journal, _) = journal
165                .append(&Operation::Commit(None, Location::new(0)))
166                .await?;
167            journal = journal.sync().await?;
168        }
169
170        let (last_commit_loc, inactivity_floor_loc) = {
171            let bounds = journal.bounds();
172            let last_commit_loc = Location::new(
173                bounds
174                    .end
175                    .checked_sub(1)
176                    .expect("at least one commit should exist"),
177            );
178            let op = journal.read(*last_commit_loc).await?;
179            let inactivity_floor_loc = op
180                .has_floor()
181                .expect("last operation should be a commit with floor");
182            (last_commit_loc, inactivity_floor_loc)
183        };
184        let inactive_peaks = F::inactive_peaks(last_commit_loc + 1, inactivity_floor_loc);
185        let root = journal.root(inactive_peaks)?;
186
187        let db = Self {
188            journal,
189            root,
190            last_commit_loc,
191            inactivity_floor_loc,
192            metrics,
193        };
194        db.update_metrics();
195        Ok(db)
196    }
197
198    /// Get the value at location `loc` in the database.
199    ///
200    /// # Errors
201    ///
202    /// Returns [`Error::LocationOutOfBounds`] if `loc` >=
203    /// `self.bounds().end`.
204    pub async fn get(&self, loc: Location<F>) -> Result<Option<V::Value>, Error<F>> {
205        let _timer = self.metrics.get_timer();
206        self.metrics.get_calls.inc();
207        self.metrics.lookups_requested.inc();
208        let op_count = self.journal.bounds().end;
209        if loc >= op_count {
210            return Err(Error::LocationOutOfBounds(loc, Location::new(op_count)));
211        }
212        let op = self.journal.read(*loc).await?;
213
214        let result = op.into_value();
215        Ok(result)
216    }
217
218    /// Batch read values at multiple locations.
219    ///
220    /// Locations must be strictly increasing.
221    /// Returns results in the same order as the input locations.
222    ///
223    /// # Errors
224    ///
225    /// Returns [`Error::LocationOutOfBounds`] if any location >= `bounds().end`.
226    pub async fn get_many(&self, locs: &[Location<F>]) -> Result<Vec<Option<V::Value>>, Error<F>> {
227        if locs.is_empty() {
228            return Ok(Vec::new());
229        }
230
231        let _timer = self.metrics.get_many_timer();
232        self.metrics.get_many_calls.inc();
233        self.metrics.lookups_requested.inc_by(locs.len() as u64);
234        assert!(
235            locs.is_sorted_by(|a, b| a < b),
236            "locations must be strictly increasing"
237        );
238        let op_count = self.journal.bounds().end;
239        for &loc in locs {
240            if loc >= op_count {
241                return Err(Error::LocationOutOfBounds(loc, Location::new(op_count)));
242            }
243        }
244        let positions: Vec<u64> = locs.iter().map(|loc| **loc).collect();
245        let ops = self.journal.read_many(&positions).await?;
246        let result = ops.into_iter().map(|op| op.into_value()).collect();
247        Ok(result)
248    }
249
250    /// Returns the location of the last commit.
251    pub const fn last_commit_loc(&self) -> Location<F> {
252        self.last_commit_loc
253    }
254
255    /// Returns the inactivity floor declared by the last committed batch.
256    pub const fn inactivity_floor_loc(&self) -> Location<F> {
257        self.inactivity_floor_loc
258    }
259
260    /// Return [start, end) where `start` and `end - 1` are the Locations of the oldest and newest
261    /// retained operations respectively.
262    pub fn bounds(&self) -> std::ops::Range<Location<F>> {
263        let bounds = self.journal.bounds();
264        Location::new(bounds.start)..Location::new(bounds.end)
265    }
266
267    /// Update state gauges from the current database state.
268    fn update_metrics(&self) {
269        let bounds = self.journal.bounds();
270        self.metrics.update(
271            bounds.end,
272            bounds.start,
273            *self.inactivity_floor_loc,
274            *self.last_commit_loc,
275        );
276    }
277
278    /// Return the most recent location from which this database can safely be synced, and the
279    /// upper bound on [`Self::prune`]'s `loc`. For keyless databases, this equals the
280    /// inactivity floor declared by the last committed batch.
281    pub const fn sync_boundary(&self) -> Location<F> {
282        self.inactivity_floor_loc
283    }
284
285    /// Get the metadata associated with the last commit.
286    pub async fn get_metadata(&self) -> Result<Option<V::Value>, Error<F>> {
287        let op = self.journal.read(*self.last_commit_loc).await?;
288        let Operation::Commit(metadata, _floor) = op else {
289            return Ok(None);
290        };
291
292        Ok(metadata)
293    }
294
295    /// Return the root of the db.
296    pub const fn root(&self) -> H::Digest {
297        self.root
298    }
299
300    /// Return a reference to the merkleization strategy.
301    pub const fn strategy(&self) -> &S {
302        self.journal.strategy()
303    }
304
305    /// Generate and return:
306    ///  1. a proof of all operations applied to the db in the range starting at (and including)
307    ///     location `start_loc`, and ending at the first of either:
308    ///     - the last operation performed, or
309    ///     - the operation `max_ops` from the start.
310    ///  2. the operations corresponding to the leaves in this range.
311    ///
312    /// # Errors
313    ///
314    /// - Returns [`Error::Merkle`] with [`crate::merkle::Error::RangeOutOfBounds`] if `start_loc`
315    ///   >= the number of operations.
316    /// - Returns [`Error::Journal`] with [`crate::journal::Error::ItemPruned`] if `start_loc` has
317    ///   been pruned.
318    pub async fn proof(
319        &self,
320        start_loc: Location<F>,
321        max_ops: NonZeroU64,
322    ) -> Result<(Proof<F, H::Digest>, Vec<Operation<F, V>>), Error<F>> {
323        self.historical_proof(self.bounds().end, start_loc, max_ops)
324            .await
325    }
326
327    /// Analogous to proof, but with respect to the state of the Merkle structure when it had
328    /// `op_count` operations.
329    ///
330    /// `op_count` must be the size of a commit boundary.
331    ///
332    /// # Errors
333    ///
334    /// - Returns [`Error::Merkle`] with [`crate::merkle::Error::RangeOutOfBounds`] if `start_loc`
335    ///   >= `op_count` or `op_count` > number of operations.
336    /// - Returns [`Error::Journal`] with [`crate::journal::Error::ItemPruned`] if `start_loc` has
337    ///   been pruned.
338    /// - Returns [`Error::HistoricalFloorPruned`] if `op_count - 1` is retained but is not a commit
339    ///   op.
340    #[allow(clippy::type_complexity)]
341    #[tracing::instrument(
342        name = "qmdb.keyless.db.historical_proof",
343        level = "info",
344        skip_all,
345        fields(
346            op_count = *op_count,
347            start_loc = *start_loc,
348            max_ops = max_ops.get(),
349        ),
350    )]
351    pub async fn historical_proof(
352        &self,
353        op_count: Location<F>,
354        start_loc: Location<F>,
355        max_ops: NonZeroU64,
356    ) -> Result<(Proof<F, H::Digest>, Vec<Operation<F, V>>), Error<F>> {
357        if op_count > self.journal.size() {
358            return Err(crate::merkle::Error::RangeOutOfBounds(op_count).into());
359        }
360
361        let inactive_peaks =
362            crate::qmdb::inactive_peaks_at::<F, _>(&self.journal, op_count).await?;
363
364        Ok(self
365            .journal
366            .historical_proof(op_count, start_loc, max_ops, inactive_peaks)
367            .await?)
368    }
369
370    /// Return the pinned Merkle nodes for a lower operation boundary of `loc`.
371    pub async fn pinned_nodes_at(&self, loc: Location<F>) -> Result<Vec<H::Digest>, Error<F>> {
372        self.journal
373            .merkle
374            .pinned_nodes_at(loc)
375            .await
376            .map_err(Into::into)
377    }
378
379    /// Prune historical operations prior to `loc`. This does not affect the db's root.
380    ///
381    /// `prune` requires no prior commit. After a crash, the database remains recoverable;
382    /// uncommitted operations are not guaranteed to survive.
383    ///
384    /// # Errors
385    ///
386    /// - Returns [`Error::PruneBeyondMinRequired`] if `loc` > the inactivity floor.
387    #[tracing::instrument(name = "qmdb.keyless.db.prune", level = "info", skip_all)]
388    #[boxed]
389    pub async fn prune(mut self, loc: Location<F>) -> Result<Self, Error<F>> {
390        let _timer = self.metrics.prune_timer();
391        self.metrics.prune_calls.inc();
392        if loc > self.inactivity_floor_loc {
393            return Err(Error::PruneBeyondMinRequired(
394                loc,
395                self.inactivity_floor_loc,
396            ));
397        }
398        (self.journal, _) = self.journal.prune(loc).await?;
399        self.update_metrics();
400        Ok(self)
401    }
402
403    /// Rewind the database to `size` operations, where `size` is the location of the next append.
404    ///
405    /// This rewinds both the operations journal and its Merkle structure to the historical state
406    /// at `size`. The inactivity floor is restored from the rewind target commit operation, so
407    /// the post-rewind floor matches the floor that was in effect at that commit.
408    ///
409    /// # Errors
410    ///
411    /// - Returns [`Error::Journal`] with [`crate::journal::Error::InvalidRewind`] if `size` is 0
412    ///   or exceeds the current committed size.
413    /// - Returns [`Error::Journal`] with [`crate::journal::Error::ItemPruned`] if the operation at
414    ///   `size - 1` has been pruned.
415    /// - Returns [`Error::UnexpectedData`] if the operation at `size - 1` is not a commit.
416    ///
417    /// Any error from this method is fatal for this handle. Rewind may mutate journal state
418    /// before this method finishes updating in-memory rewind state. Callers must drop this
419    /// database handle after any `Err` from `rewind` and reopen from storage.
420    ///
421    /// A successful rewind is not restart-stable until a subsequent [`Self::commit`] or
422    /// [`Self::sync`] completes, or until the handle returned by a subsequent
423    /// [`Self::start_sync`] completes.
424    #[tracing::instrument(name = "qmdb.keyless.db.rewind", level = "info", skip_all)]
425    #[boxed]
426    pub async fn rewind(mut self, size: Location<F>) -> Result<Self, Error<F>> {
427        let rewind_size = *size;
428        let current_size = *self.last_commit_loc + 1;
429        if rewind_size == current_size {
430            return Ok(self);
431        }
432        if rewind_size == 0 || rewind_size > current_size {
433            return Err(Error::Journal(crate::journal::Error::InvalidRewind(
434                rewind_size,
435            )));
436        }
437
438        let rewind_last_loc = Location::new(rewind_size - 1);
439        let rewind_floor = {
440            let bounds = self.journal.bounds();
441            if rewind_size <= bounds.start {
442                return Err(Error::Journal(crate::journal::Error::ItemPruned(
443                    *rewind_last_loc,
444                )));
445            }
446            let rewind_last_op = self.journal.read(*rewind_last_loc).await?;
447            let Operation::Commit(_, floor) = rewind_last_op else {
448                return Err(Error::UnexpectedData(rewind_last_loc));
449            };
450            floor
451        };
452
453        // Journal rewind happens before in-memory commit-location updates. If a later step fails,
454        // this handle may be internally diverged and must be dropped by the caller.
455        self.journal = self.journal.rewind(rewind_size).await?;
456        self.last_commit_loc = rewind_last_loc;
457        self.inactivity_floor_loc = rewind_floor;
458        let inactive_peaks = F::inactive_peaks(size, rewind_floor);
459        self.root = self.journal.root(inactive_peaks)?;
460        self.update_metrics();
461        Ok(self)
462    }
463
464    /// Sync all database state to disk. While this isn't necessary to ensure durability of
465    /// committed operations, periodic invocation may reduce memory usage and the time required to
466    /// recover the database on restart.
467    #[tracing::instrument(name = "qmdb.keyless.db.sync", level = "info", skip_all)]
468    pub async fn sync(mut self) -> Result<Self, Error<F>> {
469        let _timer = self.metrics.sync_timer();
470        self.metrics.sync_calls.inc();
471        self.journal = self.journal.sync().await?;
472        Ok(self)
473    }
474
475    /// Begin durably persisting the journal state published by prior [`Keyless::apply_batch`]
476    /// calls.
477    ///
478    /// Awaiting the returned [Handle] provides the same durability guarantee as [Self::commit],
479    /// plus a best-effort attempt to bound the recovery needed on startup. Use [Self::sync] to
480    /// guarantee none is needed. A new sync waits for the prior sync before starting. Failures
481    /// of the deferred durability work surface on the returned handle. A failed data sync also
482    /// fails the next durability operation. A failed recovery-watermark sync is not observed by
483    /// [Self::commit], and a failed merkle-node sync may not be. Both resurface on the next
484    /// [Self::sync].
485    #[tracing::instrument(name = "qmdb.keyless.db.start_sync", level = "info", skip_all)]
486    pub async fn start_sync(mut self) -> Result<(Self, Handle<()>), Error<F>> {
487        self.metrics.start_sync_calls.inc();
488        let handle;
489        (self.journal, handle) = self.journal.start_sync().await?;
490        Ok((self, handle))
491    }
492
493    /// Durably commit the journal state published by prior [`Keyless::apply_batch`] calls.
494    #[tracing::instrument(name = "qmdb.keyless.db.commit", level = "info", skip_all)]
495    pub async fn commit(mut self) -> Result<Self, Error<F>> {
496        let _timer = self.metrics.commit_timer();
497        self.metrics.commit_calls.inc();
498        self.journal = self.journal.commit().await?;
499        Ok(self)
500    }
501
502    /// Destroy the db, removing all data from disk.
503    #[boxed]
504    pub async fn destroy(self) -> Result<(), Error<F>> {
505        Ok(self.journal.destroy().await?)
506    }
507
508    /// The [`Commitment`](batch_chain::Commitment) for the database's current state.
509    pub(crate) fn commitment(&self) -> batch_chain::Commitment<F, H::Digest> {
510        batch_chain::Commitment::new(self.last_commit_loc + 1, self.root)
511    }
512
513    /// Create a new speculative batch of operations with this database as its parent.
514    pub fn new_batch(&self) -> batch::UnmerkleizedBatch<F, H, V, S> {
515        batch::UnmerkleizedBatch::new(self, self.commitment())
516    }
517
518    /// Create an initial [`batch::MerkleizedBatch`] from the committed DB state.
519    pub fn to_batch(&self) -> Arc<batch::MerkleizedBatch<F, H::Digest, V, S>> {
520        Arc::new(batch::MerkleizedBatch {
521            journal_batch: self.journal.to_merkleized_batch(),
522            parent: None,
523            bounds: batch_chain::Bounds::from_db(self.commitment(), self.inactivity_floor_loc),
524        })
525    }
526
527    /// Check that `batch` can be applied to the database in its current state, without
528    /// applying it.
529    ///
530    /// [`Self::apply_batch`] runs the same validation but consumes the database when it
531    /// fails; callers that want to reject a bad batch and keep the handle can check first.
532    pub fn validate_batch(
533        &self,
534        batch: &batch::MerkleizedBatch<F, H::Digest, V, S>,
535    ) -> Result<(), Error<F>> {
536        batch
537            .bounds
538            .validate_apply_to(self.commitment(), self.inactivity_floor_loc)
539    }
540
541    /// Apply a [`batch::MerkleizedBatch`] to the database.
542    ///
543    /// A batch is valid only if every batch applied to the database since this batch's
544    /// ancestor chain was created is an ancestor of this batch. Applying a batch from a
545    /// different fork returns [`Error::StaleBatch`] (see [`crate::qmdb::batch_chain`] for
546    /// more details).
547    ///
548    /// Every commit operation in the batch chain (each unapplied ancestor's commit plus the
549    /// tip's) must satisfy two per-commit invariants:
550    ///
551    /// 1. The floor is monotonically non-decreasing across the chain, starting from the
552    ///    database's current inactivity floor.
553    /// 2. The floor is at most the commit operation's own location (`total_size - 1` at that
554    ///    point). A floor past the commit would let a later `prune(floor)` remove the last
555    ///    readable commit from the journal.
556    ///
557    /// Violations return [`Error::FloorRegressed`] or [`Error::FloorBeyondSize`] identifying
558    /// the offending floor and the bound it crossed (the prior validated floor, or the commit
559    /// location, respectively). Floor validation happens before any journal mutation, so on floor
560    /// errors the on-disk state is unchanged and reopening recovers the database as it was.
561    ///
562    /// Returns the range of locations written.
563    ///
564    /// This publishes the batch to the in-memory database state and appends it to the journal.
565    /// Call [`Keyless::commit`] or [`Keyless::sync`], or await the handle returned by
566    /// [`Keyless::start_sync`], to make the applied state durable.
567    #[tracing::instrument(name = "qmdb.keyless.db.apply_batch", level = "info", skip_all)]
568    pub async fn apply_batch(
569        mut self,
570        batch: Arc<batch::MerkleizedBatch<F, H::Digest, V, S>>,
571    ) -> Result<(Self, core::ops::Range<Location<F>>), Error<F>> {
572        let _timer = self.metrics.apply_batch_timer();
573        self.metrics.apply_batch_calls.inc();
574        self.validate_batch(&batch)?;
575        let start_loc = self.last_commit_loc + 1;
576
577        self.journal = self.journal.apply_batch(&batch.journal_batch).await?;
578
579        self.last_commit_loc = batch.bounds.tip.size - 1;
580        self.inactivity_floor_loc = batch.bounds.inactivity_floor;
581        self.root = batch.root();
582        let end_loc = batch.bounds.tip.size;
583        debug!(size = ?end_loc, "applied batch");
584        let range = start_loc..end_loc;
585        self.update_metrics();
586        self.metrics
587            .operations_applied
588            .inc_by(*range.end - *range.start);
589        Ok((self, range))
590    }
591}
592
593impl<F, E, V, C, H, S> crate::qmdb::sync::Source for Keyless<F, E, V, C, H, S>
594where
595    F: Family,
596    E: Context,
597    V: ValueEncoding,
598    C: Mutable<Item = Operation<F, V>>,
599    H: Hasher,
600    S: Strategy,
601    Operation<F, V>: EncodeShared,
602{
603    type Family = F;
604    type Digest = H::Digest;
605    type Op = Operation<F, V>;
606    type Error = Error<F>;
607
608    async fn serve(
609        &self,
610        request: crate::qmdb::sync::Request<F>,
611    ) -> Result<
612        (
613            crate::qmdb::sync::Response<F, Self::Op, Self::Digest>,
614            crate::qmdb::sync::FeedbackTx,
615        ),
616        Self::Error,
617    > {
618        self.journal.serve(request).await
619    }
620}
621
622#[cfg(test)]
623pub(crate) mod tests {
624    use super::*;
625    use crate::qmdb::{verify_proof, verify_proof_and_pinned_nodes};
626    use commonware_cryptography::Sha256;
627    use commonware_parallel::Strategy;
628    use commonware_runtime::{Supervisor as _, deterministic};
629    use commonware_utils::NZU64;
630    use std::{future::Future, pin::Pin};
631
632    pub(crate) type Reopen<D> =
633        Box<dyn Fn(deterministic::Context) -> Pin<Box<dyn Future<Output = D> + Send>>>;
634
635    type TestKeyless<F, V, C, H, S> = Keyless<F, deterministic::Context, V, C, H, S>;
636
637    /// Test value factory: creates distinct values from an index.
638    pub(crate) trait TestValue: Clone + PartialEq + std::fmt::Debug + Send + Sync {
639        fn make(i: u64) -> Self;
640    }
641
642    impl TestValue for Vec<u8> {
643        fn make(i: u64) -> Self {
644            vec![(i % 255) as u8; ((i % 13) + 7) as usize]
645        }
646    }
647
648    impl TestValue for commonware_utils::sequence::U64 {
649        fn make(i: u64) -> Self {
650            Self::new(i * 10 + 1)
651        }
652    }
653
654    /// Emits the named test against `mmr::Family` and `mmb::Family`.
655    macro_rules! keyless_tests {
656        ($($name:ident => $scenario:ident, $fixture:ident;)*) => {
657            $(
658                #[test_traced]
659                fn $name() {
660                    deterministic::Runner::default().start(|ctx| async move {
661                        keyless_tests!(@fixture $fixture, $scenario, mmr, ctx);
662                    });
663                }
664            )*
665            paste::paste! {
666                $(
667                    #[test_traced]
668                    fn [<$name _mmb>]() {
669                        deterministic::Runner::default().start(|ctx| async move {
670                            keyless_tests!(@fixture $fixture, $scenario, mmb, ctx);
671                        });
672                    }
673                )*
674            }
675        };
676        (@fixture db, $scenario:ident, $family:ident, $ctx:ident) => {
677            let db = open_db::<$family::Family>($ctx.child("db")).await;
678            tests::$scenario(db).await;
679        };
680        (@fixture reopen, $scenario:ident, $family:ident, $ctx:ident) => {
681            let db = open_db::<$family::Family>($ctx.child("db")).await;
682            tests::$scenario($ctx, db, reopen::<$family::Family>()).await;
683        };
684        (@fixture reopen_indexed, $scenario:ident, $family:ident, $ctx:ident) => {
685            let db =
686                open_db::<$family::Family>($ctx.child("db").with_attribute("index", 1)).await;
687            tests::$scenario($ctx, db, reopen::<$family::Family>()).await;
688        };
689    }
690
691    pub(super) use keyless_tests;
692
693    #[boxed]
694    pub(crate) async fn run_empty<F: Family, V, C, H, S: Strategy>(
695        context: deterministic::Context,
696        db: TestKeyless<F, V, C, H, S>,
697        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
698    ) where
699        V: ValueEncoding<Value: TestValue>,
700        C: Mutable<Item = Operation<F, V>>,
701        H: Hasher,
702        Operation<F, V>: EncodeShared,
703    {
704        let bounds = db.bounds();
705        assert_eq!(bounds.end, 1); // initial commit should exist
706        assert_eq!(bounds.start, Location::new(0));
707        assert_eq!(db.get_metadata().await.unwrap(), None);
708        assert_eq!(db.last_commit_loc(), Location::new(0));
709
710        // Make sure closing/reopening gets us back to the same state, even after adding an uncommitted op.
711        let root = db.root();
712        {
713            db.new_batch().append(V::Value::make(1));
714            // Don't merkleize/apply -- simulate failed commit
715        }
716        drop(db);
717
718        let db = reopen(context.child("db").with_attribute("index", 2)).await;
719        assert_eq!(db.root(), root);
720        assert_eq!(db.bounds().end, 1);
721        assert_eq!(db.get_metadata().await.unwrap(), None);
722
723        // Test calling commit on an empty db which should make it (durably) non-empty.
724        let metadata = V::Value::make(99);
725        let merkleized = db
726            .new_batch()
727            .merkleize(&db, Some(metadata.clone()), db.inactivity_floor_loc())
728            .await;
729        let (db, _) = db.apply_batch(merkleized).await.unwrap();
730        let db = db.commit().await.unwrap();
731        assert_eq!(db.bounds().end, 2); // 2 commit ops
732        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone()));
733        assert_eq!(
734            db.get(Location::new(1)).await.unwrap(),
735            Some(metadata.clone())
736        ); // the commit op
737        let root = db.root();
738
739        // Commit op should remain after reopen even without clean shutdown.
740        let db = reopen(context.child("db").with_attribute("index", 3)).await;
741        assert_eq!(db.bounds().end, 2); // commit op should remain after re-open.
742        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata));
743        assert_eq!(db.root(), root);
744        assert_eq!(db.last_commit_loc(), Location::new(1));
745
746        db.destroy().await.unwrap();
747    }
748
749    /// `operations()` must cover exactly the batch's own applied range and match the
750    /// operations a post-apply `historical_proof` recovers from the log, for a db-based
751    /// batch and for a chained batch applied after its ancestor.
752    #[boxed]
753    pub(crate) async fn run_operations_match_applied_log<F: Family, V, C, H, S: Strategy>(
754        db: TestKeyless<F, V, C, H, S>,
755    ) where
756        V: ValueEncoding<Value: TestValue>,
757        C: Mutable<Item = Operation<F, V>>,
758        H: Hasher,
759        Operation<F, V>: EncodeShared + PartialEq + core::fmt::Debug,
760    {
761        let seed = db
762            .new_batch()
763            .append(V::Value::make(1))
764            .append(V::Value::make(2))
765            .merkleize(&db, None, db.inactivity_floor_loc())
766            .await;
767        let (seed_start, seed_ops) = seed.operations();
768        let seed_root = seed.root();
769        let seed_proof = seed.proof(&db).unwrap();
770        let seed_pins = seed.pinned_nodes(&db).unwrap();
771        let (db, seed_range) = db.apply_batch(seed).await.unwrap();
772        assert_eq!(seed_start, seed_range.start);
773        assert_eq!(*seed_start + seed_ops.len() as u64, *seed_range.end);
774
775        // A chained batch's operations are its own suffix only.
776        let parent = db
777            .new_batch()
778            .append(V::Value::make(3))
779            .merkleize(&db, None, db.inactivity_floor_loc())
780            .await;
781        let child = parent
782            .new_batch::<H>()
783            .append(V::Value::make(4))
784            .merkleize(&db, None, db.inactivity_floor_loc())
785            .await;
786        let (parent_start, parent_ops) = parent.operations();
787        let (child_start, child_ops) = child.operations();
788        let (parent_root, child_root) = (parent.root(), child.root());
789        let (parent_pins, child_pins) = (
790            parent.pinned_nodes(&db).unwrap(),
791            child.pinned_nodes(&db).unwrap(),
792        );
793        let (parent_proof, child_proof) = (parent.proof(&db).unwrap(), child.proof(&db).unwrap());
794        let (db, parent_range) = db.apply_batch(parent).await.unwrap();
795        let (db, child_range) = db.apply_batch(child).await.unwrap();
796        assert_eq!(parent_start, parent_range.start);
797        assert_eq!(*parent_start + parent_ops.len() as u64, *parent_range.end);
798        assert_eq!(child_start, child_range.start);
799        assert_eq!(*child_start + child_ops.len() as u64, *child_range.end);
800
801        // A write-free batch still captures its commit-only suffix.
802        let empty = db
803            .new_batch()
804            .merkleize(&db, None, db.inactivity_floor_loc())
805            .await;
806        let (empty_start, empty_ops) = empty.operations();
807        let (empty_root, empty_proof) = (empty.root(), empty.proof(&db).unwrap());
808        let empty_pins = empty.pinned_nodes(&db).unwrap();
809        let (db, empty_range) = db.apply_batch(empty).await.unwrap();
810        assert_eq!(empty_start, empty_range.start);
811        assert_eq!(*empty_start + empty_ops.len() as u64, *empty_range.end);
812
813        // Every captured delta and proof must match what the log recovers for its
814        // range, and verify against the batch's own root with and without the pins.
815        for (start, ops, proof, pins, root) in [
816            (seed_start, seed_ops, seed_proof, seed_pins, seed_root),
817            (
818                parent_start,
819                parent_ops,
820                parent_proof,
821                parent_pins,
822                parent_root,
823            ),
824            (child_start, child_ops, child_proof, child_pins, child_root),
825            (empty_start, empty_ops, empty_proof, empty_pins, empty_root),
826        ] {
827            let len = core::num::NonZeroU64::new(ops.len() as u64).unwrap();
828            let end = Location::new(*start + ops.len() as u64);
829            let (log_proof, log_ops) = db.historical_proof(end, start, len).await.unwrap();
830            assert_eq!(log_ops, *ops);
831            assert_eq!(log_proof, proof);
832            assert!(verify_proof::<H, _, _>(&proof, start, &ops, &root));
833            assert!(verify_proof_and_pinned_nodes::<H, _, _>(
834                &proof, start, &ops, &pins, &root
835            ));
836        }
837
838        // Flushing the applied batch prunes the store to its peaks. The late batch's base is
839        // mid-mountain, so its artifacts are refused rather than returned unverifiable.
840        let late = db
841            .new_batch()
842            .append(V::Value::make(5))
843            .merkleize(&db, None, db.inactivity_floor_loc())
844            .await;
845        let (db, _) = db.apply_batch(Arc::clone(&late)).await.unwrap();
846        let db = db.commit().await.unwrap();
847        assert!(matches!(
848            late.proof(&db),
849            Err(crate::qmdb::Error::Merkle(
850                crate::merkle::Error::ElementPruned(_)
851            ))
852        ));
853        assert!(matches!(
854            late.pinned_nodes(&db),
855            Err(crate::qmdb::Error::Merkle(
856                crate::merkle::Error::ElementPruned(_)
857            ))
858        ));
859
860        // A batch built on the flushed store reads every node below it from the pinned peaks.
861        let flushed = db
862            .new_batch()
863            .append(V::Value::make(6))
864            .merkleize(&db, None, db.inactivity_floor_loc())
865            .await;
866        let (flushed_start, flushed_ops) = flushed.operations();
867        let flushed_root = flushed.root();
868        let flushed_proof = flushed.proof(&db).unwrap();
869        let flushed_pins = flushed.pinned_nodes(&db).unwrap();
870        assert!(verify_proof_and_pinned_nodes::<H, _, _>(
871            &flushed_proof,
872            flushed_start,
873            &flushed_ops,
874            &flushed_pins,
875            &flushed_root
876        ));
877        let (db, flushed_range) = db.apply_batch(flushed).await.unwrap();
878        assert_eq!(flushed_start, flushed_range.start);
879        assert_eq!(
880            *flushed_start + flushed_ops.len() as u64,
881            *flushed_range.end
882        );
883
884        db.destroy().await.unwrap();
885    }
886
887    #[boxed]
888    pub(crate) async fn run_commit_after_sync_recovery<F: Family, V, C, H, S: Strategy>(
889        context: deterministic::Context,
890        db: TestKeyless<F, V, C, H, S>,
891        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
892    ) where
893        V: ValueEncoding<Value: TestValue>,
894        C: Mutable<Item = Operation<F, V>>,
895        H: Hasher,
896        Operation<F, V>: EncodeShared,
897    {
898        let value0 = V::Value::make(10);
899        let value1 = V::Value::make(20);
900
901        // Commit and sync the first append so recovery has an older durable boundary.
902        let first_loc = Location::new(1);
903        let merkleized = db
904            .new_batch()
905            .append(value0.clone())
906            .merkleize(&db, None, db.inactivity_floor_loc())
907            .await;
908        let (db, _) = db.apply_batch(merkleized).await.unwrap();
909        let db = db.commit().await.unwrap();
910        let db = db.sync().await.unwrap();
911
912        // Commit the next append without syncing; reopen must replay it from the journal.
913        let second_loc = db.bounds().end;
914        let merkleized = db
915            .new_batch()
916            .append(value1.clone())
917            .merkleize(&db, None, db.inactivity_floor_loc())
918            .await;
919        let (db, _) = db.apply_batch(merkleized).await.unwrap();
920        let db = db.commit().await.unwrap();
921        let committed_bounds = db.bounds();
922        let committed_root = db.root();
923        drop(db);
924
925        let db = reopen(context.child("db").with_attribute("index", 2)).await;
926        assert_eq!(db.bounds(), committed_bounds);
927        assert_eq!(db.root(), committed_root);
928        assert_eq!(db.get(first_loc).await.unwrap(), Some(value0));
929        assert_eq!(db.get(second_loc).await.unwrap(), Some(value1));
930
931        db.destroy().await.unwrap();
932    }
933
934    #[boxed]
935    pub(crate) async fn run_build_basic<F: Family, V, C, H, S: Strategy>(
936        context: deterministic::Context,
937        mut db: TestKeyless<F, V, C, H, S>,
938        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
939    ) where
940        V: ValueEncoding<Value: TestValue>,
941        C: Mutable<Item = Operation<F, V>>,
942        H: Hasher,
943        Operation<F, V>: EncodeShared,
944    {
945        // Build a db with 2 values and make sure we can get them back.
946        let v1 = V::Value::make(1);
947        let v2 = V::Value::make(2);
948
949        {
950            let batch = db.new_batch();
951            let loc1 = batch.size();
952            let batch = batch.append(v1.clone());
953            let loc2 = batch.size();
954            let batch = batch.append(v2.clone());
955            assert_eq!(loc1, Location::new(1));
956            assert_eq!(loc2, Location::new(2));
957            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
958            (db, _) = db.apply_batch(merkleized).await.unwrap();
959        }
960
961        // Make sure closing/reopening gets us back to the same state.
962        assert_eq!(db.bounds().end, 4); // 2 appends, 1 commit + 1 initial commit
963        assert_eq!(db.get_metadata().await.unwrap(), None);
964        assert_eq!(db.get(Location::new(3)).await.unwrap(), None); // the commit op
965        let root = db.root();
966        db.sync().await.unwrap();
967
968        let db = reopen(context.child("db").with_attribute("index", 2)).await;
969        assert_eq!(db.bounds().end, 4);
970        assert_eq!(db.root(), root);
971        assert_eq!(db.get(Location::new(1)).await.unwrap().unwrap(), v1);
972        assert_eq!(db.get(Location::new(2)).await.unwrap().unwrap(), v2);
973
974        // Make sure commit operation remains after drop/reopen.
975        drop(db);
976        let db = reopen(context.child("db").with_attribute("index", 3)).await;
977        assert_eq!(db.bounds().end, 4);
978        assert_eq!(db.root(), root);
979
980        db.destroy().await.unwrap();
981    }
982
983    #[boxed]
984    pub(crate) async fn run_recovery<F: Family, V, C, H, S: Strategy>(
985        context: deterministic::Context,
986        db: TestKeyless<F, V, C, H, S>,
987        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
988    ) where
989        V: ValueEncoding<Value: TestValue>,
990        C: Mutable<Item = Operation<F, V>>,
991        H: Hasher,
992        Operation<F, V>: EncodeShared,
993    {
994        let root = db.root();
995        const ELEMENTS: u64 = 100;
996
997        // Create uncommitted appends then simulate failure.
998        {
999            let mut batch = db.new_batch();
1000            for i in 0..ELEMENTS {
1001                batch = batch.append(V::Value::make(i));
1002            }
1003            // Don't merkleize/apply -- simulate failed commit
1004        }
1005        drop(db);
1006        // Should rollback to the previous root.
1007        let mut db = reopen(context.child("db").with_attribute("index", 2)).await;
1008        assert_eq!(root, db.root());
1009
1010        // Apply the updates and commit them this time.
1011        {
1012            let mut batch = db.new_batch();
1013            for i in 0..ELEMENTS {
1014                batch = batch.append(V::Value::make(i + 100));
1015            }
1016            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1017            (db, _) = db.apply_batch(merkleized).await.unwrap();
1018        }
1019        let db = db.commit().await.unwrap();
1020        let root = db.root();
1021
1022        // Create uncommitted appends then simulate failure.
1023        {
1024            let mut batch = db.new_batch();
1025            for i in 0..ELEMENTS {
1026                batch = batch.append(V::Value::make(i + 200));
1027            }
1028            // Don't merkleize/apply -- simulate failed commit
1029        }
1030        drop(db);
1031        // Should rollback to the previous root.
1032        let mut db = reopen(context.child("db").with_attribute("index", 3)).await;
1033        assert_eq!(root, db.root());
1034
1035        // Apply the updates and commit them this time.
1036        {
1037            let mut batch = db.new_batch();
1038            for i in 0..ELEMENTS {
1039                batch = batch.append(V::Value::make(i + 300));
1040            }
1041            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1042            (db, _) = db.apply_batch(merkleized).await.unwrap();
1043        }
1044        let db = db.commit().await.unwrap();
1045        let root = db.root();
1046
1047        // Make sure we can reopen and get back to the same state.
1048        drop(db);
1049        let db = reopen(context.child("db").with_attribute("index", 4)).await;
1050        assert_eq!(db.bounds().end, 2 * ELEMENTS + 3);
1051        assert_eq!(db.root(), root);
1052
1053        db.destroy().await.unwrap();
1054    }
1055
1056    #[boxed]
1057    pub(crate) async fn run_proof<F: Family, V, C, S: Strategy>(
1058        mut db: TestKeyless<F, V, C, Sha256, S>,
1059    ) where
1060        V: ValueEncoding<Value: TestValue>,
1061        C: Mutable<Item = Operation<F, V>>,
1062        Operation<F, V>: EncodeShared + std::fmt::Debug,
1063    {
1064        const ELEMENTS: u64 = 50;
1065
1066        {
1067            let mut batch = db.new_batch();
1068            for i in 0..ELEMENTS {
1069                batch = batch.append(V::Value::make(i));
1070            }
1071            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1072            (db, _) = db.apply_batch(merkleized).await.unwrap();
1073        }
1074        let root = db.root();
1075
1076        let (proof, ops) = db.proof(Location::new(0), NZU64!(100)).await.unwrap();
1077        assert!(verify_proof::<Sha256, _, _>(
1078            &proof,
1079            Location::new(0),
1080            &ops,
1081            &root,
1082        ));
1083        assert_eq!(ops.len() as u64, 1 + ELEMENTS + 1);
1084
1085        let (proof, ops) = db.proof(Location::new(10), NZU64!(5)).await.unwrap();
1086        assert!(verify_proof::<Sha256, _, _>(
1087            &proof,
1088            Location::new(10),
1089            &ops,
1090            &root,
1091        ));
1092        assert_eq!(ops.len(), 5);
1093
1094        db.destroy().await.unwrap();
1095    }
1096
1097    #[boxed]
1098    pub(crate) async fn run_metadata<F: Family, V, C, H, S: Strategy>(
1099        db: TestKeyless<F, V, C, H, S>,
1100    ) where
1101        V: ValueEncoding<Value: TestValue>,
1102        C: Mutable<Item = Operation<F, V>>,
1103        H: Hasher,
1104        Operation<F, V>: EncodeShared,
1105    {
1106        let metadata = V::Value::make(99);
1107        let merkleized = db
1108            .new_batch()
1109            .append(V::Value::make(1))
1110            .merkleize(&db, Some(metadata.clone()), db.inactivity_floor_loc())
1111            .await;
1112        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1113        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata));
1114
1115        let merkleized = db
1116            .new_batch()
1117            .merkleize(&db, None, db.inactivity_floor_loc())
1118            .await;
1119        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1120        assert_eq!(db.get_metadata().await.unwrap(), None);
1121
1122        db.destroy().await.unwrap();
1123    }
1124
1125    #[boxed]
1126    pub(crate) async fn run_pruning<F: Family, V, C, H, S: Strategy>(
1127        context: deterministic::Context,
1128        db: TestKeyless<F, V, C, H, S>,
1129        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
1130    ) where
1131        V: ValueEncoding<Value: TestValue>,
1132        C: Mutable<Item = Operation<F, V>>,
1133        H: Hasher,
1134        Operation<F, V>: EncodeShared,
1135    {
1136        // Initial floor is 0, so pruning past 0 should fail.
1137        assert_eq!(db.inactivity_floor_loc(), Location::new(0));
1138        let result = db.prune(Location::new(1)).await;
1139        assert!(
1140            matches!(result, Err(Error::PruneBeyondMinRequired(prune_loc, floor))
1141                if prune_loc == Location::new(1) && floor == Location::new(0))
1142        );
1143
1144        let db = reopen(context.child("reopen_empty")).await;
1145
1146        // Add values and commit, advancing the floor to the new commit location.
1147        let first_commit_loc = Location::<F>::new(3);
1148        let merkleized = db
1149            .new_batch()
1150            .append(V::Value::make(1))
1151            .append(V::Value::make(2))
1152            .merkleize(&db, None, first_commit_loc)
1153            .await;
1154        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1155        assert_eq!(db.last_commit_loc(), first_commit_loc);
1156        assert_eq!(db.inactivity_floor_loc(), first_commit_loc);
1157
1158        // Append one more, advancing the floor with it.
1159        let second_commit_loc = Location::<F>::new(5);
1160        let merkleized = db
1161            .new_batch()
1162            .append(V::Value::make(3))
1163            .merkleize(&db, None, second_commit_loc)
1164            .await;
1165        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1166
1167        // Valid prune: up to the floor (previous commit location).
1168        let root = db.root();
1169        let db = db.prune(first_commit_loc).await.unwrap();
1170        assert_eq!(db.root(), root);
1171
1172        // Pruning beyond the current floor fails.
1173        let new_floor = db.inactivity_floor_loc();
1174        let beyond = new_floor + 1;
1175        let result = db.prune(beyond).await;
1176        assert!(
1177            matches!(result, Err(Error::PruneBeyondMinRequired(prune_loc, floor))
1178                if prune_loc == beyond && floor == new_floor)
1179        );
1180    }
1181
1182    #[boxed]
1183    pub(crate) async fn run_empty_db_recovery<F: Family, V, C, H, S: Strategy>(
1184        context: deterministic::Context,
1185        db: TestKeyless<F, V, C, H, S>,
1186        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
1187    ) where
1188        V: ValueEncoding<Value: TestValue>,
1189        C: Mutable<Item = Operation<F, V>>,
1190        H: Hasher,
1191        Operation<F, V>: EncodeShared,
1192    {
1193        let root = db.root();
1194        const ELEMENTS: u64 = 200;
1195
1196        // Reopen DB without clean shutdown and make sure the state is the same.
1197        let db = reopen(context.child("db").with_attribute("index", 2)).await;
1198        assert_eq!(db.bounds().end, 1); // initial commit should exist
1199        assert_eq!(db.root(), root);
1200
1201        // Simulate failure after inserting operations without a commit.
1202        {
1203            let mut batch = db.new_batch();
1204            for i in 0..ELEMENTS {
1205                batch = batch.append(V::Value::make(i));
1206            }
1207            // Don't merkleize/apply -- simulate failed commit
1208        }
1209        drop(db);
1210        let db = reopen(context.child("db").with_attribute("index", 3)).await;
1211        assert_eq!(db.bounds().end, 1); // initial commit should exist
1212        assert_eq!(db.root(), root);
1213
1214        // Repeat: simulate failure after inserting operations without a commit.
1215        {
1216            let mut batch = db.new_batch();
1217            for i in 0..ELEMENTS {
1218                batch = batch.append(V::Value::make(i + 500));
1219            }
1220            // Don't merkleize/apply -- simulate failed commit
1221        }
1222        drop(db);
1223        let db = reopen(context.child("db").with_attribute("index", 4)).await;
1224        assert_eq!(db.bounds().end, 1); // initial commit should exist
1225        assert_eq!(db.root(), root);
1226
1227        // One last check: multiple batches of uncommitted appends.
1228        {
1229            let mut batch = db.new_batch();
1230            for i in 0..ELEMENTS * 3 {
1231                batch = batch.append(V::Value::make(i + 1000));
1232            }
1233            // Don't merkleize/apply -- simulate failed commit
1234        }
1235        drop(db);
1236        let mut db = reopen(context.child("db").with_attribute("index", 5)).await;
1237        assert_eq!(db.bounds().end, 1); // initial commit should exist
1238        assert_eq!(db.root(), root);
1239        assert_eq!(db.last_commit_loc(), Location::new(0));
1240
1241        // Apply the ops one last time but fully commit them this time, then clean up.
1242        {
1243            let mut batch = db.new_batch();
1244            for i in 0..ELEMENTS {
1245                batch = batch.append(V::Value::make(i + 2000));
1246            }
1247            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1248            (db, _) = db.apply_batch(merkleized).await.unwrap();
1249        }
1250        db.commit().await.unwrap();
1251        let db = reopen(context.child("db").with_attribute("index", 6)).await;
1252        assert!(db.bounds().end > 1);
1253        assert_ne!(db.root(), root);
1254
1255        db.destroy().await.unwrap();
1256    }
1257
1258    #[boxed]
1259    pub(crate) async fn run_replay_with_trailing_appends<F: Family, V, C, H, S: Strategy>(
1260        context: deterministic::Context,
1261        mut db: TestKeyless<F, V, C, H, S>,
1262        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
1263    ) where
1264        V: ValueEncoding<Value: TestValue>,
1265        C: Mutable<Item = Operation<F, V>>,
1266        H: Hasher,
1267        Operation<F, V>: EncodeShared,
1268    {
1269        // Add some initial operations and commit.
1270        {
1271            let mut batch = db.new_batch();
1272            for i in 0..10u64 {
1273                batch = batch.append(V::Value::make(i));
1274            }
1275            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1276            (db, _) = db.apply_batch(merkleized).await.unwrap();
1277        }
1278        let db = db.commit().await.unwrap();
1279        let committed_root = db.root();
1280        let committed_size = db.bounds().end;
1281
1282        // Add exactly one more append (uncommitted).
1283        {
1284            db.new_batch().append(V::Value::make(99));
1285            // Don't merkleize/apply -- simulate failed commit
1286        }
1287        drop(db);
1288
1289        // Reopen and verify correct recovery.
1290        let mut db = reopen(context.child("db").with_attribute("index", 2)).await;
1291        assert_eq!(
1292            db.bounds().end,
1293            committed_size,
1294            "Should rewind to last commit"
1295        );
1296        assert_eq!(db.root(), committed_root, "Root should match last commit");
1297        assert_eq!(
1298            db.last_commit_loc(),
1299            committed_size - 1,
1300            "Last commit location should be correct"
1301        );
1302
1303        // Verify we can append and commit new data after recovery.
1304        let new_value = V::Value::make(77);
1305        {
1306            let batch = db.new_batch();
1307            let loc = batch.size();
1308            let batch = batch.append(new_value.clone());
1309            assert_eq!(
1310                loc, committed_size,
1311                "New append should get the expected location"
1312            );
1313            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1314            (db, _) = db.apply_batch(merkleized).await.unwrap();
1315        }
1316        let db = db.commit().await.unwrap();
1317
1318        assert_eq!(db.get(committed_size).await.unwrap(), Some(new_value));
1319
1320        let new_committed_root = db.root();
1321        let new_committed_size = db.bounds().end;
1322
1323        // Add multiple uncommitted appends.
1324        {
1325            let mut batch = db.new_batch();
1326            for i in 0..5u64 {
1327                batch = batch.append(V::Value::make(200 + i));
1328            }
1329            // Don't merkleize/apply -- simulate failed commit
1330        }
1331        drop(db);
1332
1333        // Reopen and verify correct recovery.
1334        let db = reopen(context.child("db").with_attribute("index", 3)).await;
1335        assert_eq!(
1336            db.bounds().end,
1337            new_committed_size,
1338            "Should rewind to last commit with multiple trailing appends"
1339        );
1340        assert_eq!(
1341            db.root(),
1342            new_committed_root,
1343            "Root should match last commit after multiple appends"
1344        );
1345        assert_eq!(
1346            db.last_commit_loc(),
1347            new_committed_size - 1,
1348            "Last commit location should be correct after multiple appends"
1349        );
1350
1351        db.destroy().await.unwrap();
1352    }
1353
1354    /// `get_many` on the DB and on unmerkleized/merkleized batches returns
1355    /// results consistent with individual `get` calls.
1356    #[boxed]
1357    pub(crate) async fn run_get_many<F: Family, V, C, S: Strategy>(
1358        db: TestKeyless<F, V, C, Sha256, S>,
1359    ) where
1360        V: ValueEncoding<Value: TestValue>,
1361        C: Mutable<Item = Operation<F, V>>,
1362        Operation<F, V>: EncodeShared,
1363    {
1364        let v1 = V::Value::make(1);
1365        let v2 = V::Value::make(2);
1366        let v3 = V::Value::make(3);
1367
1368        // Commit v1 and v2 to disk.
1369        let batch = db.new_batch();
1370        let loc1 = batch.size();
1371        let batch = batch.append(v1.clone());
1372        let loc2 = batch.size();
1373        let batch = batch.append(v2.clone());
1374        let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1375        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1376        let db = db.commit().await.unwrap();
1377
1378        // DB-level get_many.
1379        let results = db.get_many(&[loc1, loc2]).await.unwrap();
1380        assert_eq!(results, vec![Some(v1.clone()), Some(v2.clone())]);
1381
1382        // Empty input.
1383        let results = db.get_many(&[]).await.unwrap();
1384        assert!(results.is_empty());
1385
1386        // Unmerkleized batch: pending appends + DB fallthrough.
1387        let batch = db.new_batch();
1388        let loc3 = batch.size();
1389        let batch = batch.append(v3.clone());
1390        let results = batch.get_many(&[loc1, loc3], &db).await.unwrap();
1391        assert_eq!(results, vec![Some(v1.clone()), Some(v3.clone())]);
1392
1393        // Merkleized batch: parent chain + DB fallthrough.
1394        let parent = db
1395            .new_batch()
1396            .append(v3.clone())
1397            .merkleize(&db, None, db.inactivity_floor_loc())
1398            .await;
1399        let child = parent.new_batch::<Sha256>().append(V::Value::make(4));
1400        let results = child.get_many(&[loc1, loc2], &db).await.unwrap();
1401        assert_eq!(results, vec![Some(v1.clone()), Some(v2.clone())]);
1402
1403        db.destroy().await.unwrap();
1404    }
1405
1406    #[boxed]
1407    pub(crate) async fn run_batch_chained<F: Family, V, C, S: Strategy>(
1408        db: TestKeyless<F, V, C, Sha256, S>,
1409    ) where
1410        V: ValueEncoding<Value: TestValue>,
1411        C: Mutable<Item = Operation<F, V>>,
1412        Operation<F, V>: EncodeShared,
1413    {
1414        let v1 = V::Value::make(10);
1415        let v2 = V::Value::make(20);
1416        let v3 = V::Value::make(30);
1417
1418        let parent = db.new_batch();
1419        let loc1 = parent.size();
1420        let parent = parent.append(v1.clone());
1421        let parent_m = parent.merkleize(&db, None, db.inactivity_floor_loc()).await;
1422
1423        let child = parent_m.new_batch::<Sha256>();
1424        let loc2 = child.size();
1425        let child = child.append(v2.clone());
1426        let loc3 = child.size();
1427        let child = child.append(v3.clone());
1428        let child_m = child.merkleize(&db, None, db.inactivity_floor_loc()).await;
1429        let child_root = child_m.root();
1430
1431        let (db, _) = db.apply_batch(child_m).await.unwrap();
1432        let db = db.commit().await.unwrap();
1433
1434        assert_eq!(db.root(), child_root);
1435        assert_eq!(db.get(loc1).await.unwrap(), Some(v1));
1436        assert_eq!(db.get(loc2).await.unwrap(), Some(v2));
1437        assert_eq!(db.get(loc3).await.unwrap(), Some(v3));
1438
1439        db.destroy().await.unwrap();
1440    }
1441
1442    #[boxed]
1443    pub(crate) async fn run_stale_batch<F: Family, V, C, H, S: Strategy>(
1444        context: deterministic::Context,
1445        db: TestKeyless<F, V, C, H, S>,
1446        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
1447    ) where
1448        V: ValueEncoding<Value: TestValue>,
1449        C: Mutable<Item = Operation<F, V>>,
1450        H: Hasher,
1451        Operation<F, V>: EncodeShared,
1452    {
1453        let batch_a = db
1454            .new_batch()
1455            .append(V::Value::make(10))
1456            .merkleize(&db, None, db.inactivity_floor_loc())
1457            .await;
1458        let batch_b = db
1459            .new_batch()
1460            .append(V::Value::make(20))
1461            .merkleize(&db, None, db.inactivity_floor_loc())
1462            .await;
1463
1464        let (db, _) = db.apply_batch(batch_a).await.unwrap();
1465        let db = db.commit().await.unwrap();
1466        let root = db.root();
1467        let last_commit_loc = db.last_commit_loc();
1468
1469        let result = db.apply_batch(batch_b).await;
1470        assert!(matches!(result, Err(Error::StaleBatch)));
1471
1472        // The rejection mutated nothing: reopening recovers the committed state.
1473        let db = reopen(context.child("reopen")).await;
1474        assert_eq!(db.root(), root);
1475        assert_eq!(db.last_commit_loc(), last_commit_loc);
1476        db.destroy().await.unwrap();
1477    }
1478
1479    #[boxed]
1480    pub(crate) async fn run_partial_ancestor_commit<F: Family, V, C, H, S: Strategy>(
1481        db: TestKeyless<F, V, C, H, S>,
1482    ) where
1483        V: ValueEncoding<Value: TestValue>,
1484        C: Mutable<Item = Operation<F, V>>,
1485        H: Hasher,
1486        Operation<F, V>: EncodeShared,
1487    {
1488        // Chain: DB <- A <- B <- C
1489        let a = db
1490            .new_batch()
1491            .append(V::Value::make(10))
1492            .merkleize(&db, None, db.inactivity_floor_loc())
1493            .await;
1494        let b = a
1495            .new_batch::<H>()
1496            .append(V::Value::make(20))
1497            .merkleize(&db, None, db.inactivity_floor_loc())
1498            .await;
1499        let c = b
1500            .new_batch::<H>()
1501            .append(V::Value::make(30))
1502            .merkleize(&db, None, db.inactivity_floor_loc())
1503            .await;
1504
1505        let expected_root = c.root();
1506
1507        // Apply only A, then apply C directly (B's items applied via ancestor batches).
1508        let (db, _) = db.apply_batch(a).await.unwrap();
1509        let (db, _) = db.apply_batch(c).await.unwrap();
1510
1511        // Root must match what the full chain produces.
1512        assert_eq!(db.root(), expected_root);
1513
1514        db.destroy().await.unwrap();
1515    }
1516
1517    #[boxed]
1518    pub(crate) async fn run_delayed_merkleize_after_ancestor_apply<
1519        F: Family,
1520        V,
1521        C,
1522        H,
1523        S: Strategy,
1524    >(
1525        db: TestKeyless<F, V, C, H, S>,
1526    ) where
1527        V: ValueEncoding<Value: TestValue>,
1528        C: Mutable<Item = Operation<F, V>>,
1529        H: Hasher,
1530        Operation<F, V>: EncodeShared,
1531    {
1532        let floor = db.inactivity_floor_loc();
1533        let a = db
1534            .new_batch()
1535            .append(V::Value::make(10))
1536            .merkleize(&db, None, floor)
1537            .await;
1538        let b = a
1539            .new_batch::<H>()
1540            .append(V::Value::make(20))
1541            .merkleize(&db, None, floor)
1542            .await;
1543        let c = b.new_batch::<H>().append(V::Value::make(30));
1544
1545        let (db, _) = db.apply_batch(a).await.unwrap();
1546        let c = c.merkleize(&db, None, floor).await;
1547        let expected_root = c.root();
1548        let (db, _) = db.apply_batch(c).await.unwrap();
1549
1550        assert_eq!(db.root(), expected_root);
1551        db.destroy().await.unwrap();
1552    }
1553
1554    #[boxed]
1555    pub(crate) async fn run_to_batch<F: Family, V, C, S: Strategy>(
1556        db: TestKeyless<F, V, C, Sha256, S>,
1557    ) where
1558        V: ValueEncoding<Value: TestValue>,
1559        C: Mutable<Item = Operation<F, V>>,
1560        Operation<F, V>: EncodeShared,
1561    {
1562        let batch = db.new_batch();
1563        let loc1 = batch.size();
1564        let batch = batch.append(V::Value::make(10));
1565        let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1566        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1567
1568        let snapshot = db.to_batch();
1569        assert_eq!(snapshot.root(), db.root());
1570
1571        let child_batch = snapshot.new_batch::<Sha256>();
1572        let loc2 = child_batch.size();
1573        let child_batch = child_batch.append(V::Value::make(20));
1574        let merkleized = child_batch
1575            .merkleize(&db, None, db.inactivity_floor_loc())
1576            .await;
1577        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1578
1579        assert_eq!(db.get(loc1).await.unwrap(), Some(V::Value::make(10)));
1580        assert_eq!(db.get(loc2).await.unwrap(), Some(V::Value::make(20)));
1581
1582        db.destroy().await.unwrap();
1583    }
1584
1585    #[boxed]
1586    pub(crate) async fn run_non_empty_recovery<F: Family, V, C, H, S: Strategy>(
1587        context: deterministic::Context,
1588        mut db: TestKeyless<F, V, C, H, S>,
1589        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
1590    ) where
1591        V: ValueEncoding<Value: TestValue>,
1592        C: Mutable<Item = Operation<F, V>>,
1593        H: Hasher,
1594        Operation<F, V>: EncodeShared,
1595    {
1596        // Append many values then commit, advancing the floor to the new commit so we can
1597        // later prune up to it.
1598        const ELEMENTS: u64 = 200;
1599        {
1600            let mut batch = db.new_batch();
1601            for i in 0..ELEMENTS {
1602                batch = batch.append(V::Value::make(i));
1603            }
1604            let new_commit = db.last_commit_loc() + 1 + ELEMENTS;
1605            let merkleized = batch.merkleize(&db, None, new_commit).await;
1606            (db, _) = db.apply_batch(merkleized).await.unwrap();
1607        }
1608        let db = db.commit().await.unwrap();
1609        let root = db.root();
1610        let op_count = db.bounds().end;
1611
1612        // Reopen DB without clean shutdown and make sure the state is the same.
1613        let db = reopen(context.child("db").with_attribute("index", 2)).await;
1614        assert_eq!(db.bounds().end, op_count);
1615        assert_eq!(db.root(), root);
1616        assert_eq!(db.last_commit_loc(), op_count - 1);
1617        drop(db);
1618
1619        // Insert many operations without commit, then simulate failure.
1620        let db = reopen(context.child("recovery_a")).await;
1621        {
1622            let mut batch = db.new_batch();
1623            for i in 0..ELEMENTS {
1624                batch = batch.append(V::Value::make(i + 1000));
1625            }
1626            // Don't merkleize/apply -- simulate failed commit
1627        }
1628        drop(db);
1629        let db = reopen(context.child("recovery_b")).await;
1630        assert_eq!(db.bounds().end, op_count);
1631        assert_eq!(db.root(), root);
1632        drop(db);
1633
1634        // Repeat after pruning to the last commit.
1635        let db = reopen(context.child("db").with_attribute("index", 3)).await;
1636        let last_commit = db.last_commit_loc();
1637        let db = db.prune(last_commit).await.unwrap();
1638        assert_eq!(db.bounds().end, op_count);
1639        assert_eq!(db.root(), root);
1640        db.sync().await.unwrap();
1641
1642        let db = reopen(context.child("recovery_c")).await;
1643        {
1644            let mut batch = db.new_batch();
1645            for i in 0..ELEMENTS {
1646                batch = batch.append(V::Value::make(i + 2000));
1647            }
1648        }
1649        drop(db);
1650        let db = reopen(context.child("recovery_d")).await;
1651        assert_eq!(db.bounds().end, op_count);
1652        assert_eq!(db.root(), root);
1653        drop(db);
1654
1655        // Apply the ops one last time but fully commit them this time, then clean up.
1656        let mut db = reopen(context.child("db").with_attribute("index", 4)).await;
1657        {
1658            let mut batch = db.new_batch();
1659            for i in 0..ELEMENTS {
1660                batch = batch.append(V::Value::make(i + 3000));
1661            }
1662            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1663            (db, _) = db.apply_batch(merkleized).await.unwrap();
1664        }
1665        db.commit().await.unwrap();
1666        let db = reopen(context.child("db").with_attribute("index", 5)).await;
1667        let bounds = db.bounds();
1668        assert!(bounds.end > op_count);
1669        assert_ne!(db.root(), root);
1670        assert_eq!(db.last_commit_loc(), bounds.end - 1);
1671
1672        db.destroy().await.unwrap();
1673    }
1674
1675    #[boxed]
1676    pub(crate) async fn run_proof_comprehensive<F: Family, V, C, S: Strategy>(
1677        mut db: TestKeyless<F, V, C, Sha256, S>,
1678    ) where
1679        V: ValueEncoding<Value: TestValue>,
1680        C: Mutable<Item = Operation<F, V>>,
1681        Operation<F, V>: EncodeShared + std::fmt::Debug,
1682    {
1683        // Build a db with some values.
1684        const ELEMENTS: u64 = 100;
1685        {
1686            let mut batch = db.new_batch();
1687            for i in 0u64..ELEMENTS {
1688                batch = batch.append(V::Value::make(i));
1689            }
1690            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1691            (db, _) = db.apply_batch(merkleized).await.unwrap();
1692        }
1693
1694        // Test that historical proof fails with op_count > number of operations.
1695        assert!(matches!(
1696            db.historical_proof(db.bounds().end + 1, Location::new(5), NZU64!(10))
1697                .await,
1698            Err(Error::<F>::Merkle(crate::merkle::Error::RangeOutOfBounds(
1699                _
1700            )))
1701        ));
1702
1703        let root = db.root();
1704
1705        for (start_loc, max_ops) in [
1706            (0, 10),
1707            (10, 5),
1708            (50, 20),
1709            (90, 15),
1710            (0, 1),
1711            (ELEMENTS - 1, 1),
1712            (ELEMENTS, 1),
1713        ] {
1714            let (proof, ops) = db
1715                .proof(Location::new(start_loc), NZU64!(max_ops))
1716                .await
1717                .unwrap();
1718            assert!(
1719                verify_proof::<Sha256, _, _>(&proof, Location::new(start_loc), &ops, &root,),
1720                "Failed to verify proof for range starting at {start_loc} with max {max_ops} ops",
1721            );
1722            let expected_ops = std::cmp::min(max_ops, *db.bounds().end - start_loc);
1723            assert_eq!(ops.len() as u64, expected_ops);
1724
1725            let wrong_root = Sha256::hash(&[&[0xFF; 32]]);
1726            assert!(!verify_proof::<Sha256, _, _>(
1727                &proof,
1728                Location::new(start_loc),
1729                &ops,
1730                &wrong_root,
1731            ));
1732            if start_loc > 0 {
1733                assert!(!verify_proof::<Sha256, _, _>(
1734                    &proof,
1735                    Location::new(start_loc - 1),
1736                    &ops,
1737                    &root,
1738                ));
1739            }
1740        }
1741
1742        db.destroy().await.unwrap();
1743    }
1744
1745    #[boxed]
1746    pub(crate) async fn run_proof_with_pruning<F: Family, V, C, S: Strategy>(
1747        context: deterministic::Context,
1748        mut db: TestKeyless<F, V, C, Sha256, S>,
1749        reopen: Reopen<TestKeyless<F, V, C, Sha256, S>>,
1750    ) where
1751        V: ValueEncoding<Value: TestValue>,
1752        C: Mutable<Item = Operation<F, V>>,
1753        Operation<F, V>: EncodeShared + std::fmt::Debug,
1754    {
1755        const ELEMENTS: u64 = 100;
1756        {
1757            let mut batch = db.new_batch();
1758            for i in 0u64..ELEMENTS {
1759                batch = batch.append(V::Value::make(i));
1760            }
1761            let new_commit = db.last_commit_loc() + 1 + ELEMENTS;
1762            let merkleized = batch.merkleize(&db, None, new_commit).await;
1763            (db, _) = db.apply_batch(merkleized).await.unwrap();
1764        }
1765
1766        {
1767            let mut batch = db.new_batch();
1768            for i in ELEMENTS..ELEMENTS * 2 {
1769                batch = batch.append(V::Value::make(i));
1770            }
1771            let new_commit = db.last_commit_loc() + 1 + ELEMENTS;
1772            let merkleized = batch.merkleize(&db, None, new_commit).await;
1773            (db, _) = db.apply_batch(merkleized).await.unwrap();
1774        }
1775        let root = db.root();
1776
1777        const PRUNE_LOC: u64 = 30;
1778        let db = db.prune(Location::new(PRUNE_LOC)).await.unwrap();
1779        let oldest_retained = db.bounds().start;
1780        assert_eq!(db.root(), root);
1781
1782        db.sync().await.unwrap();
1783        let db = reopen(context).await;
1784        assert_eq!(db.root(), root);
1785
1786        for (start_loc, max_ops) in [
1787            (oldest_retained, 10),
1788            (Location::new(50), 20),
1789            (Location::new(150), 10),
1790            (Location::new(190), 15),
1791        ] {
1792            if start_loc < oldest_retained {
1793                continue;
1794            }
1795            let (proof, ops) = db.proof(start_loc, NZU64!(max_ops)).await.unwrap();
1796            assert!(verify_proof::<Sha256, _, _>(&proof, start_loc, &ops, &root,));
1797        }
1798
1799        let aggressive_prune: Location<F> = Location::new(150);
1800        let db = db.prune(aggressive_prune).await.unwrap();
1801
1802        let new_oldest = db.bounds().start;
1803        let (proof, ops) = db.proof(new_oldest, NZU64!(20)).await.unwrap();
1804        assert!(verify_proof::<Sha256, _, _>(
1805            &proof, new_oldest, &ops, &root,
1806        ));
1807
1808        let almost_all = db.bounds().end - 5;
1809        let db = db.prune(almost_all).await.unwrap();
1810        let final_oldest = db.bounds().start;
1811        if final_oldest < db.bounds().end {
1812            let (final_proof, final_ops) = db.proof(final_oldest, NZU64!(10)).await.unwrap();
1813            assert!(verify_proof::<Sha256, _, _>(
1814                &final_proof,
1815                final_oldest,
1816                &final_ops,
1817                &root,
1818            ));
1819        }
1820
1821        db.destroy().await.unwrap();
1822    }
1823
1824    #[boxed]
1825    pub(crate) async fn run_get_out_of_bounds<F: Family, V, C, H, S: Strategy>(
1826        db: TestKeyless<F, V, C, H, S>,
1827    ) where
1828        V: ValueEncoding<Value: TestValue>,
1829        C: Mutable<Item = Operation<F, V>>,
1830        H: Hasher,
1831        Operation<F, V>: EncodeShared,
1832    {
1833        assert!(db.get(Location::new(0)).await.unwrap().is_none());
1834
1835        let merkleized = db
1836            .new_batch()
1837            .append(V::Value::make(1))
1838            .append(V::Value::make(2))
1839            .merkleize(&db, None, db.inactivity_floor_loc())
1840            .await;
1841        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1842
1843        assert_eq!(
1844            db.get(Location::new(1)).await.unwrap(),
1845            Some(V::Value::make(1))
1846        );
1847        assert!(db.get(Location::new(3)).await.unwrap().is_none());
1848        assert!(matches!(
1849            db.get(Location::new(4)).await,
1850            Err(Error::LocationOutOfBounds(loc, size))
1851                if loc == Location::new(4) && size == Location::new(4)
1852        ));
1853
1854        db.destroy().await.unwrap();
1855    }
1856
1857    #[boxed]
1858    pub(crate) async fn run_batch_get<F: Family, V, C, H, S: Strategy>(
1859        mut db: TestKeyless<F, V, C, H, S>,
1860    ) where
1861        V: ValueEncoding<Value: TestValue>,
1862        C: Mutable<Item = Operation<F, V>>,
1863        H: Hasher,
1864        Operation<F, V>: EncodeShared,
1865    {
1866        let base_vals: Vec<V::Value> = (0..3).map(|i| V::Value::make(10 + i)).collect();
1867        let mut base_locs = Vec::new();
1868        {
1869            let mut batch = db.new_batch();
1870            for v in &base_vals {
1871                let loc = batch.size();
1872                batch = batch.append(v.clone());
1873                base_locs.push(loc);
1874            }
1875            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1876            (db, _) = db.apply_batch(merkleized).await.unwrap();
1877        }
1878
1879        let batch = db.new_batch();
1880        for (i, loc) in base_locs.iter().enumerate() {
1881            assert_eq!(
1882                batch.get(*loc, &db).await.unwrap(),
1883                Some(base_vals[i].clone()),
1884            );
1885        }
1886
1887        let new_val = V::Value::make(99);
1888        let new_loc = batch.size();
1889        let batch = batch.append(new_val.clone());
1890        assert_eq!(batch.get(new_loc, &db).await.unwrap(), Some(new_val));
1891        assert_eq!(batch.get(new_loc + 1, &db).await.unwrap(), None);
1892
1893        db.destroy().await.unwrap();
1894    }
1895
1896    #[boxed]
1897    pub(crate) async fn run_batch_stacked_get<F: Family, V, C, S: Strategy>(
1898        db: TestKeyless<F, V, C, Sha256, S>,
1899    ) where
1900        V: ValueEncoding<Value: TestValue>,
1901        C: Mutable<Item = Operation<F, V>>,
1902        Operation<F, V>: EncodeShared,
1903    {
1904        let v1 = V::Value::make(1);
1905        let v2 = V::Value::make(2);
1906
1907        let parent = db.new_batch();
1908        let loc1 = parent.size();
1909        let parent = parent.append(v1.clone());
1910        let parent_m = parent.merkleize(&db, None, db.inactivity_floor_loc()).await;
1911
1912        let child = parent_m.new_batch::<Sha256>();
1913        assert_eq!(child.get(loc1, &db).await.unwrap(), Some(v1));
1914
1915        let loc2 = child.size();
1916        let child = child.append(v2.clone());
1917        assert_eq!(child.get(loc2, &db).await.unwrap(), Some(v2));
1918        assert_eq!(child.get(Location::new(9999), &db).await.unwrap(), None);
1919
1920        db.destroy().await.unwrap();
1921    }
1922
1923    #[boxed]
1924    pub(crate) async fn run_batch_speculative_root<F: Family, V, C, H, S: Strategy>(
1925        db: TestKeyless<F, V, C, H, S>,
1926    ) where
1927        V: ValueEncoding<Value: TestValue>,
1928        C: Mutable<Item = Operation<F, V>>,
1929        H: Hasher,
1930        Operation<F, V>: EncodeShared,
1931    {
1932        let mut batch = db.new_batch();
1933        for i in 0u64..10 {
1934            batch = batch.append(V::Value::make(i));
1935        }
1936        let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
1937        let speculative = merkleized.root();
1938        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1939        assert_eq!(db.root(), speculative);
1940
1941        let merkleized = db
1942            .new_batch()
1943            .append(V::Value::make(100))
1944            .merkleize(&db, Some(V::Value::make(55)), db.inactivity_floor_loc())
1945            .await;
1946        let speculative = merkleized.root();
1947        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1948        assert_eq!(db.root(), speculative);
1949
1950        db.destroy().await.unwrap();
1951    }
1952
1953    #[boxed]
1954    pub(crate) async fn run_merkleized_batch_get<F: Family, V, C, S: Strategy>(
1955        db: TestKeyless<F, V, C, Sha256, S>,
1956    ) where
1957        V: ValueEncoding<Value: TestValue>,
1958        C: Mutable<Item = Operation<F, V>>,
1959        Operation<F, V>: EncodeShared,
1960    {
1961        let base_val = V::Value::make(10);
1962        let merkleized = db
1963            .new_batch()
1964            .append(base_val.clone())
1965            .merkleize(&db, None, db.inactivity_floor_loc())
1966            .await;
1967        let (db, _) = db.apply_batch(merkleized).await.unwrap();
1968
1969        let new_val = V::Value::make(20);
1970        let merkleized = db
1971            .new_batch()
1972            .append(new_val.clone())
1973            .merkleize(&db, None, db.inactivity_floor_loc())
1974            .await;
1975
1976        assert_eq!(
1977            merkleized.get(Location::new(1), &db).await.unwrap(),
1978            Some(base_val),
1979        );
1980        assert_eq!(
1981            merkleized.get(Location::new(3), &db).await.unwrap(),
1982            Some(new_val),
1983        );
1984        assert_eq!(merkleized.get(Location::new(4), &db).await.unwrap(), None);
1985
1986        db.destroy().await.unwrap();
1987    }
1988
1989    #[boxed]
1990    pub(crate) async fn run_batch_chained_apply_sequential<F: Family, V, C, H, S: Strategy>(
1991        db: TestKeyless<F, V, C, H, S>,
1992    ) where
1993        V: ValueEncoding<Value: TestValue>,
1994        C: Mutable<Item = Operation<F, V>>,
1995        H: Hasher,
1996        Operation<F, V>: EncodeShared,
1997    {
1998        let v1 = V::Value::make(1);
1999        let v2 = V::Value::make(2);
2000
2001        let parent = db.new_batch();
2002        let loc1 = parent.size();
2003        let parent = parent.append(v1.clone());
2004        let parent_m = parent.merkleize(&db, None, db.inactivity_floor_loc()).await;
2005        let parent_root = parent_m.root();
2006
2007        let (db, _) = db.apply_batch(parent_m).await.unwrap();
2008        assert_eq!(db.root(), parent_root);
2009        assert_eq!(db.get(loc1).await.unwrap(), Some(v1));
2010
2011        let batch2 = db.new_batch();
2012        let loc2 = batch2.size();
2013        let batch2 = batch2.append(v2.clone());
2014        let batch2_m = batch2.merkleize(&db, None, db.inactivity_floor_loc()).await;
2015        let batch2_root = batch2_m.root();
2016        let (db, _) = db.apply_batch(batch2_m).await.unwrap();
2017        assert_eq!(db.root(), batch2_root);
2018        assert_eq!(db.get(loc2).await.unwrap(), Some(v2));
2019
2020        db.destroy().await.unwrap();
2021    }
2022
2023    #[boxed]
2024    pub(crate) async fn run_batch_many_sequential<F: Family, V, C, S: Strategy>(
2025        mut db: TestKeyless<F, V, C, Sha256, S>,
2026    ) where
2027        V: ValueEncoding<Value: TestValue>,
2028        C: Mutable<Item = Operation<F, V>>,
2029        Operation<F, V>: EncodeShared + std::fmt::Debug,
2030    {
2031        const BATCHES: u64 = 20;
2032        const APPENDS_PER_BATCH: u64 = 5;
2033        let mut all_values: Vec<V::Value> = Vec::new();
2034        let mut all_locs: Vec<Location<F>> = Vec::new();
2035
2036        for batch_idx in 0..BATCHES {
2037            let mut batch = db.new_batch();
2038            for j in 0..APPENDS_PER_BATCH {
2039                let v = V::Value::make(batch_idx * 10 + j);
2040                let loc = batch.size();
2041                batch = batch.append(v.clone());
2042                all_values.push(v);
2043                all_locs.push(loc);
2044            }
2045            let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
2046            (db, _) = db.apply_batch(merkleized).await.unwrap();
2047        }
2048
2049        for (i, loc) in all_locs.iter().enumerate() {
2050            assert_eq!(db.get(*loc).await.unwrap(), Some(all_values[i].clone()));
2051        }
2052
2053        let root = db.root();
2054        let (proof, ops) = db.proof(Location::new(0), NZU64!(1000)).await.unwrap();
2055        assert!(verify_proof::<Sha256, _, _>(
2056            &proof,
2057            Location::new(0),
2058            &ops,
2059            &root,
2060        ));
2061        assert_eq!(db.bounds().end, 1 + BATCHES * (APPENDS_PER_BATCH + 1));
2062
2063        db.destroy().await.unwrap();
2064    }
2065
2066    #[boxed]
2067    pub(crate) async fn run_batch_empty<F: Family, V, C, H, S: Strategy>(
2068        db: TestKeyless<F, V, C, H, S>,
2069    ) where
2070        V: ValueEncoding<Value: TestValue>,
2071        C: Mutable<Item = Operation<F, V>>,
2072        H: Hasher,
2073        Operation<F, V>: EncodeShared,
2074    {
2075        let merkleized = db
2076            .new_batch()
2077            .append(V::Value::make(1))
2078            .merkleize(&db, None, db.inactivity_floor_loc())
2079            .await;
2080        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2081        let root_before = db.root();
2082        let size_before = db.bounds().end;
2083
2084        let merkleized = db
2085            .new_batch()
2086            .merkleize(&db, None, db.inactivity_floor_loc())
2087            .await;
2088        let speculative = merkleized.root();
2089        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2090
2091        assert_ne!(db.root(), root_before);
2092        assert_eq!(db.root(), speculative);
2093        assert_eq!(db.bounds().end, size_before + 1);
2094
2095        db.destroy().await.unwrap();
2096    }
2097
2098    #[boxed]
2099    pub(crate) async fn run_batch_chained_merkleized_get<F: Family, V, C, S: Strategy>(
2100        db: TestKeyless<F, V, C, Sha256, S>,
2101    ) where
2102        V: ValueEncoding<Value: TestValue>,
2103        C: Mutable<Item = Operation<F, V>>,
2104        Operation<F, V>: EncodeShared,
2105    {
2106        let base_val = V::Value::make(10);
2107        let floor = db.inactivity_floor_loc();
2108        let merkleized = db
2109            .new_batch()
2110            .append(base_val.clone())
2111            .merkleize(&db, None, floor)
2112            .await;
2113        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2114
2115        let v1 = V::Value::make(1);
2116        let parent = db.new_batch();
2117        let loc1 = parent.size();
2118        let parent_m = parent
2119            .append(v1.clone())
2120            .merkleize(&db, None, db.inactivity_floor_loc())
2121            .await;
2122
2123        let v2 = V::Value::make(2);
2124        let child = parent_m.new_batch::<Sha256>();
2125        let loc2 = child.size();
2126        let child_m = child
2127            .append(v2.clone())
2128            .merkleize(&db, None, db.inactivity_floor_loc())
2129            .await;
2130
2131        assert_eq!(
2132            child_m.get(Location::new(1), &db).await.unwrap(),
2133            Some(base_val),
2134        );
2135        assert_eq!(child_m.get(loc1, &db).await.unwrap(), Some(v1));
2136        assert_eq!(child_m.get(loc2, &db).await.unwrap(), Some(v2));
2137
2138        db.destroy().await.unwrap();
2139    }
2140
2141    #[boxed]
2142    pub(crate) async fn run_batch_large<F: Family, V, C, S: Strategy>(
2143        db: TestKeyless<F, V, C, Sha256, S>,
2144    ) where
2145        V: ValueEncoding<Value: TestValue>,
2146        C: Mutable<Item = Operation<F, V>>,
2147        Operation<F, V>: EncodeShared + std::fmt::Debug,
2148    {
2149        const N: u64 = 500;
2150        let mut values = Vec::new();
2151        let mut locs = Vec::new();
2152
2153        let mut batch = db.new_batch();
2154        for i in 0..N {
2155            let v = V::Value::make(i);
2156            locs.push(batch.size());
2157            batch = batch.append(v.clone());
2158            values.push(v);
2159        }
2160        let merkleized = batch.merkleize(&db, None, db.inactivity_floor_loc()).await;
2161        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2162
2163        for (i, loc) in locs.iter().enumerate() {
2164            assert_eq!(db.get(*loc).await.unwrap(), Some(values[i].clone()));
2165        }
2166
2167        let root = db.root();
2168        let (proof, ops) = db.proof(Location::new(0), NZU64!(1000)).await.unwrap();
2169        assert!(verify_proof::<Sha256, _, _>(
2170            &proof,
2171            Location::new(0),
2172            &ops,
2173            &root,
2174        ));
2175        assert_eq!(db.bounds().end, 1 + N + 1);
2176
2177        db.destroy().await.unwrap();
2178    }
2179
2180    #[boxed]
2181    pub(crate) async fn run_stale_batch_chained<F: Family, V, C, S: Strategy>(
2182        db: TestKeyless<F, V, C, Sha256, S>,
2183    ) where
2184        V: ValueEncoding<Value: TestValue>,
2185        C: Mutable<Item = Operation<F, V>>,
2186        Operation<F, V>: EncodeShared,
2187    {
2188        let common_parent = db
2189            .new_batch()
2190            .append(V::Value::make(10))
2191            .merkleize(&db, None, db.inactivity_floor_loc())
2192            .await;
2193        let sibling_a = common_parent
2194            .new_batch::<Sha256>()
2195            .append(V::Value::make(11))
2196            .merkleize(&db, None, db.inactivity_floor_loc())
2197            .await;
2198        let sibling_b = common_parent
2199            .new_batch::<Sha256>()
2200            .append(V::Value::make(12))
2201            .merkleize(&db, None, db.inactivity_floor_loc())
2202            .await;
2203        let (db, _) = db.apply_batch(sibling_a).await.unwrap();
2204        assert!(matches!(
2205            db.validate_batch(&sibling_b),
2206            Err(Error::StaleBatch)
2207        ));
2208
2209        let parent_a = db
2210            .new_batch()
2211            .append(V::Value::make(1))
2212            .merkleize(&db, None, db.inactivity_floor_loc())
2213            .await;
2214        let parent_b = db
2215            .new_batch()
2216            .append(V::Value::make(2))
2217            .merkleize(&db, None, db.inactivity_floor_loc())
2218            .await;
2219        let child_b = parent_b
2220            .new_batch::<Sha256>()
2221            .append(V::Value::make(3))
2222            .merkleize(&db, None, db.inactivity_floor_loc())
2223            .await;
2224
2225        let (db, _) = db.apply_batch(parent_a).await.unwrap();
2226        assert!(matches!(
2227            db.validate_batch(&child_b),
2228            Err(Error::StaleBatch)
2229        ));
2230        db.destroy().await.unwrap();
2231    }
2232
2233    #[boxed]
2234    pub(crate) async fn run_sequential_commit_parent_then_child<F: Family, V, C, S: Strategy>(
2235        db: TestKeyless<F, V, C, Sha256, S>,
2236    ) where
2237        V: ValueEncoding<Value: TestValue>,
2238        C: Mutable<Item = Operation<F, V>>,
2239        Operation<F, V>: EncodeShared,
2240    {
2241        let parent = db
2242            .new_batch()
2243            .append(V::Value::make(1))
2244            .merkleize(&db, None, db.inactivity_floor_loc())
2245            .await;
2246        let child = parent
2247            .new_batch::<Sha256>()
2248            .append(V::Value::make(2))
2249            .merkleize(&db, None, db.inactivity_floor_loc())
2250            .await;
2251
2252        let (db, _) = db.apply_batch(parent).await.unwrap();
2253        let (db, _) = db.apply_batch(child).await.unwrap();
2254
2255        db.destroy().await.unwrap();
2256    }
2257
2258    #[boxed]
2259    pub(crate) async fn run_stale_batch_child_before_parent<F: Family, V, C, S: Strategy>(
2260        db: TestKeyless<F, V, C, Sha256, S>,
2261    ) where
2262        V: ValueEncoding<Value: TestValue>,
2263        C: Mutable<Item = Operation<F, V>>,
2264        Operation<F, V>: EncodeShared,
2265    {
2266        let parent = db
2267            .new_batch()
2268            .append(V::Value::make(1))
2269            .merkleize(&db, None, db.inactivity_floor_loc())
2270            .await;
2271        let child = parent
2272            .new_batch::<Sha256>()
2273            .append(V::Value::make(2))
2274            .merkleize(&db, None, db.inactivity_floor_loc())
2275            .await;
2276
2277        let (db, _) = db.apply_batch(child).await.unwrap();
2278        assert!(matches!(
2279            db.apply_batch(parent).await,
2280            Err(Error::StaleBatch)
2281        ));
2282    }
2283
2284    #[boxed]
2285    pub(crate) async fn run_child_root_matches_pending_and_committed<F: Family, V, C, S: Strategy>(
2286        db: TestKeyless<F, V, C, Sha256, S>,
2287    ) where
2288        V: ValueEncoding<Value: TestValue>,
2289        C: Mutable<Item = Operation<F, V>>,
2290        Operation<F, V>: EncodeShared,
2291    {
2292        // Build the child while the parent is still pending.
2293        let parent = db
2294            .new_batch()
2295            .append(V::Value::make(1))
2296            .merkleize(&db, None, db.inactivity_floor_loc())
2297            .await;
2298        let pending_child = parent
2299            .new_batch::<Sha256>()
2300            .append(V::Value::make(2))
2301            .merkleize(&db, None, db.inactivity_floor_loc())
2302            .await;
2303
2304        // Commit the parent, then rebuild the same logical child from the
2305        // committed DB state and compare roots.
2306        let (db, _) = db.apply_batch(parent).await.unwrap();
2307        let db = db.commit().await.unwrap();
2308
2309        let committed_child = db
2310            .new_batch()
2311            .append(V::Value::make(2))
2312            .merkleize(&db, None, db.inactivity_floor_loc())
2313            .await;
2314
2315        assert_eq!(pending_child.root(), committed_child.root());
2316
2317        db.destroy().await.unwrap();
2318    }
2319
2320    async fn commit_appends<F: Family, V, C, H, S: Strategy>(
2321        db: TestKeyless<F, V, C, H, S>,
2322        values: impl IntoIterator<Item = V::Value>,
2323        metadata: Option<V::Value>,
2324    ) -> (TestKeyless<F, V, C, H, S>, core::ops::Range<Location<F>>)
2325    where
2326        V: ValueEncoding<Value: TestValue>,
2327        C: Mutable<Item = Operation<F, V>>,
2328        H: Hasher,
2329        Operation<F, V>: EncodeShared,
2330    {
2331        // Tests that don't specifically exercise floor behavior advance the floor to the new
2332        // commit location, so pruning up to the last commit works analogously to the pre-floor
2333        // semantics.
2334        let base_size = *db.last_commit_loc() + 1;
2335        let appends_iter: Vec<_> = values.into_iter().collect();
2336        let new_commit_loc = Location::new(base_size + appends_iter.len() as u64);
2337        let mut batch = db.new_batch();
2338        for value in appends_iter {
2339            batch = batch.append(value);
2340        }
2341        let merkleized = batch.merkleize(&db, metadata, new_commit_loc).await;
2342        let (db, range) = db.apply_batch(merkleized).await.unwrap();
2343        let db = db.commit().await.unwrap();
2344        (db, range)
2345    }
2346
2347    #[boxed]
2348    pub(crate) async fn run_rewind_recovery<F: Family, V, C, H, S: Strategy>(
2349        context: deterministic::Context,
2350        db: TestKeyless<F, V, C, H, S>,
2351        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2352    ) where
2353        V: ValueEncoding<Value: TestValue>,
2354        C: Mutable<Item = Operation<F, V>>,
2355        H: Hasher,
2356        Operation<F, V>: EncodeShared,
2357    {
2358        let initial_root = db.root();
2359        let initial_size = db.bounds().end;
2360
2361        let value_a = V::Value::make(1);
2362        let value_b = V::Value::make(2);
2363        let metadata_a = V::Value::make(3);
2364        let (db, first_range) = commit_appends(
2365            db,
2366            [value_a.clone(), value_b.clone()],
2367            Some(metadata_a.clone()),
2368        )
2369        .await;
2370
2371        let root_before = db.root();
2372        let size_before = db.bounds().end;
2373        let commit_before = db.last_commit_loc();
2374        assert_eq!(size_before, first_range.end);
2375
2376        let value_c = V::Value::make(4);
2377        let metadata_b = V::Value::make(5);
2378        let (db, second_range) =
2379            commit_appends(db, [value_c.clone()], Some(metadata_b.clone())).await;
2380        assert_eq!(second_range.start, size_before);
2381        assert_ne!(db.root(), root_before);
2382        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata_b));
2383
2384        let db = db.rewind(size_before).await.unwrap();
2385        assert_eq!(db.root(), root_before);
2386        assert_eq!(db.bounds().end, size_before);
2387        assert_eq!(db.last_commit_loc(), commit_before);
2388        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata_a.clone()));
2389        assert_eq!(
2390            db.get(Location::new(1)).await.unwrap(),
2391            Some(value_a.clone())
2392        );
2393        assert_eq!(
2394            db.get(Location::new(2)).await.unwrap(),
2395            Some(value_b.clone())
2396        );
2397        assert!(
2398            matches!(
2399                db.get(Location::new(4)).await,
2400                Err(Error::LocationOutOfBounds(_, size)) if size == size_before
2401            ),
2402            "rewound append should be out of bounds",
2403        );
2404
2405        db.commit().await.unwrap();
2406        let db = reopen(context.child("reopen")).await;
2407        assert_eq!(db.root(), root_before);
2408        assert_eq!(db.bounds().end, size_before);
2409        assert_eq!(db.last_commit_loc(), commit_before);
2410        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata_a));
2411        assert_eq!(
2412            db.get(Location::new(1)).await.unwrap(),
2413            Some(value_a.clone())
2414        );
2415        assert_eq!(
2416            db.get(Location::new(2)).await.unwrap(),
2417            Some(value_b.clone())
2418        );
2419        assert!(matches!(
2420            db.get(Location::new(4)).await,
2421            Err(Error::LocationOutOfBounds(_, size)) if size == size_before
2422        ));
2423
2424        let db = db.rewind(initial_size).await.unwrap();
2425        assert_eq!(db.root(), initial_root);
2426        assert_eq!(db.bounds().end, initial_size);
2427        assert_eq!(db.get_metadata().await.unwrap(), None);
2428        assert!(matches!(
2429            db.get(Location::new(1)).await,
2430            Err(Error::LocationOutOfBounds(_, size)) if size == initial_size
2431        ));
2432
2433        db.commit().await.unwrap();
2434        let db = reopen(context.child("reopen_initial_boundary")).await;
2435        assert_eq!(db.root(), initial_root);
2436        assert_eq!(db.bounds().end, initial_size);
2437        assert_eq!(db.get_metadata().await.unwrap(), None);
2438        assert!(matches!(
2439            db.get(Location::new(1)).await,
2440            Err(Error::LocationOutOfBounds(_, size)) if size == initial_size
2441        ));
2442
2443        db.destroy().await.unwrap();
2444    }
2445
2446    #[boxed]
2447    pub(crate) async fn run_rewind_pruned_target_errors<F: Family, V, C, H, S: Strategy>(
2448        context: deterministic::Context,
2449        db: TestKeyless<F, V, C, H, S>,
2450        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2451    ) where
2452        V: ValueEncoding<Value: TestValue>,
2453        C: Mutable<Item = Operation<F, V>>,
2454        H: Hasher,
2455        Operation<F, V>: EncodeShared,
2456    {
2457        let (mut db, first_range) = commit_appends(db, (0..16).map(V::Value::make), None).await;
2458
2459        let mut round = 0u64;
2460        loop {
2461            round += 1;
2462            assert!(
2463                round <= 64,
2464                "failed to prune enough history for rewind test"
2465            );
2466
2467            (db, _) =
2468                commit_appends(db, (0..16).map(|i| V::Value::make(round * 100 + i)), None).await;
2469            let last_commit = db.last_commit_loc();
2470            db = db.prune(last_commit).await.unwrap();
2471
2472            if db.bounds().start > first_range.start {
2473                break;
2474            }
2475        }
2476
2477        let oldest_retained = db.bounds().start;
2478        let Err(boundary_err) = db.rewind(oldest_retained).await else {
2479            panic!("expected rewind to fail");
2480        };
2481        assert!(
2482            matches!(
2483                boundary_err,
2484                Error::Journal(crate::journal::Error::ItemPruned(_))
2485            ),
2486            "unexpected rewind error at retained boundary: {boundary_err:?}"
2487        );
2488
2489        let db = reopen(context.child("reopen_boundary")).await;
2490        let Err(err) = db.rewind(first_range.start).await else {
2491            panic!("expected rewind to fail");
2492        };
2493        assert!(
2494            matches!(err, Error::Journal(crate::journal::Error::ItemPruned(_))),
2495            "unexpected rewind error: {err:?}"
2496        );
2497    }
2498
2499    #[boxed]
2500    pub(crate) async fn run_floor_tracking<F: Family, V, C, H, S: Strategy>(
2501        context: deterministic::Context,
2502        db: TestKeyless<F, V, C, H, S>,
2503        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2504    ) where
2505        V: ValueEncoding<Value: TestValue>,
2506        C: Mutable<Item = Operation<F, V>>,
2507        H: Hasher,
2508        Operation<F, V>: EncodeShared,
2509    {
2510        // Freshly created db has floor = 0.
2511        assert_eq!(db.inactivity_floor_loc(), Location::new(0));
2512
2513        // Apply a batch with a declared floor; the db's floor should update.
2514        let floor_a = Location::<F>::new(2);
2515        let merkleized = db
2516            .new_batch()
2517            .append(V::Value::make(1))
2518            .append(V::Value::make(2))
2519            .merkleize(&db, None, floor_a)
2520            .await;
2521        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2522        let db = db.commit().await.unwrap();
2523        assert_eq!(db.inactivity_floor_loc(), floor_a);
2524
2525        // Reopen: floor should survive restart (it's part of the last commit operation).
2526        drop(db);
2527        let db = reopen(context.child("reopen")).await;
2528        assert_eq!(db.inactivity_floor_loc(), floor_a);
2529
2530        // Floor may stay the same across a commit (monotonic non-decreasing).
2531        let merkleized = db
2532            .new_batch()
2533            .append(V::Value::make(3))
2534            .merkleize(&db, None, floor_a)
2535            .await;
2536        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2537        assert_eq!(db.inactivity_floor_loc(), floor_a);
2538
2539        // Floor may advance further.
2540        let floor_b = Location::<F>::new(5);
2541        let merkleized = db
2542            .new_batch()
2543            .append(V::Value::make(4))
2544            .merkleize(&db, None, floor_b)
2545            .await;
2546        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2547        assert_eq!(db.inactivity_floor_loc(), floor_b);
2548
2549        db.destroy().await.unwrap();
2550    }
2551
2552    #[boxed]
2553    pub(crate) async fn run_floor_regression_rejected<F: Family, V, C, H, S: Strategy>(
2554        context: deterministic::Context,
2555        db: TestKeyless<F, V, C, H, S>,
2556        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2557    ) where
2558        V: ValueEncoding<Value: TestValue>,
2559        C: Mutable<Item = Operation<F, V>>,
2560        H: Hasher,
2561        Operation<F, V>: EncodeShared,
2562    {
2563        // Advance floor to 3 and commit so the state is durable.
2564        let merkleized = db
2565            .new_batch()
2566            .append(V::Value::make(1))
2567            .append(V::Value::make(2))
2568            .merkleize(&db, None, Location::new(3))
2569            .await;
2570        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2571        let db = db.commit().await.unwrap();
2572        assert_eq!(db.inactivity_floor_loc(), Location::new(3));
2573        let root_before = db.root();
2574        let last_commit_before = db.last_commit_loc();
2575
2576        // Try to commit with a lower floor; apply_batch rejects.
2577        let merkleized = db
2578            .new_batch()
2579            .append(V::Value::make(3))
2580            .merkleize(&db, None, Location::new(1))
2581            .await;
2582        let Err(err) = db.apply_batch(merkleized).await else {
2583            panic!("expected apply_batch to fail");
2584        };
2585        assert!(
2586            matches!(err, Error::FloorRegressed(new, current) if *new == 1 && *current == 3),
2587            "unexpected error: {err:?}"
2588        );
2589
2590        // Reopen the partition and verify the rejected batch persisted nothing.
2591        let db = reopen(context.child("reopen")).await;
2592        assert_eq!(db.inactivity_floor_loc(), Location::new(3));
2593        assert_eq!(db.last_commit_loc(), last_commit_before);
2594        assert_eq!(db.root(), root_before);
2595
2596        db.destroy().await.unwrap();
2597    }
2598
2599    #[boxed]
2600    pub(crate) async fn run_floor_beyond_commit_loc_rejected<F: Family, V, C, H, S: Strategy>(
2601        context: deterministic::Context,
2602        db: TestKeyless<F, V, C, H, S>,
2603        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2604    ) where
2605        V: ValueEncoding<Value: TestValue>,
2606        C: Mutable<Item = Operation<F, V>>,
2607        H: Hasher,
2608        Operation<F, V>: EncodeShared,
2609    {
2610        // Batch of 2 appends + 1 commit lands at locations [1..4); commit at 3, total_size = 4.
2611        // A floor > 3 (the commit location) is invalid — even floor == 4 (one past the commit)
2612        // is rejected so a subsequent prune cannot remove the last readable commit.
2613        let floor = db.inactivity_floor_loc();
2614        let last_commit_loc = db.last_commit_loc();
2615        let root = db.root();
2616        let merkleized = db
2617            .new_batch()
2618            .append(V::Value::make(1))
2619            .append(V::Value::make(2))
2620            .merkleize(&db, None, Location::new(999))
2621            .await;
2622        let Err(err) = db.apply_batch(merkleized).await else {
2623            panic!("expected apply_batch to fail");
2624        };
2625        assert!(
2626            matches!(err, Error::FloorBeyondSize(floor, commit) if *floor == 999 && *commit == 3),
2627            "unexpected error: {err:?}"
2628        );
2629
2630        // Reopen and confirm nothing persisted.
2631        let db = reopen(context.child("reopen_boundary")).await;
2632        assert_eq!(db.inactivity_floor_loc(), floor);
2633        assert_eq!(db.last_commit_loc(), last_commit_loc);
2634        assert_eq!(db.root(), root);
2635
2636        // Boundary: floor == total_size (= commit_loc + 1) is also rejected.
2637        let merkleized = db
2638            .new_batch()
2639            .append(V::Value::make(3))
2640            .append(V::Value::make(4))
2641            .merkleize(&db, None, Location::new(4))
2642            .await;
2643        let Err(err) = db.apply_batch(merkleized).await else {
2644            panic!("expected apply_batch to fail");
2645        };
2646        assert!(
2647            matches!(err, Error::FloorBeyondSize(floor, commit) if *floor == 4 && *commit == 3),
2648            "unexpected error: {err:?}"
2649        );
2650    }
2651
2652    #[boxed]
2653    pub(crate) async fn run_rewind_restores_floor<F: Family, V, C, H, S: Strategy>(
2654        db: TestKeyless<F, V, C, H, S>,
2655    ) where
2656        V: ValueEncoding<Value: TestValue>,
2657        C: Mutable<Item = Operation<F, V>>,
2658        H: Hasher,
2659        Operation<F, V>: EncodeShared,
2660    {
2661        // First commit: floor advances to 3 (= commit location).
2662        let floor_a = Location::<F>::new(3);
2663        let merkleized = db
2664            .new_batch()
2665            .append(V::Value::make(1))
2666            .append(V::Value::make(2))
2667            .merkleize(&db, None, floor_a)
2668            .await;
2669        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2670        let db = db.commit().await.unwrap();
2671        let rewind_target = db.last_commit_loc() + 1;
2672
2673        // Second commit: floor advances to 6.
2674        let floor_b = Location::<F>::new(6);
2675        let merkleized = db
2676            .new_batch()
2677            .append(V::Value::make(3))
2678            .append(V::Value::make(4))
2679            .merkleize(&db, None, floor_b)
2680            .await;
2681        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2682        let db = db.commit().await.unwrap();
2683        assert_eq!(db.inactivity_floor_loc(), floor_b);
2684
2685        // Rewind to the first commit; floor should restore to floor_a.
2686        let db = db.rewind(rewind_target).await.unwrap();
2687        assert_eq!(db.inactivity_floor_loc(), floor_a);
2688
2689        // Prune is now gated at floor_a: pruning up to the floor works.
2690        let db = db.prune(floor_a).await.unwrap();
2691
2692        // Pruning past the floor fails.
2693        let beyond = floor_a + 1;
2694        let Err(err) = db.prune(beyond).await else {
2695            panic!("expected prune to fail");
2696        };
2697        assert!(matches!(err, Error::PruneBeyondMinRequired(_, _)));
2698    }
2699
2700    /// Floor is embedded in the Commit operation and therefore in the Merkle root: two databases
2701    /// with identical appends but different floors must produce different roots.
2702    #[boxed]
2703    pub(crate) async fn run_floor_changes_root<F: Family, V, C, H, S: Strategy>(
2704        db_a: TestKeyless<F, V, C, H, S>,
2705        db_b: TestKeyless<F, V, C, H, S>,
2706    ) where
2707        V: ValueEncoding<Value: TestValue>,
2708        C: Mutable<Item = Operation<F, V>>,
2709        H: Hasher,
2710        Operation<F, V>: EncodeShared,
2711    {
2712        let appends = [V::Value::make(1), V::Value::make(2)];
2713
2714        // db_a commits with floor=0.
2715        let mut batch_a = db_a.new_batch();
2716        for v in appends.iter() {
2717            batch_a = batch_a.append(v.clone());
2718        }
2719        let merkleized = batch_a.merkleize(&db_a, None, Location::new(0)).await;
2720        let (db_a, _) = db_a.apply_batch(merkleized).await.unwrap();
2721
2722        // db_b commits the same appends but with floor=3 (= commit location).
2723        let mut batch_b = db_b.new_batch();
2724        for v in appends.iter() {
2725            batch_b = batch_b.append(v.clone());
2726        }
2727        let merkleized = batch_b.merkleize(&db_b, None, Location::new(3)).await;
2728        let (db_b, _) = db_b.apply_batch(merkleized).await.unwrap();
2729
2730        assert_ne!(db_a.root(), db_b.root());
2731
2732        db_a.destroy().await.unwrap();
2733        db_b.destroy().await.unwrap();
2734    }
2735
2736    /// A floor equal to the commit operation's location is on the tight boundary of acceptance.
2737    #[boxed]
2738    pub(crate) async fn run_floor_at_commit_loc_accepted<F: Family, V, C, H, S: Strategy>(
2739        db: TestKeyless<F, V, C, H, S>,
2740    ) where
2741        V: ValueEncoding<Value: TestValue>,
2742        C: Mutable<Item = Operation<F, V>>,
2743        H: Hasher,
2744        Operation<F, V>: EncodeShared,
2745    {
2746        // 2 appends + 1 commit on top of the initial commit: commit lands at location 3.
2747        // floor == 3 (= commit_loc) is the maximum accepted value under the per-commit bound.
2748        let commit_loc = Location::<F>::new(3);
2749        let merkleized = db
2750            .new_batch()
2751            .append(V::Value::make(1))
2752            .append(V::Value::make(2))
2753            .merkleize(&db, None, commit_loc)
2754            .await;
2755        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2756        assert_eq!(db.inactivity_floor_loc(), commit_loc);
2757
2758        db.destroy().await.unwrap();
2759    }
2760
2761    /// End-to-end: commit → drop → reopen → rewind → verify floor restored after a crash.
2762    #[boxed]
2763    pub(crate) async fn run_rewind_after_reopen_with_floor<F: Family, V, C, H, S: Strategy>(
2764        context: deterministic::Context,
2765        db: TestKeyless<F, V, C, H, S>,
2766        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2767    ) where
2768        V: ValueEncoding<Value: TestValue>,
2769        C: Mutable<Item = Operation<F, V>>,
2770        H: Hasher,
2771        Operation<F, V>: EncodeShared,
2772    {
2773        // First commit: 2 appends + commit, floor advances to 3.
2774        let floor_a = Location::<F>::new(3);
2775        let merkleized = db
2776            .new_batch()
2777            .append(V::Value::make(1))
2778            .append(V::Value::make(2))
2779            .merkleize(&db, None, floor_a)
2780            .await;
2781        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2782        let db = db.commit().await.unwrap();
2783        let rewind_target = db.last_commit_loc() + 1;
2784
2785        // Second commit: 2 appends + commit, floor advances to 6.
2786        let floor_b = Location::<F>::new(6);
2787        let merkleized = db
2788            .new_batch()
2789            .append(V::Value::make(3))
2790            .append(V::Value::make(4))
2791            .merkleize(&db, None, floor_b)
2792            .await;
2793        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2794        db.commit().await.unwrap();
2795
2796        // Reopen to simulate a crash after both commits were durable.
2797        let db = reopen(context.child("reopen")).await;
2798        assert_eq!(db.inactivity_floor_loc(), floor_b);
2799
2800        // Rewind to the first commit; floor should restore to floor_a.
2801        let db = db.rewind(rewind_target).await.unwrap();
2802        assert_eq!(db.inactivity_floor_loc(), floor_a);
2803        assert_eq!(db.last_commit_loc(), Location::new(3));
2804
2805        // Commit the rewind so it's durable, then reopen and confirm the floor again.
2806        db.commit().await.unwrap();
2807        let db = reopen(context.child("reopen").with_attribute("index", 2)).await;
2808        assert_eq!(db.inactivity_floor_loc(), floor_a);
2809
2810        db.destroy().await.unwrap();
2811    }
2812
2813    /// A chained batch that applies a tip with a floor *lower than* its parent's floor must
2814    /// be rejected — the parent's `Commit` is written to the journal by the same
2815    /// `journal.apply_batch` call, so its floor participates in the per-commit monotonicity
2816    /// invariant.
2817    #[boxed]
2818    pub(crate) async fn run_ancestor_floor_regression_rejected<F, V, C, H, S: Strategy>(
2819        context: deterministic::Context,
2820        db: TestKeyless<F, V, C, H, S>,
2821        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2822    ) where
2823        F: Family,
2824        V: ValueEncoding<Value: TestValue>,
2825        C: Mutable<Item = Operation<F, V>>,
2826        H: Hasher,
2827        Operation<F, V>: EncodeShared,
2828    {
2829        // parent: 1 append + commit at loc 2 with floor=2 (the parent's commit_loc).
2830        let parent = db
2831            .new_batch()
2832            .append(V::Value::make(1))
2833            .merkleize(&db, None, Location::new(2))
2834            .await;
2835        // child: 1 append + commit at loc 4 with floor=1 (regressed from parent's floor=2).
2836        let child = parent
2837            .new_batch::<H>()
2838            .append(V::Value::make(2))
2839            .merkleize(&db, None, Location::new(1))
2840            .await;
2841
2842        let root_before = db.root();
2843        let last_commit_before = db.last_commit_loc();
2844        let floor_before = db.inactivity_floor_loc();
2845
2846        let Err(err) = db.apply_batch(child).await else {
2847            panic!("expected apply_batch to fail");
2848        };
2849        assert!(
2850            matches!(err, Error::FloorRegressed(new, prev) if *new == 1 && *prev == 2),
2851            "unexpected error: {err:?}"
2852        );
2853
2854        // Reopen the partition and verify the rejected chain persisted nothing.
2855        let db = reopen(context.child("reopen")).await;
2856        assert_eq!(db.root(), root_before);
2857        assert_eq!(db.last_commit_loc(), last_commit_before);
2858        assert_eq!(db.inactivity_floor_loc(), floor_before);
2859
2860        db.destroy().await.unwrap();
2861    }
2862
2863    /// A chained batch where an *ancestor's* floor exceeds its own commit location must be
2864    /// rejected — identifying the ancestor's bound, not the tip's.
2865    #[boxed]
2866    pub(crate) async fn run_ancestor_floor_beyond_commit_loc_rejected<F, V, C, H, S: Strategy>(
2867        db: TestKeyless<F, V, C, H, S>,
2868    ) where
2869        F: Family,
2870        V: ValueEncoding<Value: TestValue>,
2871        C: Mutable<Item = Operation<F, V>>,
2872        H: Hasher,
2873        Operation<F, V>: EncodeShared,
2874    {
2875        // parent: 1 append + commit at loc 2. Declare floor = 3 (one past the commit).
2876        let parent = db
2877            .new_batch()
2878            .append(V::Value::make(1))
2879            .merkleize(&db, None, Location::new(3))
2880            .await;
2881        // child: valid on its own (floor = 0 ≤ child's commit_loc), but parent's floor is bad.
2882        let child = parent
2883            .new_batch::<H>()
2884            .append(V::Value::make(2))
2885            .merkleize(&db, None, Location::new(0))
2886            .await;
2887
2888        let Err(err) = db.apply_batch(child).await else {
2889            panic!("expected apply_batch to fail");
2890        };
2891        // Error should identify the ancestor's commit_loc (2), not the tip's.
2892        assert!(
2893            matches!(err, Error::FloorBeyondSize(floor, commit) if *floor == 3 && *commit == 2),
2894            "unexpected error: {err:?}"
2895        );
2896    }
2897
2898    /// After committing with `floor = commit_loc` and pruning down to it, the live set is
2899    /// exactly one operation — the commit itself. This is the minimum non-empty live set
2900    /// achievable under the per-commit bound. The DB must remain fully usable: the commit is
2901    /// readable, the root is preserved, reopen recovers `inactivity_floor_loc` from the sole
2902    /// remaining op, and a follow-on batch applies cleanly on top.
2903    #[boxed]
2904    pub(crate) async fn run_single_commit_live_set<F, V, C, H, S: Strategy>(
2905        context: deterministic::Context,
2906        db: TestKeyless<F, V, C, H, S>,
2907        reopen: Reopen<TestKeyless<F, V, C, H, S>>,
2908    ) where
2909        F: Family,
2910        V: ValueEncoding<Value: TestValue>,
2911        C: Mutable<Item = Operation<F, V>>,
2912        H: Hasher,
2913        Operation<F, V>: EncodeShared,
2914    {
2915        // Initial commit is at loc 0. 3 appends + commit → commit lands at loc 4.
2916        // Declare floor = 4 (= commit_loc), the tight maximum.
2917        let metadata = V::Value::make(42);
2918        let commit_loc = Location::<F>::new(4);
2919        let merkleized = db
2920            .new_batch()
2921            .append(V::Value::make(1))
2922            .append(V::Value::make(2))
2923            .append(V::Value::make(3))
2924            .merkleize(&db, Some(metadata.clone()), commit_loc)
2925            .await;
2926        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2927        let db = db.commit().await.unwrap();
2928        assert_eq!(db.last_commit_loc(), commit_loc);
2929        assert_eq!(db.inactivity_floor_loc(), commit_loc);
2930        let root_after_commit = db.root();
2931
2932        // Prune at the floor — the maximum prune allowed under the invariant.
2933        // Pruning is blob-aligned, so `bounds.start` may not physically advance all the way
2934        // to `commit_loc`; what matters semantically is that the floor has authorized pruning
2935        // of everything below the commit and that any further prune is rejected.
2936        let db = db.prune(commit_loc).await.unwrap();
2937        let bounds = db.bounds();
2938        assert!(
2939            bounds.start <= commit_loc,
2940            "prune must not advance bounds.start past the floor"
2941        );
2942        assert_eq!(bounds.end, commit_loc + 1);
2943
2944        // The commit op remains readable; its metadata is intact.
2945        assert_eq!(db.get(commit_loc).await.unwrap(), Some(metadata.clone()));
2946        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone()));
2947        assert_eq!(db.last_commit_loc(), commit_loc);
2948        assert_eq!(db.inactivity_floor_loc(), commit_loc);
2949        // Prune does not affect the root (documented invariant on `prune`).
2950        assert_eq!(db.root(), root_after_commit);
2951
2952        // Persist the prune, then verify pruning one past the floor is rejected — the
2953        // floor is the hard ceiling.
2954        let db = db.sync().await.unwrap();
2955        let Err(err) = db.prune(commit_loc + 1).await else {
2956            panic!("expected prune to fail");
2957        };
2958        assert!(matches!(err, Error::PruneBeyondMinRequired(p, f)
2959                if *p == *commit_loc + 1 && *f == *commit_loc));
2960
2961        // Reopen: `init_from_journal` must recover the floor from the last commit op.
2962        let db = reopen(context.child("reopened")).await;
2963        let reopened_bounds = db.bounds();
2964        assert_eq!(reopened_bounds.end, commit_loc + 1);
2965        assert_eq!(db.last_commit_loc(), commit_loc);
2966        assert_eq!(db.inactivity_floor_loc(), commit_loc);
2967        assert_eq!(db.root(), root_after_commit);
2968        assert_eq!(db.get_metadata().await.unwrap(), Some(metadata.clone()));
2969
2970        // A follow-on batch applies on top. Monotonicity requires the new floor to be at
2971        // least `commit_loc` (= 4); advancing to the new tight max (= 7) exercises the
2972        // ancestor-to-tip floor transition from a minimum-live-set starting point.
2973        let next_commit_loc = Location::<F>::new(7);
2974        let v5 = V::Value::make(5);
2975        let v6 = V::Value::make(6);
2976        let merkleized = db
2977            .new_batch()
2978            .append(v5.clone())
2979            .append(v6.clone())
2980            .merkleize(&db, None, next_commit_loc)
2981            .await;
2982        let (db, _) = db.apply_batch(merkleized).await.unwrap();
2983        let db = db.commit().await.unwrap();
2984        assert_eq!(db.last_commit_loc(), next_commit_loc);
2985        assert_eq!(db.inactivity_floor_loc(), next_commit_loc);
2986
2987        // New appends readable; the original commit op is also still in the live set (not
2988        // re-pruned), so reading it still returns its metadata.
2989        assert_eq!(db.get(Location::new(5)).await.unwrap(), Some(v5));
2990        assert_eq!(db.get(Location::new(6)).await.unwrap(), Some(v6));
2991        assert_eq!(db.get(commit_loc).await.unwrap(), Some(metadata));
2992
2993        db.destroy().await.unwrap();
2994    }
2995
2996    /// A multi-level chain with strictly-monotonic, within-bounds floors applies cleanly.
2997    #[boxed]
2998    pub(crate) async fn run_chained_apply_with_valid_floors_succeeds<F, V, C, H, S: Strategy>(
2999        db: TestKeyless<F, V, C, H, S>,
3000    ) where
3001        F: Family,
3002        V: ValueEncoding<Value: TestValue>,
3003        C: Mutable<Item = Operation<F, V>>,
3004        H: Hasher,
3005        Operation<F, V>: EncodeShared,
3006    {
3007        // parent:     1 append, commit at loc 2, floor = 2.
3008        // child:      1 append, commit at loc 4, floor = 3.
3009        // grandchild: 1 append, commit at loc 6, floor = 5.
3010        let parent = db
3011            .new_batch()
3012            .append(V::Value::make(1))
3013            .merkleize(&db, None, Location::new(2))
3014            .await;
3015        let child = parent
3016            .new_batch::<H>()
3017            .append(V::Value::make(2))
3018            .merkleize(&db, None, Location::new(3))
3019            .await;
3020        let grandchild = child
3021            .new_batch::<H>()
3022            .append(V::Value::make(3))
3023            .merkleize(&db, None, Location::new(5))
3024            .await;
3025
3026        let (db, _) = db.apply_batch(grandchild).await.unwrap();
3027
3028        // Grandchild's commit is the last op; tip's floor is the live floor.
3029        assert_eq!(db.last_commit_loc(), Location::new(6));
3030        assert_eq!(db.inactivity_floor_loc(), Location::new(5));
3031
3032        db.destroy().await.unwrap();
3033    }
3034}