commonware_storage/qmdb/any/
mod.rs

1//! An _Any_ authenticated database provides succinct proofs of any value ever associated with a
2//! key.
3//!
4//! The specific variants provided within this module include:
5//! - Unordered: The database does not maintain or require any ordering over the key space.
6//!   - Fixed-size values
7//!   - Variable-size values
8//! - Ordered: The database maintains a total order over active keys.
9//!   - Fixed-size values
10//!   - Variable-size values
11
12use crate::{
13    journal::{
14        authenticated,
15        contiguous::fixed::{Config as JConfig, Journal},
16    },
17    mmr::journaled::Config as MmrConfig,
18    qmdb::{operation::Committable, Error, Merkleized},
19    translator::Translator,
20};
21use commonware_codec::CodecFixedShared;
22use commonware_cryptography::Hasher;
23use commonware_parallel::ThreadPool;
24use commonware_runtime::{buffer::PoolRef, Clock, Metrics, Storage};
25use std::num::{NonZeroU64, NonZeroUsize};
26
27pub(crate) mod db;
28mod operation;
29#[cfg(any(test, feature = "test-traits"))]
30pub mod states;
31mod value;
32pub(crate) use value::{FixedValue, ValueEncoding, VariableValue};
33pub mod ordered;
34pub mod unordered;
35
36/// Configuration for an `Any` authenticated db with fixed-size values.
37#[derive(Clone)]
38pub struct FixedConfig<T: Translator> {
39    /// The name of the [Storage] partition used for the MMR's backing journal.
40    pub mmr_journal_partition: String,
41
42    /// The items per blob configuration value used by the MMR journal.
43    pub mmr_items_per_blob: NonZeroU64,
44
45    /// The size of the write buffer to use for each blob in the MMR journal.
46    pub mmr_write_buffer: NonZeroUsize,
47
48    /// The name of the [Storage] partition used for the MMR's metadata.
49    pub mmr_metadata_partition: String,
50
51    /// The name of the [Storage] partition used to persist the (pruned) log of operations.
52    pub log_journal_partition: String,
53
54    /// The items per blob configuration value used by the log journal.
55    pub log_items_per_blob: NonZeroU64,
56
57    /// The size of the write buffer to use for each blob in the log journal.
58    pub log_write_buffer: NonZeroUsize,
59
60    /// The translator used by the compressed index.
61    pub translator: T,
62
63    /// An optional thread pool to use for parallelizing batch operations.
64    pub thread_pool: Option<ThreadPool>,
65
66    /// The buffer pool to use for caching data.
67    pub buffer_pool: PoolRef,
68}
69
70/// Configuration for an `Any` authenticated db with variable-sized values.
71#[derive(Clone)]
72pub struct VariableConfig<T: Translator, C> {
73    /// The name of the [Storage] partition used for the MMR's backing journal.
74    pub mmr_journal_partition: String,
75
76    /// The items per blob configuration value used by the MMR journal.
77    pub mmr_items_per_blob: NonZeroU64,
78
79    /// The size of the write buffer to use for each blob in the MMR journal.
80    pub mmr_write_buffer: NonZeroUsize,
81
82    /// The name of the [Storage] partition used for the MMR's metadata.
83    pub mmr_metadata_partition: String,
84
85    /// The name of the [Storage] partition used to persist the log of operations.
86    pub log_partition: String,
87
88    /// The size of the write buffer to use for each blob in the log journal.
89    pub log_write_buffer: NonZeroUsize,
90
91    /// Optional compression level (using `zstd`) to apply to log data before storing.
92    pub log_compression: Option<u8>,
93
94    /// The codec configuration to use for encoding and decoding log items.
95    pub log_codec_config: C,
96
97    /// The number of items to put in each blob of the journal.
98    pub log_items_per_blob: NonZeroU64,
99
100    /// The translator used by the compressed index.
101    pub translator: T,
102
103    /// An optional thread pool to use for parallelizing batch operations.
104    pub thread_pool: Option<ThreadPool>,
105
106    /// The buffer pool to use for caching data.
107    pub buffer_pool: PoolRef,
108}
109
110type AuthenticatedLog<E, O, H, S = Merkleized<H>> = authenticated::Journal<E, Journal<E, O>, H, S>;
111
112/// Initialize the authenticated log from the given config, returning it along with the inactivity
113/// floor specified by the last commit.
114pub(crate) async fn init_fixed_authenticated_log<
115    E: Storage + Clock + Metrics,
116    O: Committable + CodecFixedShared,
117    H: Hasher,
118    T: Translator,
119>(
120    context: E,
121    cfg: FixedConfig<T>,
122) -> Result<AuthenticatedLog<E, O, H>, Error> {
123    let mmr_config = MmrConfig {
124        journal_partition: cfg.mmr_journal_partition,
125        metadata_partition: cfg.mmr_metadata_partition,
126        items_per_blob: cfg.mmr_items_per_blob,
127        write_buffer: cfg.mmr_write_buffer,
128        thread_pool: cfg.thread_pool,
129        buffer_pool: cfg.buffer_pool.clone(),
130    };
131
132    let journal_config = JConfig {
133        partition: cfg.log_journal_partition,
134        items_per_blob: cfg.log_items_per_blob,
135        write_buffer: cfg.log_write_buffer,
136        buffer_pool: cfg.buffer_pool,
137    };
138
139    let log = AuthenticatedLog::new(
140        context.with_label("log"),
141        mmr_config,
142        journal_config,
143        O::is_commit,
144    )
145    .await?;
146
147    Ok(log)
148}
149
150#[cfg(test)]
151// pub(crate) so qmdb/current can use the generic tests.
152pub(crate) mod test {
153    use super::*;
154    use crate::{
155        qmdb::any::{FixedConfig, VariableConfig},
156        translator::TwoCap,
157    };
158    use commonware_utils::{NZUsize, NZU16, NZU64};
159    use std::num::NonZeroU16;
160
161    // Janky page & cache sizes to exercise boundary conditions.
162    const PAGE_SIZE: NonZeroU16 = NZU16!(101);
163    const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(11);
164
165    pub(super) fn fixed_db_config(suffix: &str) -> FixedConfig<TwoCap> {
166        FixedConfig {
167            mmr_journal_partition: format!("journal_{suffix}"),
168            mmr_metadata_partition: format!("metadata_{suffix}"),
169            mmr_items_per_blob: NZU64!(11),
170            mmr_write_buffer: NZUsize!(1024),
171            log_journal_partition: format!("log_journal_{suffix}"),
172            log_items_per_blob: NZU64!(7),
173            log_write_buffer: NZUsize!(1024),
174            translator: TwoCap,
175            thread_pool: None,
176            buffer_pool: PoolRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
177        }
178    }
179
180    pub(super) fn variable_db_config(suffix: &str) -> VariableConfig<TwoCap, ()> {
181        VariableConfig {
182            mmr_journal_partition: format!("journal_{suffix}"),
183            mmr_metadata_partition: format!("metadata_{suffix}"),
184            mmr_items_per_blob: NZU64!(11),
185            mmr_write_buffer: NZUsize!(1024),
186            log_partition: format!("log_journal_{suffix}"),
187            log_items_per_blob: NZU64!(7),
188            log_write_buffer: NZUsize!(1024),
189            log_compression: None,
190            log_codec_config: (),
191            translator: TwoCap,
192            thread_pool: None,
193            buffer_pool: PoolRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
194        }
195    }
196
197    use crate::{
198        kv::Updatable,
199        qmdb::{
200            any::states::{CleanAny, MerkleizedNonDurableAny, MutableAny, UnmerkleizedDurableAny},
201            store::MerkleizedStore,
202        },
203        Persistable,
204    };
205    use commonware_cryptography::{sha256::Digest, Sha256};
206
207    /// Test that merkleization state changes don't reset `steps`.
208    pub(crate) async fn test_any_db_steps_not_reset<D>(db: D)
209    where
210        D: CleanAny<Key = Digest> + MerkleizedStore<Value = Digest, Digest = Digest>,
211        D::Mutable: Updatable<Key = Digest, Value = Digest, Error = crate::qmdb::Error>,
212    {
213        // Create a db with a couple keys.
214        let mut db = db.into_mutable();
215
216        assert!(db
217            .create(Sha256::fill(1u8), Sha256::fill(2u8))
218            .await
219            .unwrap());
220        assert!(db
221            .create(Sha256::fill(3u8), Sha256::fill(4u8))
222            .await
223            .unwrap());
224        let (clean_db, _) = db.commit(None).await.unwrap();
225        let mut db = clean_db.into_mutable();
226
227        // Updating an existing key should make steps non-zero.
228        db.update(Sha256::fill(1u8), Sha256::fill(5u8))
229            .await
230            .unwrap();
231        let steps = db.steps();
232        assert_ne!(steps, 0);
233
234        // Steps shouldn't change from merkleization.
235        let db = db.into_merkleized().await.unwrap();
236        let db = db.into_mutable();
237        assert_eq!(db.steps(), steps);
238
239        // Cleanup
240        let (db, _) = db.commit(None).await.unwrap();
241        let db = db.into_merkleized().await.unwrap();
242        db.destroy().await.unwrap();
243    }
244}
commonware_storage/qmdb/any/mod.rs

commonware_storage/qmdb/any/
mod.rs
