commonware_storage/qmdb/any/
mod.rs

1//! An _Any_ authenticated database provides succinct proofs of any value ever associated with a
2//! key.
3//!
4//! The specific variants provided within this module include:
5//! - Unordered: The database does not maintain or require any ordering over the key space.
6//!   - Fixed-size values
7//!   - Variable-size values
8//! - Ordered: The database maintains a total order over active keys.
9//!   - Fixed-size values
10//!   - Variable-size values
11
12use crate::{
13    journal::{
14        authenticated,
15        contiguous::fixed::{Config as JConfig, Journal},
16    },
17    mmr::journaled::Config as MmrConfig,
18    qmdb::{operation::Committable, Error, Merkleized},
19    translator::Translator,
20};
21use commonware_codec::CodecFixedShared;
22use commonware_cryptography::Hasher;
23use commonware_parallel::ThreadPool;
24use commonware_runtime::{buffer::paged::CacheRef, Clock, Metrics, Storage};
25use std::num::{NonZeroU64, NonZeroUsize};
26
27pub(crate) mod db;
28pub(crate) mod operation;
29#[cfg(any(test, feature = "test-traits"))]
30pub mod states;
31pub(crate) mod value;
32pub(crate) use value::{FixedValue, ValueEncoding, VariableValue};
33pub mod ordered;
34pub(crate) mod sync;
35pub mod unordered;
36
37/// Configuration for an `Any` authenticated db with fixed-size values.
38#[derive(Clone)]
39pub struct FixedConfig<T: Translator> {
40    /// The name of the [Storage] partition used for the MMR's backing journal.
41    pub mmr_journal_partition: String,
42
43    /// The items per blob configuration value used by the MMR journal.
44    pub mmr_items_per_blob: NonZeroU64,
45
46    /// The size of the write buffer to use for each blob in the MMR journal.
47    pub mmr_write_buffer: NonZeroUsize,
48
49    /// The name of the [Storage] partition used for the MMR's metadata.
50    pub mmr_metadata_partition: String,
51
52    /// The name of the [Storage] partition used to persist the (pruned) log of operations.
53    pub log_journal_partition: String,
54
55    /// The items per blob configuration value used by the log journal.
56    pub log_items_per_blob: NonZeroU64,
57
58    /// The size of the write buffer to use for each blob in the log journal.
59    pub log_write_buffer: NonZeroUsize,
60
61    /// The translator used by the compressed index.
62    pub translator: T,
63
64    /// An optional thread pool to use for parallelizing batch operations.
65    pub thread_pool: Option<ThreadPool>,
66
67    /// The page cache to use for caching data.
68    pub page_cache: CacheRef,
69}
70
71/// Configuration for an `Any` authenticated db with variable-sized values.
72#[derive(Clone)]
73pub struct VariableConfig<T: Translator, C> {
74    /// The name of the [Storage] partition used for the MMR's backing journal.
75    pub mmr_journal_partition: String,
76
77    /// The items per blob configuration value used by the MMR journal.
78    pub mmr_items_per_blob: NonZeroU64,
79
80    /// The size of the write buffer to use for each blob in the MMR journal.
81    pub mmr_write_buffer: NonZeroUsize,
82
83    /// The name of the [Storage] partition used for the MMR's metadata.
84    pub mmr_metadata_partition: String,
85
86    /// The name of the [Storage] partition used to persist the log of operations.
87    pub log_partition: String,
88
89    /// The size of the write buffer to use for each blob in the log journal.
90    pub log_write_buffer: NonZeroUsize,
91
92    /// Optional compression level (using `zstd`) to apply to log data before storing.
93    pub log_compression: Option<u8>,
94
95    /// The codec configuration to use for encoding and decoding log items.
96    pub log_codec_config: C,
97
98    /// The number of items to put in each blob of the journal.
99    pub log_items_per_blob: NonZeroU64,
100
101    /// The translator used by the compressed index.
102    pub translator: T,
103
104    /// An optional thread pool to use for parallelizing batch operations.
105    pub thread_pool: Option<ThreadPool>,
106
107    /// The page cache to use for caching data.
108    pub page_cache: CacheRef,
109}
110
111type AuthenticatedLog<E, O, H, S = Merkleized<H>> = authenticated::Journal<E, Journal<E, O>, H, S>;
112
113/// Initialize the authenticated log from the given config, returning it along with the inactivity
114/// floor specified by the last commit.
115pub(crate) async fn init_fixed_authenticated_log<
116    E: Storage + Clock + Metrics,
117    O: Committable + CodecFixedShared,
118    H: Hasher,
119    T: Translator,
120>(
121    context: E,
122    cfg: FixedConfig<T>,
123) -> Result<AuthenticatedLog<E, O, H>, Error> {
124    let mmr_config = MmrConfig {
125        journal_partition: cfg.mmr_journal_partition,
126        metadata_partition: cfg.mmr_metadata_partition,
127        items_per_blob: cfg.mmr_items_per_blob,
128        write_buffer: cfg.mmr_write_buffer,
129        thread_pool: cfg.thread_pool,
130        page_cache: cfg.page_cache.clone(),
131    };
132
133    let journal_config = JConfig {
134        partition: cfg.log_journal_partition,
135        items_per_blob: cfg.log_items_per_blob,
136        write_buffer: cfg.log_write_buffer,
137        page_cache: cfg.page_cache,
138    };
139
140    let log = AuthenticatedLog::new(
141        context.with_label("log"),
142        mmr_config,
143        journal_config,
144        O::is_commit,
145    )
146    .await?;
147
148    Ok(log)
149}
150
151#[cfg(test)]
152// pub(crate) so qmdb/current can use the generic tests.
153pub(crate) mod test {
154    use super::*;
155    use crate::{
156        qmdb::any::{FixedConfig, VariableConfig},
157        translator::TwoCap,
158    };
159    use commonware_utils::{NZUsize, NZU16, NZU64};
160    use std::num::NonZeroU16;
161
162    // Janky page & cache sizes to exercise boundary conditions.
163    const PAGE_SIZE: NonZeroU16 = NZU16!(101);
164    const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(11);
165
166    pub(super) fn fixed_db_config(suffix: &str) -> FixedConfig<TwoCap> {
167        FixedConfig {
168            mmr_journal_partition: format!("journal_{suffix}"),
169            mmr_metadata_partition: format!("metadata_{suffix}"),
170            mmr_items_per_blob: NZU64!(11),
171            mmr_write_buffer: NZUsize!(1024),
172            log_journal_partition: format!("log_journal_{suffix}"),
173            log_items_per_blob: NZU64!(7),
174            log_write_buffer: NZUsize!(1024),
175            translator: TwoCap,
176            thread_pool: None,
177            page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
178        }
179    }
180
181    pub(super) fn variable_db_config(suffix: &str) -> VariableConfig<TwoCap, ()> {
182        VariableConfig {
183            mmr_journal_partition: format!("journal_{suffix}"),
184            mmr_metadata_partition: format!("metadata_{suffix}"),
185            mmr_items_per_blob: NZU64!(11),
186            mmr_write_buffer: NZUsize!(1024),
187            log_partition: format!("log_journal_{suffix}"),
188            log_items_per_blob: NZU64!(7),
189            log_write_buffer: NZUsize!(1024),
190            log_compression: None,
191            log_codec_config: (),
192            translator: TwoCap,
193            thread_pool: None,
194            page_cache: CacheRef::new(PAGE_SIZE, PAGE_CACHE_SIZE),
195        }
196    }
197
198    use crate::{
199        kv::Updatable,
200        qmdb::{
201            any::states::{CleanAny, MerkleizedNonDurableAny, MutableAny, UnmerkleizedDurableAny},
202            store::MerkleizedStore,
203        },
204        Persistable,
205    };
206    use commonware_cryptography::{sha256::Digest, Sha256};
207
208    /// Test that merkleization state changes don't reset `steps`.
209    pub(crate) async fn test_any_db_steps_not_reset<D>(db: D)
210    where
211        D: CleanAny<Key = Digest> + MerkleizedStore<Value = Digest, Digest = Digest>,
212        D::Mutable: Updatable<Key = Digest, Value = Digest, Error = crate::qmdb::Error>,
213    {
214        // Create a db with a couple keys.
215        let mut db = db.into_mutable();
216
217        assert!(db
218            .create(Sha256::fill(1u8), Sha256::fill(2u8))
219            .await
220            .unwrap());
221        assert!(db
222            .create(Sha256::fill(3u8), Sha256::fill(4u8))
223            .await
224            .unwrap());
225        let (clean_db, _) = db.commit(None).await.unwrap();
226        let mut db = clean_db.into_mutable();
227
228        // Updating an existing key should make steps non-zero.
229        db.update(Sha256::fill(1u8), Sha256::fill(5u8))
230            .await
231            .unwrap();
232        let steps = db.steps();
233        assert_ne!(steps, 0);
234
235        // Steps shouldn't change from merkleization.
236        let db = db.into_merkleized().await.unwrap();
237        let db = db.into_mutable();
238        assert_eq!(db.steps(), steps);
239
240        // Cleanup
241        let (db, _) = db.commit(None).await.unwrap();
242        let db = db.into_merkleized().await.unwrap();
243        db.destroy().await.unwrap();
244    }
245}
commonware_storage/qmdb/any/mod.rs

commonware_storage/qmdb/any/
mod.rs
