commonware_storage/qmdb/any/
mod.rs

1//! Authenticated databases that provides succinct proofs of _any_ value ever associated with
2//! a key. The submodules provide two classes of variants, one specialized for fixed-size values and
3//! the other allowing variable-size values.
4
5use crate::{
6    journal::{
7        authenticated,
8        contiguous::fixed::{Config as JConfig, Journal},
9    },
10    mmr::{journaled::Config as MmrConfig, mem::Clean, Location},
11    qmdb::{
12        operation::Committable,
13        store::{CleanStore, DirtyStore},
14        Error,
15    },
16    translator::Translator,
17};
18use commonware_codec::CodecFixed;
19use commonware_cryptography::{DigestOf, Hasher};
20use commonware_runtime::{buffer::PoolRef, Clock, Metrics, Storage, ThreadPool};
21use commonware_utils::Array;
22use std::{
23    future::Future,
24    num::{NonZeroU64, NonZeroUsize},
25    ops::Range,
26};
27
28pub(crate) mod db;
29mod operation;
30
31mod value;
32pub(crate) use value::{FixedValue, ValueEncoding, VariableValue};
33
34mod ext;
35pub mod ordered;
36pub mod unordered;
37
38pub use ext::AnyExt;
39
40/// Extension trait for "Any" QMDBs in a clean (merkleized) state.
41pub trait CleanAny:
42    CleanStore<Dirty: DirtyAny<Key = Self::Key, Value = Self::Value, Clean = Self>>
43{
44    /// The key type for this database.
45    type Key: Array;
46
47    /// Get the value for a given key, or None if it has no value.
48    fn get(&self, key: &Self::Key) -> impl Future<Output = Result<Option<Self::Value>, Error>>;
49
50    /// Commit pending operations to the database, ensuring durability.
51    /// Returns the location range of committed operations.
52    fn commit(
53        &mut self,
54        metadata: Option<Self::Value>,
55    ) -> impl Future<Output = Result<Range<Location>, Error>>;
56
57    /// Sync all database state to disk.
58    fn sync(&mut self) -> impl Future<Output = Result<(), Error>>;
59
60    /// Prune historical operations prior to `prune_loc`.
61    fn prune(&mut self, prune_loc: Location) -> impl Future<Output = Result<(), Error>>;
62
63    /// Close the db. Uncommitted operations will be lost or rolled back on restart.
64    fn close(self) -> impl Future<Output = Result<(), Error>>;
65
66    /// Destroy the db, removing all data from disk.
67    fn destroy(self) -> impl Future<Output = Result<(), Error>>;
68}
69
70/// Extension trait for "Any" QMDBs in a dirty (deferred merkleization) state.
71pub trait DirtyAny: DirtyStore {
72    /// The key type for this database.
73    type Key: Array;
74
75    /// Get the value for a given key, or None if it has no value.
76    fn get(&self, key: &Self::Key) -> impl Future<Output = Result<Option<Self::Value>, Error>>;
77
78    /// Update `key` to have value `value`. Subject to rollback until next `commit`.
79    fn update(
80        &mut self,
81        key: Self::Key,
82        value: Self::Value,
83    ) -> impl Future<Output = Result<(), Error>>;
84
85    /// Create a new key-value pair. Returns true if created, false if key already existed.
86    /// Subject to rollback until next `commit`.
87    fn create(
88        &mut self,
89        key: Self::Key,
90        value: Self::Value,
91    ) -> impl Future<Output = Result<bool, Error>>;
92
93    /// Delete `key` and its value. Returns true if deleted, false if already inactive.
94    /// Subject to rollback until next `commit`.
95    fn delete(&mut self, key: Self::Key) -> impl Future<Output = Result<bool, Error>>;
96}
97
98/// Configuration for an `Any` authenticated db with fixed-size values.
99#[derive(Clone)]
100pub struct FixedConfig<T: Translator> {
101    /// The name of the [Storage] partition used for the MMR's backing journal.
102    pub mmr_journal_partition: String,
103
104    /// The items per blob configuration value used by the MMR journal.
105    pub mmr_items_per_blob: NonZeroU64,
106
107    /// The size of the write buffer to use for each blob in the MMR journal.
108    pub mmr_write_buffer: NonZeroUsize,
109
110    /// The name of the [Storage] partition used for the MMR's metadata.
111    pub mmr_metadata_partition: String,
112
113    /// The name of the [Storage] partition used to persist the (pruned) log of operations.
114    pub log_journal_partition: String,
115
116    /// The items per blob configuration value used by the log journal.
117    pub log_items_per_blob: NonZeroU64,
118
119    /// The size of the write buffer to use for each blob in the log journal.
120    pub log_write_buffer: NonZeroUsize,
121
122    /// The translator used by the compressed index.
123    pub translator: T,
124
125    /// An optional thread pool to use for parallelizing batch operations.
126    pub thread_pool: Option<ThreadPool>,
127
128    /// The buffer pool to use for caching data.
129    pub buffer_pool: PoolRef,
130}
131
132/// Configuration for an `Any` authenticated db with variable-sized values.
133#[derive(Clone)]
134pub struct VariableConfig<T: Translator, C> {
135    /// The name of the [Storage] partition used for the MMR's backing journal.
136    pub mmr_journal_partition: String,
137
138    /// The items per blob configuration value used by the MMR journal.
139    pub mmr_items_per_blob: NonZeroU64,
140
141    /// The size of the write buffer to use for each blob in the MMR journal.
142    pub mmr_write_buffer: NonZeroUsize,
143
144    /// The name of the [Storage] partition used for the MMR's metadata.
145    pub mmr_metadata_partition: String,
146
147    /// The name of the [Storage] partition used to persist the log of operations.
148    pub log_partition: String,
149
150    /// The size of the write buffer to use for each blob in the log journal.
151    pub log_write_buffer: NonZeroUsize,
152
153    /// Optional compression level (using `zstd`) to apply to log data before storing.
154    pub log_compression: Option<u8>,
155
156    /// The codec configuration to use for encoding and decoding log items.
157    pub log_codec_config: C,
158
159    /// The number of items to put in each blob of the journal.
160    pub log_items_per_blob: NonZeroU64,
161
162    /// The translator used by the compressed index.
163    pub translator: T,
164
165    /// An optional thread pool to use for parallelizing batch operations.
166    pub thread_pool: Option<ThreadPool>,
167
168    /// The buffer pool to use for caching data.
169    pub buffer_pool: PoolRef,
170}
171
172type AuthenticatedLog<E, O, H, S = Clean<DigestOf<H>>> =
173    authenticated::Journal<E, Journal<E, O>, H, S>;
174
175/// Initialize the authenticated log from the given config, returning it along with the inactivity
176/// floor specified by the last commit.
177pub(crate) async fn init_fixed_authenticated_log<
178    E: Storage + Clock + Metrics,
179    O: Committable + CodecFixed<Cfg = ()>,
180    H: Hasher,
181    T: Translator,
182>(
183    context: E,
184    cfg: FixedConfig<T>,
185) -> Result<AuthenticatedLog<E, O, H>, Error> {
186    let mmr_config = MmrConfig {
187        journal_partition: cfg.mmr_journal_partition,
188        metadata_partition: cfg.mmr_metadata_partition,
189        items_per_blob: cfg.mmr_items_per_blob,
190        write_buffer: cfg.mmr_write_buffer,
191        thread_pool: cfg.thread_pool,
192        buffer_pool: cfg.buffer_pool.clone(),
193    };
194
195    let journal_config = JConfig {
196        partition: cfg.log_journal_partition,
197        items_per_blob: cfg.log_items_per_blob,
198        write_buffer: cfg.log_write_buffer,
199        buffer_pool: cfg.buffer_pool,
200    };
201
202    let log = AuthenticatedLog::new(
203        context.with_label("log"),
204        mmr_config,
205        journal_config,
206        O::is_commit,
207    )
208    .await?;
209
210    Ok(log)
211}
212
213#[cfg(test)]
214mod test {
215    use super::*;
216    use crate::{
217        qmdb::any::{FixedConfig, VariableConfig},
218        translator::TwoCap,
219    };
220    use commonware_utils::{NZUsize, NZU64};
221
222    // Janky page & cache sizes to exercise boundary conditions.
223    const PAGE_SIZE: usize = 101;
224    const PAGE_CACHE_SIZE: usize = 11;
225
226    pub(super) fn fixed_db_config(suffix: &str) -> FixedConfig<TwoCap> {
227        FixedConfig {
228            mmr_journal_partition: format!("journal_{suffix}"),
229            mmr_metadata_partition: format!("metadata_{suffix}"),
230            mmr_items_per_blob: NZU64!(11),
231            mmr_write_buffer: NZUsize!(1024),
232            log_journal_partition: format!("log_journal_{suffix}"),
233            log_items_per_blob: NZU64!(7),
234            log_write_buffer: NZUsize!(1024),
235            translator: TwoCap,
236            thread_pool: None,
237            buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)),
238        }
239    }
240
241    pub(super) fn variable_db_config(suffix: &str) -> VariableConfig<TwoCap, ()> {
242        VariableConfig {
243            mmr_journal_partition: format!("journal_{suffix}"),
244            mmr_metadata_partition: format!("metadata_{suffix}"),
245            mmr_items_per_blob: NZU64!(11),
246            mmr_write_buffer: NZUsize!(1024),
247            log_partition: format!("log_journal_{suffix}"),
248            log_items_per_blob: NZU64!(7),
249            log_write_buffer: NZUsize!(1024),
250            log_compression: None,
251            log_codec_config: (),
252            translator: TwoCap,
253            thread_pool: None,
254            buffer_pool: PoolRef::new(NZUsize!(PAGE_SIZE), NZUsize!(PAGE_CACHE_SIZE)),
255        }
256    }
257}
commonware_storage/qmdb/any/mod.rs

commonware_storage/qmdb/any/
mod.rs
