1use super::{Config as BaseConfig, Immutable, operation::Operation as BaseOperation};
6use crate::{
7 Context,
8 journal::{
9 authenticated,
10 contiguous::fixed::{self, Config as JournalConfig},
11 },
12 merkle::Family,
13 qmdb::{
14 Error, ROOT_BAGGING,
15 any::{FixedValue, value::FixedEncoding},
16 },
17 translator::Translator,
18};
19use commonware_cryptography::Hasher;
20use commonware_parallel::Strategy;
21use commonware_utils::Array;
22
23pub type Operation<F, K, V> = BaseOperation<F, K, FixedEncoding<V>>;
25
26pub type Db<F, E, K, V, H, T, S> =
28 Immutable<F, E, K, FixedEncoding<V>, fixed::Journal<E, Operation<F, K, V>>, H, T, S>;
29
30pub type CompactDb<F, E, K, V, H, S> = super::CompactDb<F, E, K, FixedEncoding<V>, H, (), S>;
32
33type Journal<F, E, K, V, H, S> =
34 authenticated::Journal<F, E, fixed::Journal<E, Operation<F, K, V>>, H, S>;
35
36pub type Config<T, S> = BaseConfig<T, JournalConfig, S>;
38
39pub type CompactConfig<S> = super::CompactConfig<(), S>;
41
42impl<F: Family, E: Context, K: Array, V: FixedValue, H: Hasher, T: Translator, S: Strategy>
43 Db<F, E, K, V, H, T, S>
44{
45 pub async fn init(context: E, cfg: Config<T, S>) -> Result<Self, Error<F>> {
48 let journal: Journal<F, E, K, V, H, S> = Journal::new(
49 context.child("journal"),
50 cfg.merkle_config,
51 cfg.log,
52 Operation::<F, K, V>::is_commit,
53 ROOT_BAGGING,
54 )
55 .await?;
56 Self::init_from_journal(journal, context, cfg.translator, cfg.init_buffer).await
57 }
58}
59
60impl<F: Family, E: Context, K: Array, V: FixedValue, H: Hasher, S: Strategy>
61 CompactDb<F, E, K, V, H, S>
62{
63 pub async fn init(context: E, cfg: CompactConfig<S>) -> Result<Self, Error<F>> {
65 let merkle = crate::merkle::compact::Merkle::new(cfg.strategy);
66 Self::init_from_merkle(merkle, context.child("witness"), cfg.witness, ()).await
67 }
68}
69
70#[cfg(test)]
71mod tests {
72 use super::*;
73 use crate::{
74 merkle::{Location, full::Config as MmrConfig, mmb, mmr},
75 qmdb::immutable::tests::{self, immutable_tests},
76 translator::TwoCap,
77 };
78 use commonware_cryptography::{Sha256, sha256::Digest};
79 use commonware_macros::{boxed, test_traced};
80 use commonware_parallel::Sequential;
81 use commonware_runtime::{
82 BufferPooler, Metrics, Runner as _, Spawner as _, Supervisor as _,
83 buffer::paged::CacheRef,
84 deterministic,
85 mocks::{DelayedSyncContext, PendingSyncs, drive_pending_syncs},
86 reschedule,
87 };
88 use commonware_utils::{NZU16, NZU64, NZUsize};
89 use core::{future::Future, pin::Pin};
90 use futures::FutureExt as _;
91 use std::num::{NonZeroU16, NonZeroUsize};
92
93 const PAGE_SIZE: NonZeroU16 = NZU16!(77);
94 const PAGE_CACHE_SIZE: NonZeroUsize = NZUsize!(9);
95
96 fn config(suffix: &str, pooler: &impl BufferPooler) -> Config<TwoCap, Sequential> {
97 let page_cache = CacheRef::from_pooler(pooler, PAGE_SIZE, PAGE_CACHE_SIZE);
98 Config {
99 merkle_config: MmrConfig {
100 journal_partition: format!("journal-{suffix}"),
101 metadata_partition: format!("metadata-{suffix}"),
102 items_per_blob: NZU64!(11),
103 write_buffer: NZUsize!(1024),
104 replay_buffer: NZUsize!(1024),
105 strategy: Sequential,
106 page_cache: page_cache.clone(),
107 },
108 log: JournalConfig {
109 items_per_blob: NZU64!(5),
110 partition: format!("log-{suffix}"),
111 page_cache,
112 write_buffer: NZUsize!(1024),
113 replay_buffer: NZUsize!(1024),
114 },
115 translator: TwoCap,
116 init_buffer: NZUsize!(1 << 21),
117 }
118 }
119
120 async fn open_db<F: Family>(
121 context: deterministic::Context,
122 ) -> Db<F, deterministic::Context, Digest, Digest, Sha256, TwoCap, Sequential> {
123 let cfg = config("partition", &context);
124 Db::init(context, cfg).await.unwrap()
125 }
126
127 async fn open_compact<F: Family>(
128 context: deterministic::Context,
129 ) -> CompactDb<F, deterministic::Context, Digest, Digest, Sha256, Sequential> {
130 let cfg = CompactConfig {
131 strategy: Sequential,
132 witness: crate::journal::contiguous::variable::Config {
133 partition: "compact-immutable-fixed-witness".into(),
134 items_per_section: NZU64!(64),
135 compression: None,
136 codec_config: (),
137 page_cache: CacheRef::from_pooler(&context, PAGE_SIZE, PAGE_CACHE_SIZE),
138 write_buffer: NZUsize!(1024),
139 replay_buffer: NZUsize!(1024),
140 },
141 commit_codec_config: (),
142 };
143 CompactDb::init(context, cfg).await.unwrap()
144 }
145
146 type DelayedDb = Db<
148 mmr::Family,
149 DelayedSyncContext<deterministic::Context>,
150 Digest,
151 Digest,
152 Sha256,
153 TwoCap,
154 Sequential,
155 >;
156
157 fn open_delayed_db(
164 context: &deterministic::Context,
165 label: &'static str,
166 suffix: &str,
167 pending: &PendingSyncs,
168 ) -> impl Future<Output = Result<DelayedDb, Error<mmr::Family>>> {
169 let mut cfg = config(suffix, context);
170 let page_cache = CacheRef::from_pooler(context, NZU16!(1024), NZUsize!(8));
171 cfg.log.items_per_blob = NZU64!(1000);
172 cfg.log.page_cache = page_cache.clone();
173 cfg.merkle_config.items_per_blob = NZU64!(1000);
174 cfg.merkle_config.page_cache = page_cache;
175 DelayedDb::init(
176 DelayedSyncContext {
177 inner: context.child(label),
178 pending: pending.clone(),
179 },
180 cfg,
181 )
182 }
183
184 async fn apply_set(
186 db: DelayedDb,
187 key: Digest,
188 value: Digest,
189 floor: Location<mmr::Family>,
190 ) -> DelayedDb {
191 let batch = db
192 .new_batch()
193 .set(key, value)
194 .merkleize(&db, None, floor)
195 .await;
196 let (db, _) = db.apply_batch(batch).await.unwrap();
197 db
198 }
199
200 #[test_traced]
202 fn test_fixed_start_sync_overlaps_work() {
203 deterministic::Runner::default().start(|ctx| async move {
204 let pending = PendingSyncs::default();
205 let open = open_delayed_db(&ctx, "delayed", "start-sync-overlap", &pending);
206 let mut db = drive_pending_syncs(&pending, open).await.unwrap();
207 let key0 = Sha256::fill(1u8);
208 let value0 = Sha256::fill(2u8);
209 let floor = db.inactivity_floor_loc();
210 db = apply_set(db, key0, value0, floor).await;
211
212 let starts_before = pending.starts();
213 let entered_before = pending.entered();
214 let completions_before = pending.completions();
215 let handle;
216 (db, handle) = db.start_sync().await.unwrap();
217 assert!(pending.starts() > starts_before);
218 assert_eq!(pending.completions(), completions_before);
219
220 let waiter = ctx
222 .child("await_sync")
223 .spawn(|_| async move { handle.await.unwrap() });
224 while pending.entered() == entered_before {
225 reschedule().await;
226 }
227
228 assert_eq!(db.get(&key0).await.unwrap(), Some(value0));
230 let key1 = Sha256::fill(3u8);
231 let value1 = Sha256::fill(4u8);
232 let floor = db.inactivity_floor_loc();
233 db = apply_set(db, key1, value1, floor).await;
234 assert_eq!(
235 pending.completions(),
236 completions_before,
237 "the database made progress while the sync was still in flight"
238 );
239
240 pending.unblock();
241 waiter.await.unwrap();
242
243 let handle;
245 (db, handle) = db.start_sync().await.unwrap();
246 handle.await.unwrap();
247 let root = db.root();
248 drop(db);
249
250 let db = open_delayed_db(&ctx, "reopen", "start-sync-overlap", &pending)
251 .await
252 .unwrap();
253 assert_eq!(db.root(), root);
254 assert_eq!(db.get(&key1).await.unwrap(), Some(value1));
255 db.destroy().await.unwrap();
256 });
257 }
258
259 #[test_traced]
262 fn test_fixed_start_sync_failure_propagates() {
263 deterministic::Runner::default().start(|ctx| async move {
264 let pending = PendingSyncs::default();
266 pending.unblock();
267 let mut db = open_delayed_db(&ctx, "delayed", "start-sync-fail", &pending)
268 .await
269 .unwrap();
270 let floor = db.inactivity_floor_loc();
271 db = apply_set(db, Sha256::fill(1u8), Sha256::fill(2u8), floor).await;
272
273 pending.arm_fail();
275
276 let handle;
277 (db, handle) = db.start_sync().await.unwrap();
278 assert!(
279 handle.await.is_err(),
280 "the sync handle surfaces the failure"
281 );
282 let starts_before = pending.starts();
283 assert!(
285 db.commit().await.is_err(),
286 "the next durability op surfaces the failed in-flight sync"
287 );
288 assert_eq!(
289 pending.starts(),
290 starts_before,
291 "the surfaced error is the retained failure, not a fresh sync's"
292 );
293 });
294 }
295
296 #[test_traced]
298 fn test_fixed_start_sync_recovery() {
299 deterministic::Runner::default().start(|ctx| async move {
300 let pending = PendingSyncs::default();
301 pending.unblock();
302 let mut db = open_delayed_db(&ctx, "delayed", "start-sync-recovery", &pending)
303 .await
304 .unwrap();
305 let key = Sha256::fill(1u8);
306 let value = Sha256::fill(2u8);
307 let floor = db.inactivity_floor_loc();
308 db = apply_set(db, key, value, floor).await;
309
310 let handle;
311 (db, handle) = db.start_sync().await.unwrap();
312 handle.await.unwrap();
313 let root = db.root();
314 drop(db);
315
316 let db = open_delayed_db(&ctx, "reopen", "start-sync-recovery", &pending)
317 .await
318 .unwrap();
319 assert_eq!(db.root(), root);
320 assert_eq!(db.get(&key).await.unwrap(), Some(value));
321 db.destroy().await.unwrap();
322 });
323 }
324
325 #[test_traced]
327 fn test_fixed_start_sync_prune_waits() {
328 deterministic::Runner::default().start(|ctx| async move {
329 let pending = PendingSyncs::default();
330 let open = open_delayed_db(&ctx, "delayed", "start-sync-prune", &pending);
331 let mut db = drive_pending_syncs(&pending, open).await.unwrap();
332 db = apply_set(db, Sha256::fill(1u8), Sha256::fill(2u8), Location::new(0)).await;
334 db = apply_set(db, Sha256::fill(3u8), Sha256::fill(4u8), Location::new(2)).await;
335
336 let starts_before = pending.starts();
337 let handle;
338 (db, handle) = db.start_sync().await.unwrap();
339 assert!(pending.starts() > starts_before);
340
341 let floor = db.inactivity_floor_loc();
342 assert!(*floor > 0);
343 let db = {
344 let mut prune = std::pin::pin!(db.prune(floor));
345 assert!(
346 prune.as_mut().now_or_never().is_none(),
347 "prune proceeded while the started sync was pending"
348 );
349 pending.unblock();
350 prune.await.unwrap()
351 };
352 handle.await.unwrap();
353 db.destroy().await.unwrap();
354 });
355 }
356
357 #[test_traced]
359 fn test_fixed_start_sync_rewind_waits() {
360 deterministic::Runner::default().start(|ctx| async move {
361 let pending = PendingSyncs::default();
362 let open = open_delayed_db(&ctx, "delayed", "start-sync-rewind", &pending);
363 let mut db = drive_pending_syncs(&pending, open).await.unwrap();
364 db = apply_set(db, Sha256::fill(1u8), Sha256::fill(2u8), Location::new(0)).await;
365 db = drive_pending_syncs(&pending, db.commit()).await.unwrap();
366 let committed_root = db.root();
367 let committed_size = db.bounds().end;
368 db = apply_set(db, Sha256::fill(3u8), Sha256::fill(4u8), Location::new(0)).await;
369
370 let handle;
371 (db, handle) = db.start_sync().await.unwrap();
372
373 let db = {
374 let mut rewind = std::pin::pin!(db.rewind(committed_size));
375 assert!(
376 rewind.as_mut().now_or_never().is_none(),
377 "rewind proceeded while the started sync was pending"
378 );
379 pending.unblock();
380 rewind.await.unwrap()
381 };
382 handle.await.unwrap();
383 assert_eq!(db.root(), committed_root);
384 db.destroy().await.unwrap();
385 });
386 }
387
388 #[test_traced("INFO")]
389 fn test_fixed_metrics() {
390 deterministic::Runner::default().start(|ctx| async move {
391 let db = open_db::<mmr::Family>(ctx.child("db")).await;
392 let key = Sha256::fill(1u8);
393 let value = Sha256::fill(2u8);
394 let floor = db.inactivity_floor_loc();
395 let batch = db
396 .new_batch()
397 .set(key, value)
398 .merkleize(&db, None, floor)
399 .await;
400 let (db, _) = db.apply_batch(batch).await.unwrap();
401 assert_eq!(db.get(&key).await.unwrap(), Some(value));
402 assert_eq!(db.get_many(&[&key]).await.unwrap(), vec![Some(value)]);
403 let db = db.commit().await.unwrap();
404 let db = db.sync().await.unwrap();
405 let (db, handle) = db.start_sync().await.unwrap();
406 handle.await.unwrap();
407 let _db = db.prune(crate::merkle::Location::new(0)).await.unwrap();
408
409 let metrics = ctx.encode();
410 for expected in [
411 "db_size 3",
412 "db_pruning_boundary 0",
413 "db_retained 3",
414 "db_inactivity_floor 0",
415 "db_last_commit 2",
416 "db_get_calls_total 1",
417 "db_get_many_calls_total 1",
418 "db_lookups_requested_total 2",
419 "db_apply_batch_calls_total 1",
420 "db_operations_applied_total 2",
421 "db_commit_calls_total 1",
422 "db_sync_calls_total 1",
423 "db_start_sync_calls_total 1",
424 "db_prune_calls_total 1",
425 "db_get_duration_count 1",
426 "db_get_many_duration_count 1",
427 "db_apply_batch_duration_count 1",
428 "db_commit_duration_count 1",
429 "db_sync_duration_count 1",
430 "db_prune_duration_count 1",
431 ] {
432 assert!(metrics.contains(expected), "missing {expected}\n{metrics}");
433 }
434 });
435 }
436
437 #[allow(clippy::type_complexity)]
438 fn open<F: Family>(
439 ctx: deterministic::Context,
440 ) -> Pin<
441 Box<
442 dyn Future<
443 Output = Db<
444 F,
445 deterministic::Context,
446 Digest,
447 Digest,
448 Sha256,
449 TwoCap,
450 Sequential,
451 >,
452 > + Send,
453 >,
454 > {
455 Box::pin(open_db::<F>(ctx))
456 }
457
458 fn is_send<T: Send>(_: T) {}
459
460 #[allow(dead_code)]
461 fn assert_db_futures_are_send(
462 db: Db<mmr::Family, deterministic::Context, Digest, Digest, Sha256, TwoCap, Sequential>,
463 key: Digest,
464 loc: crate::merkle::mmr::Location,
465 ) {
466 is_send(db.get(&key));
467 is_send(db.get_metadata());
468 is_send(db.proof(loc, NZU64!(1)));
469 is_send(db.sync());
470 }
471
472 #[allow(dead_code)]
473 fn assert_rewind_is_send(
474 db: Db<mmr::Family, deterministic::Context, Digest, Digest, Sha256, TwoCap, Sequential>,
475 loc: crate::merkle::mmr::Location,
476 ) {
477 is_send(db.rewind(loc));
478 }
479
480 fn small_sections_config(
481 suffix: &str,
482 pooler: &impl BufferPooler,
483 ) -> Config<TwoCap, Sequential> {
484 let mut cfg = config(suffix, pooler);
485 cfg.log.items_per_blob = NZU64!(1);
486 cfg
487 }
488
489 async fn open_small_sections_db<F: Family>(
490 context: deterministic::Context,
491 ) -> Db<F, deterministic::Context, Digest, Digest, Sha256, TwoCap, Sequential> {
492 let cfg = small_sections_config("partition", &context);
493 Db::init(context, cfg).await.unwrap()
494 }
495
496 #[allow(clippy::type_complexity)]
497 fn open_small_sections<F: Family>(
498 ctx: deterministic::Context,
499 ) -> Pin<
500 Box<
501 dyn Future<
502 Output = Db<
503 F,
504 deterministic::Context,
505 Digest,
506 Digest,
507 Sha256,
508 TwoCap,
509 Sequential,
510 >,
511 > + Send,
512 >,
513 > {
514 Box::pin(open_small_sections_db::<F>(ctx))
515 }
516
517 immutable_tests! {
518 test_fixed_empty => run_empty, open;
519 test_fixed_build_basic => run_build_basic, open;
520 test_fixed_proof_verify => run_proof_verify, open;
521 test_fixed_prune => run_prune, open;
522 test_fixed_batch_chain => run_batch_chain, open;
523 test_fixed_operations_match_applied_log => run_operations_match_applied_log, open;
524 test_fixed_build_and_authenticate => run_build_and_authenticate, open;
525 test_fixed_recovery_from_failed_merkle_sync => run_recovery_from_failed_merkle_sync, open;
526 test_fixed_recovery_from_failed_log_sync => run_recovery_from_failed_log_sync, open;
527 test_fixed_pruning => run_pruning, open;
528 test_fixed_prune_beyond_floor => run_prune_beyond_floor, open;
529 test_fixed_batch_get_read_through => run_batch_get_read_through, open;
530 test_fixed_batch_stacked_get => run_batch_stacked_get, open;
531 test_fixed_batch_stacked_apply => run_batch_stacked_apply, open;
532 test_fixed_batch_speculative_root => run_batch_speculative_root, open;
533 test_fixed_merkleized_batch_get => run_merkleized_batch_get, open;
534 test_fixed_batch_sequential_apply => run_batch_sequential_apply, open;
535 test_fixed_batch_many_sequential => run_batch_many_sequential, open;
536 test_fixed_batch_empty_batch => run_batch_empty_batch, open;
537 test_fixed_batch_chained_merkleized_get => run_batch_chained_merkleized_get, open;
538 test_fixed_batch_large => run_batch_large, open;
539 test_fixed_batch_chained_key_override => run_batch_chained_key_override, open;
540 test_fixed_batch_sequential_key_override => run_batch_sequential_key_override, open_small_sections;
541 test_fixed_batch_metadata => run_batch_metadata, open;
542 test_fixed_stale_batch_rejected => run_stale_batch_rejected, open;
543 test_fixed_stale_batch_chained => run_stale_batch_chained, open;
544 test_fixed_sequential_commit_parent_then_child => run_sequential_commit_parent_then_child, open;
545 test_fixed_stale_batch_child_applied_before_parent => run_stale_batch_child_applied_before_parent, open;
546 test_fixed_child_root_matches_pending_and_committed => run_child_root_matches_pending_and_committed, open;
547 test_fixed_to_batch => run_to_batch, open;
548 test_fixed_rewind_recovery => run_rewind_recovery, open;
549 test_fixed_rewind_pruned_target_errors => run_rewind_pruned_target_errors, open_small_sections;
550 test_fixed_inactivity_floor_tracking => run_inactivity_floor_tracking, open;
551 test_fixed_floor_monotonicity => run_floor_monotonicity, open;
552 test_fixed_floor_monotonicity_violation => run_floor_monotonicity_violation, open;
553 test_fixed_floor_beyond_size => run_floor_beyond_size, open;
554 test_fixed_chained_ancestor_floor_regression => run_chained_ancestor_floor_regression, open;
555 test_fixed_chained_ancestor_floor_beyond_size => run_chained_ancestor_floor_beyond_size, open;
556 test_fixed_rewind_restores_floor => run_rewind_restores_floor, open;
557 test_fixed_single_commit_live_set => run_single_commit_live_set, open;
558 test_fixed_rewind_after_reopen_with_floor_change => run_rewind_after_reopen_with_floor_change, open;
559 test_fixed_rewind_after_reopen_partial_floor_gap => run_rewind_after_reopen_partial_floor_gap, open;
560 test_fixed_commit_after_sync_recovery => run_commit_after_sync_recovery, open;
561 test_fixed_prune_after_uncommitted_apply_batch_recovery => run_prune_after_uncommitted_apply_batch_recovery, open;
562 test_fixed_rewind_preserves_collision_bucket => run_rewind_preserves_collision_bucket, open;
563 test_fixed_get_many => run_get_many, open;
564 test_fixed_get_many_unexpected_data => run_get_many_unexpected_data, open;
565 test_fixed_rewind_after_reopen_repeated_key_gap => run_rewind_after_reopen_repeated_key_gap, open;
566 test_fixed_rewind_after_reopen_mixed_gap_retained => run_rewind_after_reopen_mixed_gap_retained, open;
567 test_fixed_rewind_repeated_key_live => run_rewind_repeated_key_live, open;
568 test_fixed_rewind_after_reopen_repeated_key_retained => run_rewind_after_reopen_repeated_key_retained, open;
569 }
570
571 #[boxed]
572 async fn assert_compact_root_compatibility<F: Family>(ctx: deterministic::Context) {
573 let db = open_db::<F>(ctx.child("db")).await;
574 let compact = open_compact::<F>(ctx.child("compact")).await;
575 assert_eq!(db.root(), compact.root());
576
577 let k1 = Sha256::fill(1u8);
578 let v1 = Sha256::fill(11u8);
579 let k2 = Sha256::fill(2u8);
580 let v2 = Sha256::fill(22u8);
581 let metadata = Sha256::fill(99u8);
582
583 let floor = db.inactivity_floor_loc();
584 let retained = db
585 .new_batch()
586 .set(k1, v1)
587 .set(k2, v2)
588 .merkleize(&db, Some(metadata), floor)
589 .await;
590 let compact_batch = compact
591 .new_batch()
592 .set(k1, v1)
593 .set(k2, v2)
594 .merkleize(&compact, Some(metadata), floor)
595 .await;
596
597 assert_eq!(retained.root(), compact_batch.root());
598
599 let (db, _) = db.apply_batch(retained).await.unwrap();
600 let (compact, _) = compact.apply_batch(compact_batch).await.unwrap();
601 let db = db.commit().await.unwrap();
602 let compact = compact.sync().await.unwrap();
603
604 assert_eq!(db.root(), compact.root());
605 assert_eq!(compact.get_metadata(), Some(metadata));
606
607 drop(compact);
608 let reopened = open_compact::<F>(ctx.child("reopen")).await;
609 assert_eq!(db.root(), reopened.root());
610 assert_eq!(reopened.get_metadata(), Some(metadata));
611
612 reopened.destroy().await.unwrap();
613 db.destroy().await.unwrap();
614 }
615
616 #[test_traced("INFO")]
617 fn test_fixed_compact_root_compatibility() {
618 let executor = deterministic::Runner::default();
619 executor.start(|ctx| async move {
620 assert_compact_root_compatibility::<mmr::Family>(ctx).await;
621 });
622 }
623
624 #[test_traced("INFO")]
625 fn test_fixed_compact_root_compatibility_mmb() {
626 let executor = deterministic::Runner::default();
627 executor.start(|ctx| async move {
628 assert_compact_root_compatibility::<mmb::Family>(ctx).await;
629 });
630 }
631}