1use std::collections::{BTreeMap, BTreeSet, HashSet};
2use std::mem::size_of;
3use std::num::NonZeroUsize;
4use std::ops::{Deref, DerefMut};
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use anyhow::Context;
9use diesel::{Connection, SqliteConnection};
10use miden_node_proto::domain::account::AccountInfo;
11use miden_node_utils::limiter::{
12 MAX_RESPONSE_PAYLOAD_BYTES,
13 QueryParamLimiter,
14 QueryParamNoteCommitmentLimit,
15};
16use miden_node_utils::tracing::miden_instrument;
17use miden_protocol::Word;
18use miden_protocol::account::{AccountHeader, AccountId, AccountStorageHeader, StorageMapKey};
19use miden_protocol::asset::{Asset, AssetId};
20use miden_protocol::block::{
21 BlockHeader,
22 BlockNoteIndex,
23 BlockNumber,
24 BlockSignatures,
25 SignedBlock,
26};
27use miden_protocol::crypto::merkle::SparseMerklePath;
28use miden_protocol::note::{
29 NoteAttachments,
30 NoteDetails,
31 NoteId,
32 NoteInclusionProof,
33 NoteMetadata,
34 NoteScript,
35 Nullifier,
36};
37use miden_protocol::transaction::TransactionHeader;
38use miden_protocol::utils::serde::Deserializable;
39use tracing::info;
40
41use crate::db::migrations::{migrate_database, verify_latest_schema};
42use crate::db::models::conv::SqlTypeConvert;
43use crate::db::models::queries;
44pub use crate::db::models::queries::{
45 AccountCommitmentsPage,
46 NullifiersPage,
47 PublicAccountIdsPage,
48 PublicAccountStateRootsPage,
49};
50use crate::db::models::queries::{
51 BlockHeaderCommitment,
52 PrecomputedPublicAccountStates,
53 StorageMapValuesPage,
54};
55use crate::errors::{DatabaseError, NoteSyncError};
56use crate::genesis::GenesisBlock;
57use crate::state::{ScopedBlockNum, ScopedBlockRange};
58use crate::{COMPONENT, LOG_TARGET};
59
60const STORAGE_MAP_VALUE_PER_ROW_BYTES: usize =
61 2 * size_of::<Word>() + size_of::<u32>() + size_of::<u8>();
62
63fn default_storage_map_entries_limit() -> usize {
64 MAX_RESPONSE_PAYLOAD_BYTES / STORAGE_MAP_VALUE_PER_ROW_BYTES
65}
66
67mod migrations;
68#[cfg(test)]
69pub(crate) use migrations::bootstrap_database;
70
71#[cfg(test)]
72mod tests;
73
74pub(crate) mod models;
75
76pub(crate) mod schema;
81
82pub type Result<T, E = DatabaseError> = std::result::Result<T, E>;
83
84#[derive(Copy, Clone, Debug, PartialEq, Eq)]
86pub struct DatabaseOptions {
87 pub connection_pool_size: NonZeroUsize,
89}
90
91impl Default for DatabaseOptions {
92 fn default() -> Self {
93 Self {
94 connection_pool_size: miden_node_db::default_connection_pool_size(),
95 }
96 }
97}
98
99pub struct Db {
103 db: miden_node_db::Db,
104}
105
106impl Deref for Db {
107 type Target = miden_node_db::Db;
108
109 fn deref(&self) -> &Self::Target {
110 &self.db
111 }
112}
113
114impl DerefMut for Db {
115 fn deref_mut(&mut self) -> &mut Self::Target {
116 &mut self.db
117 }
118}
119
120#[derive(Debug, Clone)]
124pub struct AccountVaultValue {
125 pub block_num: BlockNumber,
126 pub vault_key: AssetId,
127 pub asset: Option<Asset>,
129}
130
131impl AccountVaultValue {
132 pub fn from_raw_row(row: (i64, Vec<u8>, Option<Vec<u8>>)) -> Result<Self, DatabaseError> {
133 let (block_num, vault_key, asset) = row;
134 let vault_key = Word::read_from_bytes(&vault_key)?;
135 Ok(Self {
136 block_num: BlockNumber::from_raw_sql(block_num)?,
137 vault_key: AssetId::try_from(vault_key)?,
138 asset: asset.map(|b| Asset::read_from_bytes(&b)).transpose()?,
139 })
140 }
141}
142
143#[derive(Debug, PartialEq)]
144pub struct NullifierInfo {
145 pub nullifier: Nullifier,
146 pub block_num: BlockNumber,
147}
148
149impl PartialEq<(Nullifier, BlockNumber)> for NullifierInfo {
150 fn eq(&self, (nullifier, block_num): &(Nullifier, BlockNumber)) -> bool {
151 &self.nullifier == nullifier && &self.block_num == block_num
152 }
153}
154
155#[derive(Debug, PartialEq)]
156pub struct TransactionRecord {
157 pub block_num: BlockNumber,
158 pub header: TransactionHeader,
159 pub output_note_proofs: Vec<NoteSyncRecord>,
162 pub consumed_note_refs: Vec<(Nullifier, NoteId)>,
165}
166
167#[derive(Debug, Clone, PartialEq)]
168pub struct NoteRecord {
169 pub block_num: BlockNumber,
170 pub note_index: BlockNoteIndex,
171 pub note_id: Word,
172 pub metadata: NoteMetadata,
173 pub details: Option<NoteDetails>,
174 pub attachments: NoteAttachments,
175 pub inclusion_path: SparseMerklePath,
176}
177
178#[derive(Debug, PartialEq)]
179pub struct NoteSyncUpdate {
180 pub notes: Vec<NoteSyncRecord>,
181 pub block_header: BlockHeader,
182}
183
184#[derive(Debug, Clone, PartialEq)]
185pub struct NoteSyncRecord {
186 pub block_num: BlockNumber,
187 pub note_index: BlockNoteIndex,
188 pub note_id: NoteId,
189 pub metadata: NoteMetadata,
190 pub attachments: NoteAttachments,
191 pub inclusion_path: SparseMerklePath,
192}
193
194impl From<NoteRecord> for NoteSyncRecord {
195 fn from(note: NoteRecord) -> Self {
196 Self {
197 block_num: note.block_num,
198 note_index: note.note_index,
199 note_id: NoteId::from_raw(note.note_id),
200 metadata: note.metadata,
201 attachments: note.attachments,
202 inclusion_path: note.inclusion_path,
203 }
204 }
205}
206
207impl Db {
208 #[miden_instrument(
210 target = COMPONENT,
211 name = "store.database.bootstrap",
212 fields(path=%database_filepath.display())
213 err,
214 )]
215 pub fn bootstrap(database_filepath: PathBuf, genesis: GenesisBlock) -> anyhow::Result<()> {
216 migrations::bootstrap_database(&database_filepath)
217 .context("failed to bootstrap database schema")?;
218
219 let mut conn: SqliteConnection = diesel::sqlite::SqliteConnection::establish(
220 database_filepath.to_str().context("database filepath is invalid")?,
221 )
222 .context("failed to open a database connection")?;
223
224 miden_node_db::configure_connection_on_creation(&mut conn)?;
225
226 let genesis_block = genesis.into_inner();
228 conn.transaction(move |conn| {
229 models::queries::apply_block(
230 conn,
231 &genesis_block,
232 &[],
233 &PrecomputedPublicAccountStates::new(),
234 )
235 })
236 .context("failed to insert genesis block")?;
237 Ok(())
238 }
239
240 #[miden_instrument(
242 target = COMPONENT,
243 )]
244 pub async fn load(database_filepath: PathBuf) -> Result<Self, DatabaseError> {
245 Self::load_with_pool_size(database_filepath, miden_node_db::default_connection_pool_size())
246 .await
247 }
248
249 #[miden_instrument(
252 target = COMPONENT,
253 )]
254 pub async fn load_with_pool_size(
255 database_filepath: PathBuf,
256 connection_pool_size: NonZeroUsize,
257 ) -> Result<Self, DatabaseError> {
258 verify_latest_schema(&database_filepath)?;
259
260 let db = miden_node_db::Db::new_with_pool_size(&database_filepath, connection_pool_size)?;
261 info!(
262 target: LOG_TARGET,
263 sqlite= %database_filepath.display(),
264 connection_pool_size = %connection_pool_size,
265 "Connected to the database"
266 );
267
268 Ok(Self { db })
269 }
270
271 #[miden_instrument(
273 target = COMPONENT,
274 )]
275 pub fn migrate(database_filepath: impl AsRef<Path>) -> Result<(), DatabaseError> {
276 migrate_database(database_filepath.as_ref())?;
277 Ok(())
278 }
279
280 #[miden_instrument(
282 level = "debug",
283 target = COMPONENT,
284 err,
285 )]
286 pub async fn select_nullifiers_paged(
287 &self,
288 page_size: std::num::NonZeroUsize,
289 after_nullifier: Option<Nullifier>,
290 ) -> Result<NullifiersPage> {
291 self.transact("read nullifiers paged", move |conn| {
292 queries::select_nullifiers_paged(conn, page_size, after_nullifier)
293 })
294 .await
295 }
296
297 #[miden_instrument(
299 level = "debug",
300 target = COMPONENT,
301 fields(
302 prefix_len,
303 prefixes = nullifier_prefixes.len(),
304 ),
305 err,
306 )]
307 pub async fn select_nullifiers_by_prefix(
308 &self,
309 prefix_len: u32,
310 nullifier_prefixes: Vec<u32>,
311 block_range: ScopedBlockRange,
312 ) -> Result<(Vec<NullifierInfo>, BlockNumber)> {
313 let block_range = block_range.into_inner();
314 assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported");
315
316 self.transact("nullifieres by prefix", move |conn| {
317 let nullifier_prefixes =
318 Vec::from_iter(nullifier_prefixes.into_iter().map(|prefix| prefix as u16));
319 queries::select_nullifiers_by_prefix(
320 conn,
321 prefix_len as u8,
322 &nullifier_prefixes[..],
323 block_range,
324 )
325 })
326 .await
327 }
328
329 #[miden_instrument(
333 level = "debug",
334 target = COMPONENT,
335 err,
336 )]
337 pub async fn select_block_header_by_block_num(
338 &self,
339 maybe_block_number: Option<ScopedBlockNum>,
340 ) -> Result<Option<BlockHeader>> {
341 self.transact("block headers by block number", move |conn| {
342 let val = queries::select_block_header_by_block_num(
343 conn,
344 maybe_block_number.map(|block_number| *block_number),
345 )?;
346 Ok(val)
347 })
348 .await
349 }
350
351 #[miden_instrument(
354 level = "debug",
355 target = COMPONENT,
356 err,
357 )]
358 pub async fn select_block_header_and_signatures_by_block_num(
359 &self,
360 block_number: ScopedBlockNum,
361 ) -> Result<Option<(BlockHeader, BlockSignatures)>> {
362 self.transact("block headers and signatures by block number", move |conn| {
363 let val =
364 queries::select_block_header_and_signatures_by_block_num(conn, *block_number)?;
365 Ok(val)
366 })
367 .await
368 }
369
370 #[miden_instrument(
372 level = "debug",
373 target = COMPONENT,
374 err,
375 )]
376 pub async fn select_block_headers(
377 &self,
378 blocks: impl Iterator<Item = ScopedBlockNum> + Send + 'static,
379 ) -> Result<Vec<BlockHeader>> {
380 self.transact("block headers from given block numbers", move |conn| {
381 let raw = queries::select_block_headers(conn, blocks.map(|block| *block))?;
382 Ok(raw)
383 })
384 .await
385 }
386
387 #[miden_instrument(
389 level = "debug",
390 target = COMPONENT,
391 err,
392 )]
393 pub async fn select_all_block_header_commitments(&self) -> Result<Vec<BlockHeaderCommitment>> {
394 self.transact("all block headers", |conn| {
395 let raw = queries::select_all_block_header_commitments(conn)?;
396 Ok(raw)
397 })
398 .await
399 }
400
401 #[miden_instrument(
403 level = "debug",
404 target = COMPONENT,
405 err,
406 )]
407 pub async fn select_account_commitments_paged(
408 &self,
409 page_size: std::num::NonZeroUsize,
410 after_account_id: Option<AccountId>,
411 ) -> Result<AccountCommitmentsPage> {
412 self.transact("read account commitments paged", move |conn| {
413 queries::select_account_commitments_paged(conn, page_size, after_account_id)
414 })
415 .await
416 }
417
418 #[miden_instrument(
420 level = "debug",
421 target = COMPONENT,
422 err,
423 )]
424 pub async fn select_public_account_ids_paged(
425 &self,
426 page_size: std::num::NonZeroUsize,
427 after_account_id: Option<AccountId>,
428 ) -> Result<PublicAccountIdsPage> {
429 self.transact("read public account IDs paged", move |conn| {
430 queries::select_public_account_ids_paged(conn, page_size, after_account_id)
431 })
432 .await
433 }
434
435 #[miden_instrument(
437 level = "debug",
438 target = COMPONENT,
439 err,
440 )]
441 pub async fn select_public_account_state_roots_paged(
442 &self,
443 page_size: std::num::NonZeroUsize,
444 after_account_id: Option<AccountId>,
445 ) -> Result<PublicAccountStateRootsPage> {
446 self.transact("read public account state roots paged", move |conn| {
447 queries::select_public_account_state_roots_paged(conn, page_size, after_account_id)
448 })
449 .await
450 }
451
452 #[miden_instrument(
454 level = "debug",
455 target = COMPONENT,
456 err,
457 )]
458 pub async fn select_account(&self, id: AccountId) -> Result<AccountInfo> {
459 self.transact("Get account details", move |conn| queries::select_account(conn, id))
460 .await
461 }
462
463 #[miden_instrument(
465 level = "debug",
466 target = COMPONENT,
467 err,
468 )]
469 pub async fn select_network_accounts_subset(
470 &self,
471 account_ids: Vec<AccountId>,
472 ) -> Result<HashSet<AccountId>> {
473 self.transact("Filter network accounts subset", move |conn| {
474 queries::select_network_accounts_subset(conn, &account_ids)
475 })
476 .await
477 }
478
479 #[miden_instrument(
483 target = COMPONENT,
484 )]
485 pub async fn select_account_code_by_commitment(
486 &self,
487 code_commitment: Word,
488 ) -> Result<Option<Vec<u8>>> {
489 self.transact("Get account code by commitment", move |conn| {
490 queries::select_account_code_by_commitment(conn, code_commitment)
491 })
492 .await
493 }
494
495 #[miden_instrument(
500 target = COMPONENT,
501 )]
502 pub async fn select_account_header_with_storage_header_at_block(
503 &self,
504 account_id: AccountId,
505 block_num: ScopedBlockNum,
506 ) -> Result<Option<(AccountHeader, AccountStorageHeader)>> {
507 self.transact("Get account header with storage header at block", move |conn| {
508 queries::select_account_header_with_storage_header_at_block(
509 conn, account_id, *block_num,
510 )
511 })
512 .await
513 }
514
515 #[miden_instrument(
516 level = "debug",
517 target = COMPONENT,
518 err,
519 )]
520 pub async fn get_note_sync_multi(
521 &self,
522 block_range: ScopedBlockRange,
523 note_tags: Arc<[u32]>,
524 ) -> Result<Vec<NoteSyncUpdate>, NoteSyncError> {
525 let block_range = block_range.into_inner();
526 self.transact("notes sync task", move |conn| {
527 queries::get_note_sync_multi(conn, ¬e_tags, block_range, MAX_RESPONSE_PAYLOAD_BYTES)
528 })
529 .await
530 }
531
532 #[miden_instrument(
535 level = "debug",
536 target = COMPONENT,
537 err,
538 )]
539 pub async fn select_notes_by_id(&self, note_ids: Vec<NoteId>) -> Result<Vec<NoteRecord>> {
540 self.transact("note by id", move |conn| {
541 queries::select_notes_by_id(conn, note_ids.as_slice())
542 })
543 .await
544 }
545
546 #[miden_instrument(
549 level = "debug",
550 target = COMPONENT,
551 err,
552 )]
553 pub async fn select_existing_note_commitments(
554 &self,
555 note_commitments: Vec<Word>,
556 up_to_block: ScopedBlockNum,
557 ) -> Result<HashSet<Word>> {
558 self.transact("note by commitment", move |conn| {
559 queries::select_existing_note_commitments(
560 conn,
561 note_commitments.as_slice(),
562 *up_to_block,
563 )
564 })
565 .await
566 }
567
568 #[miden_instrument(
571 level = "debug",
572 target = COMPONENT,
573 err,
574 )]
575 pub async fn select_note_inclusion_proofs(
576 &self,
577 note_commitments: BTreeSet<Word>,
578 up_to_block: ScopedBlockNum,
579 ) -> Result<BTreeMap<NoteId, NoteInclusionProof>> {
580 self.transact("block note inclusion proofs by commitment", move |conn| {
581 models::queries::select_note_inclusion_proofs(conn, ¬e_commitments, *up_to_block)
582 })
583 .await
584 }
585
586 #[miden_instrument(
602 target = COMPONENT,
603 err,
604 )]
605 pub(crate) async fn apply_block(
606 &self,
607 signed_block: SignedBlock,
608 notes: Vec<(NoteRecord, Option<Nullifier>)>,
609 precomputed_public_states: PrecomputedPublicAccountStates,
610 unresolved_note_nullifiers: Vec<Nullifier>,
611 prune_tip: BlockNumber,
612 ) -> Result<BTreeMap<Nullifier, NoteId>> {
613 self.transact("apply block", move |conn| {
614 models::queries::apply_block(conn, &signed_block, ¬es, &precomputed_public_states)?;
615 models::queries::prune_history(conn, prune_tip)?;
616
617 let mut resolved_note_ids = BTreeMap::new();
618 for chunk in unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) {
619 match queries::select_note_ids_by_nullifier(conn, chunk) {
620 Ok(note_ids) => resolved_note_ids.extend(note_ids),
621 Err(err) => {
622 tracing::warn!(
623 target: COMPONENT,
624 %err,
625 nullifiers.count = chunk.len(),
626 "Failed to resolve consumed note IDs for lifecycle events",
627 );
628 break;
629 },
630 }
631 }
632
633 Ok(resolved_note_ids)
634 })
635 .await
636 }
637
638 pub(crate) async fn select_storage_map_sync_values(
643 &self,
644 account_id: AccountId,
645 block_range: ScopedBlockRange,
646 entries_limit: Option<usize>,
647 ) -> Result<StorageMapValuesPage> {
648 let block_range = block_range.into_inner();
649 let entries_limit = entries_limit.unwrap_or_else(default_storage_map_entries_limit);
650
651 self.transact("select storage map sync values", move |conn| {
652 models::queries::select_account_storage_map_values_paged(
653 conn,
654 account_id,
655 block_range,
656 entries_limit,
657 )
658 })
659 .await
660 }
661
662 #[miden_instrument(
671 target = COMPONENT,
672 )]
673 pub(crate) async fn reconstruct_storage_map_from_db(
674 &self,
675 account_id: AccountId,
676 slot_name: miden_protocol::account::StorageSlotName,
677 block_num: ScopedBlockNum,
678 entries_limit: Option<usize>,
679 ) -> Result<miden_node_proto::domain::account::AccountStorageMapDetails> {
680 use miden_node_proto::domain::account::{AccountStorageMapDetails, StorageMapEntries};
681 use miden_protocol::EMPTY_WORD;
682
683 let mut values = Vec::new();
686 let mut block_range_start = BlockNumber::GENESIS;
687 let entries_limit = entries_limit.unwrap_or_else(default_storage_map_entries_limit);
688
689 let mut page = self
690 .select_storage_map_sync_values(
691 account_id,
692 block_num.range_from(block_range_start),
693 Some(entries_limit),
694 )
695 .await?;
696
697 values.extend(page.values);
698 let mut last_block_included = page.last_block_included;
699
700 if values.is_empty() && last_block_included == block_range_start {
703 return Ok(AccountStorageMapDetails::limit_exceeded(slot_name));
704 }
705
706 loop {
707 if page.last_block_included == *block_num
708 || page.last_block_included < block_range_start
709 {
710 break;
711 }
712
713 block_range_start = page.last_block_included.child();
714 page = self
715 .select_storage_map_sync_values(
716 account_id,
717 block_num.range_from(block_range_start),
718 Some(entries_limit),
719 )
720 .await?;
721
722 if page.last_block_included <= last_block_included {
723 return Ok(AccountStorageMapDetails::limit_exceeded(slot_name));
724 }
725
726 last_block_included = page.last_block_included;
727 values.extend(page.values);
728 }
729
730 if page.last_block_included != *block_num {
731 return Ok(AccountStorageMapDetails::limit_exceeded(slot_name));
732 }
733
734 let mut latest_values = BTreeMap::<StorageMapKey, Word>::new();
736 for value in values {
737 if value.slot_name == slot_name {
738 let raw_key = value.key;
739 latest_values.insert(raw_key, value.value);
740 }
741 }
742
743 latest_values.retain(|_, v| *v != EMPTY_WORD);
745
746 if latest_values.len() > AccountStorageMapDetails::MAX_RETURN_ENTRIES {
747 return Ok(AccountStorageMapDetails::limit_exceeded(slot_name));
748 }
749
750 let entries = Vec::from_iter(latest_values.into_iter());
751 Ok(AccountStorageMapDetails {
752 slot_name,
753 entries: StorageMapEntries::AllEntries(entries),
754 })
755 }
756
757 #[miden_instrument(
762 target = COMPONENT,
763 )]
764 pub async fn select_account_vault_at_block(
765 &self,
766 account_id: AccountId,
767 block_num: ScopedBlockNum,
768 ) -> Result<Vec<Asset>, DatabaseError> {
769 self.transact("select account vault at block", move |conn| {
770 queries::select_account_vault_at_block(conn, account_id, *block_num)
771 })
772 .await
773 }
774
775 pub async fn get_account_vault_sync(
776 &self,
777 account_id: AccountId,
778 block_range: ScopedBlockRange,
779 ) -> Result<(BlockNumber, Vec<AccountVaultValue>)> {
780 let block_range = block_range.into_inner();
781 self.transact("account vault sync", move |conn| {
782 queries::select_account_vault_assets(conn, account_id, block_range)
783 })
784 .await
785 }
786
787 pub async fn select_note_script_by_root(&self, root: Word) -> Result<Option<NoteScript>> {
789 self.transact("note script by root", move |conn| {
790 queries::select_note_script_by_root(conn, root)
791 })
792 .await
793 }
794
795 pub async fn select_transactions_records(
802 &self,
803 account_ids: Vec<AccountId>,
804 block_range: ScopedBlockRange,
805 ) -> Result<(BlockNumber, Vec<TransactionRecord>)> {
806 let block_range = block_range.into_inner();
807 self.transact("full transactions records", move |conn| {
808 queries::select_transactions_records(conn, &account_ids, block_range)
809 })
810 .await
811 }
812}