kaspa_consensus/consensus/
storage.rs

1use crate::{
2    config::Config,
3    model::stores::{
4        acceptance_data::DbAcceptanceDataStore,
5        block_transactions::DbBlockTransactionsStore,
6        block_window_cache::BlockWindowCacheStore,
7        daa::DbDaaStore,
8        depth::DbDepthStore,
9        ghostdag::{CompactGhostdagData, DbGhostdagStore},
10        headers::{CompactHeaderData, DbHeadersStore},
11        headers_selected_tip::DbHeadersSelectedTipStore,
12        past_pruning_points::DbPastPruningPointsStore,
13        pruning::DbPruningStore,
14        pruning_utxoset::PruningUtxosetStores,
15        reachability::{DbReachabilityStore, ReachabilityData},
16        relations::DbRelationsStore,
17        selected_chain::DbSelectedChainStore,
18        statuses::DbStatusesStore,
19        tips::DbTipsStore,
20        utxo_diffs::DbUtxoDiffsStore,
21        utxo_multisets::DbUtxoMultisetsStore,
22        virtual_state::{LkgVirtualState, VirtualStores},
23        DB,
24    },
25    processes::{ghostdag::ordering::SortableBlock, reachability::inquirer as reachability, relations},
26};
27
28use super::cache_policy_builder::CachePolicyBuilder as PolicyBuilder;
29use itertools::Itertools;
30use kaspa_consensus_core::{blockstatus::BlockStatus, BlockHashSet};
31use kaspa_database::registry::DatabaseStorePrefixes;
32use kaspa_hashes::Hash;
33use parking_lot::RwLock;
34use std::{ops::DerefMut, sync::Arc};
35
36pub struct ConsensusStorage {
37    // DB
38    db: Arc<DB>,
39
40    // Locked stores
41    pub statuses_store: Arc<RwLock<DbStatusesStore>>,
42    pub relations_stores: Arc<RwLock<Vec<DbRelationsStore>>>,
43    pub reachability_store: Arc<RwLock<DbReachabilityStore>>,
44    pub reachability_relations_store: Arc<RwLock<DbRelationsStore>>,
45    pub pruning_point_store: Arc<RwLock<DbPruningStore>>,
46    pub headers_selected_tip_store: Arc<RwLock<DbHeadersSelectedTipStore>>,
47    pub body_tips_store: Arc<RwLock<DbTipsStore>>,
48    pub pruning_utxoset_stores: Arc<RwLock<PruningUtxosetStores>>,
49    pub virtual_stores: Arc<RwLock<VirtualStores>>,
50    pub selected_chain_store: Arc<RwLock<DbSelectedChainStore>>,
51
52    // Append-only stores
53    pub ghostdag_stores: Arc<Vec<Arc<DbGhostdagStore>>>,
54    pub ghostdag_primary_store: Arc<DbGhostdagStore>,
55    pub headers_store: Arc<DbHeadersStore>,
56    pub block_transactions_store: Arc<DbBlockTransactionsStore>,
57    pub past_pruning_points_store: Arc<DbPastPruningPointsStore>,
58    pub daa_excluded_store: Arc<DbDaaStore>,
59    pub depth_store: Arc<DbDepthStore>,
60
61    // Utxo-related stores
62    pub utxo_diffs_store: Arc<DbUtxoDiffsStore>,
63    pub utxo_multisets_store: Arc<DbUtxoMultisetsStore>,
64    pub acceptance_data_store: Arc<DbAcceptanceDataStore>,
65
66    // Block window caches
67    pub block_window_cache_for_difficulty: Arc<BlockWindowCacheStore>,
68    pub block_window_cache_for_past_median_time: Arc<BlockWindowCacheStore>,
69
70    // "Last Known Good" caches
71    /// The "last known good" virtual state. To be used by any logic which does not want to wait
72    /// for a possible virtual state write to complete but can rather settle with the last known state
73    pub lkg_virtual_state: LkgVirtualState,
74}
75
76impl ConsensusStorage {
77    pub fn new(db: Arc<DB>, config: Arc<Config>) -> Arc<Self> {
78        let scale_factor = config.ram_scale;
79        let scaled = |s| (s as f64 * scale_factor) as usize;
80
81        let params = &config.params;
82        let perf_params = &config.perf;
83
84        // Lower and upper bounds
85        let pruning_depth = params.pruning_depth as usize;
86        let pruning_size_for_caches = (params.pruning_depth + params.finality_depth) as usize; // Upper bound for any block/header related data
87        let level_lower_bound = 2 * params.pruning_proof_m as usize; // Number of items lower bound for level-related caches
88
89        // Budgets in bytes. All byte budgets overall sum up to ~1GB of memory (which obviously takes more low level alloc space)
90        let daa_excluded_budget = scaled(30_000_000);
91        let statuses_budget = scaled(30_000_000);
92        let reachability_data_budget = scaled(20_000_000);
93        let reachability_sets_budget = scaled(20_000_000); // x 2 for tree children and future covering set
94        let ghostdag_compact_budget = scaled(15_000_000);
95        let headers_compact_budget = scaled(5_000_000);
96        let parents_budget = scaled(40_000_000); // x 3 for reachability and levels
97        let children_budget = scaled(5_000_000); // x 3 for reachability and levels
98        let ghostdag_budget = scaled(80_000_000); // x 2 for levels
99        let headers_budget = scaled(80_000_000);
100        let transactions_budget = scaled(40_000_000);
101        let utxo_diffs_budget = scaled(40_000_000);
102        let block_window_budget = scaled(200_000_000); // x 2 for difficulty and median time
103        let acceptance_data_budget = scaled(40_000_000);
104
105        // Unit sizes in bytes
106        let daa_excluded_bytes = size_of::<Hash>() + size_of::<BlockHashSet>(); // Expected empty sets
107        let status_bytes = size_of::<Hash>() + size_of::<BlockStatus>();
108        let reachability_data_bytes = size_of::<Hash>() + size_of::<ReachabilityData>();
109        let ghostdag_compact_bytes = size_of::<Hash>() + size_of::<CompactGhostdagData>();
110        let headers_compact_bytes = size_of::<Hash>() + size_of::<CompactHeaderData>();
111        let difficulty_window_bytes = params.difficulty_window_size(0) * size_of::<SortableBlock>();
112        let median_window_bytes = params.past_median_time_window_size(0) * size_of::<SortableBlock>();
113
114        // Cache policy builders
115        let daa_excluded_builder =
116            PolicyBuilder::new().max_items(pruning_depth).bytes_budget(daa_excluded_budget).unit_bytes(daa_excluded_bytes).untracked(); // Required only above the pruning point
117        let statuses_builder =
118            PolicyBuilder::new().max_items(pruning_size_for_caches).bytes_budget(statuses_budget).unit_bytes(status_bytes).untracked();
119        let reachability_data_builder = PolicyBuilder::new()
120            .max_items(pruning_size_for_caches)
121            .bytes_budget(reachability_data_budget)
122            .unit_bytes(reachability_data_bytes)
123            .untracked();
124        let ghostdag_compact_builder = PolicyBuilder::new()
125            .max_items(pruning_size_for_caches)
126            .bytes_budget(ghostdag_compact_budget)
127            .unit_bytes(ghostdag_compact_bytes)
128            .min_items(level_lower_bound)
129            .untracked();
130        let headers_compact_builder = PolicyBuilder::new()
131            .max_items(pruning_size_for_caches)
132            .bytes_budget(headers_compact_budget)
133            .unit_bytes(headers_compact_bytes)
134            .untracked();
135        let parents_builder = PolicyBuilder::new()
136            .bytes_budget(parents_budget)
137            .unit_bytes(size_of::<Hash>())
138            .min_items(level_lower_bound)
139            .tracked_units();
140        let children_builder = PolicyBuilder::new()
141            .bytes_budget(children_budget)
142            .unit_bytes(size_of::<Hash>())
143            .min_items(level_lower_bound)
144            .tracked_units();
145        let reachability_sets_builder =
146            PolicyBuilder::new().bytes_budget(reachability_sets_budget).unit_bytes(size_of::<Hash>()).tracked_units();
147        let difficulty_window_builder = PolicyBuilder::new()
148            .max_items(perf_params.block_window_cache_size)
149            .bytes_budget(block_window_budget)
150            .unit_bytes(difficulty_window_bytes)
151            .untracked();
152        let median_window_builder = PolicyBuilder::new()
153            .max_items(perf_params.block_window_cache_size)
154            .bytes_budget(block_window_budget)
155            .unit_bytes(median_window_bytes)
156            .untracked();
157        let ghostdag_builder = PolicyBuilder::new().bytes_budget(ghostdag_budget).min_items(level_lower_bound).tracked_bytes();
158        let headers_builder = PolicyBuilder::new().bytes_budget(headers_budget).tracked_bytes();
159        let utxo_diffs_builder = PolicyBuilder::new().bytes_budget(utxo_diffs_budget).tracked_bytes();
160        let block_data_builder = PolicyBuilder::new().max_items(perf_params.block_data_cache_size).untracked();
161        let header_data_builder = PolicyBuilder::new().max_items(perf_params.header_data_cache_size).untracked();
162        let utxo_set_builder = PolicyBuilder::new().max_items(perf_params.utxo_set_cache_size).untracked();
163        let transactions_builder = PolicyBuilder::new().bytes_budget(transactions_budget).tracked_bytes();
164        let acceptance_data_builder = PolicyBuilder::new().bytes_budget(acceptance_data_budget).tracked_bytes();
165        let past_pruning_points_builder = PolicyBuilder::new().max_items(1024).untracked();
166
167        // TODO: consider tracking UtxoDiff byte sizes more accurately including the exact size of ScriptPublicKey
168
169        // Headers
170        let statuses_store = Arc::new(RwLock::new(DbStatusesStore::new(db.clone(), statuses_builder.build())));
171        let relations_stores = Arc::new(RwLock::new(
172            (0..=params.max_block_level)
173                .map(|level| {
174                    DbRelationsStore::new(
175                        db.clone(),
176                        level,
177                        parents_builder.downscale(level).build(),
178                        children_builder.downscale(level).build(),
179                    )
180                })
181                .collect_vec(),
182        ));
183        let reachability_store = Arc::new(RwLock::new(DbReachabilityStore::new(
184            db.clone(),
185            reachability_data_builder.build(),
186            reachability_sets_builder.build(),
187        )));
188
189        let reachability_relations_store = Arc::new(RwLock::new(DbRelationsStore::with_prefix(
190            db.clone(),
191            DatabaseStorePrefixes::ReachabilityRelations.as_ref(),
192            parents_builder.build(),
193            children_builder.build(),
194        )));
195
196        let ghostdag_stores = Arc::new(
197            (0..=params.max_block_level)
198                .map(|level| {
199                    Arc::new(DbGhostdagStore::new(
200                        db.clone(),
201                        level,
202                        ghostdag_builder.downscale(level).build(),
203                        ghostdag_compact_builder.downscale(level).build(),
204                    ))
205                })
206                .collect_vec(),
207        );
208        let ghostdag_primary_store = ghostdag_stores[0].clone();
209        let daa_excluded_store = Arc::new(DbDaaStore::new(db.clone(), daa_excluded_builder.build()));
210        let headers_store = Arc::new(DbHeadersStore::new(db.clone(), headers_builder.build(), headers_compact_builder.build()));
211        let depth_store = Arc::new(DbDepthStore::new(db.clone(), header_data_builder.build()));
212        let selected_chain_store = Arc::new(RwLock::new(DbSelectedChainStore::new(db.clone(), header_data_builder.build())));
213
214        // Pruning
215        let pruning_point_store = Arc::new(RwLock::new(DbPruningStore::new(db.clone())));
216        let past_pruning_points_store = Arc::new(DbPastPruningPointsStore::new(db.clone(), past_pruning_points_builder.build()));
217        let pruning_utxoset_stores = Arc::new(RwLock::new(PruningUtxosetStores::new(db.clone(), utxo_set_builder.build())));
218
219        // Txs
220        let block_transactions_store = Arc::new(DbBlockTransactionsStore::new(db.clone(), transactions_builder.build()));
221        let utxo_diffs_store = Arc::new(DbUtxoDiffsStore::new(db.clone(), utxo_diffs_builder.build()));
222        let utxo_multisets_store = Arc::new(DbUtxoMultisetsStore::new(db.clone(), block_data_builder.build()));
223        let acceptance_data_store = Arc::new(DbAcceptanceDataStore::new(db.clone(), acceptance_data_builder.build()));
224
225        // Tips
226        let headers_selected_tip_store = Arc::new(RwLock::new(DbHeadersSelectedTipStore::new(db.clone())));
227        let body_tips_store = Arc::new(RwLock::new(DbTipsStore::new(db.clone())));
228
229        // Block windows
230        let block_window_cache_for_difficulty = Arc::new(BlockWindowCacheStore::new(difficulty_window_builder.build()));
231        let block_window_cache_for_past_median_time = Arc::new(BlockWindowCacheStore::new(median_window_builder.build()));
232
233        // Virtual stores
234        let lkg_virtual_state = LkgVirtualState::default();
235        let virtual_stores =
236            Arc::new(RwLock::new(VirtualStores::new(db.clone(), lkg_virtual_state.clone(), utxo_set_builder.build())));
237
238        // Ensure that reachability stores are initialized
239        reachability::init(reachability_store.write().deref_mut()).unwrap();
240        relations::init(reachability_relations_store.write().deref_mut());
241
242        Arc::new(Self {
243            db,
244            statuses_store,
245            relations_stores,
246            reachability_relations_store,
247            reachability_store,
248            ghostdag_stores,
249            ghostdag_primary_store,
250            pruning_point_store,
251            headers_selected_tip_store,
252            body_tips_store,
253            headers_store,
254            block_transactions_store,
255            pruning_utxoset_stores,
256            virtual_stores,
257            selected_chain_store,
258            acceptance_data_store,
259            past_pruning_points_store,
260            daa_excluded_store,
261            depth_store,
262            utxo_diffs_store,
263            utxo_multisets_store,
264            block_window_cache_for_difficulty,
265            block_window_cache_for_past_median_time,
266            lkg_virtual_state,
267        })
268    }
269}
kaspa_consensus/consensus/storage.rs

kaspa_consensus/consensus/
storage.rs
