1use std::{
5 fs,
6 path::{Path, PathBuf},
7 sync::{atomic::Ordering, Arc},
8 time::Duration,
9};
10
11use crate::{
12 error::ConfigError,
13 nodes::{
14 leaf_node::LeafKVMeta, LeafNode, CACHE_LINE_SIZE, DISK_PAGE_SIZE, MAX_KEY_LEN,
15 MAX_LEAF_PAGE_SIZE,
16 },
17 snapshot::BfTreeMeta,
18};
19use serde::Deserialize;
20use std::sync::atomic::AtomicUsize;
21
22const DEFAULT_PROMOTION_RATE_DEBUG: usize = 50;
23const DEFAULT_PROMOTION_RATE_RLEASE: usize = 30;
24const DEFAULT_MAX_MINI_PAGE_SIZE: usize = 2048; const DEFAULT_COPY_ON_ACCESS_RATIO: f64 = 0.1;
26const DEFAULT_CIRCULAR_BUFFER_SIZE: usize = 1024 * 1024 * 32;
27const DEFAULT_MIN_RECORD_SIZE: usize = 4;
28const DEFAULT_MAX_RECORD_SIZE: usize = 1952;
29const DEFAULT_LEAF_PAGE_SIZE: usize = 4096;
30const DEFAULT_MAX_KEY_LEN: usize = 16;
31
32#[derive(Debug)]
36pub struct Config {
37 pub(crate) read_promotion_rate: AtomicUsize,
38 pub(crate) scan_promotion_rate: AtomicUsize,
39 pub(crate) storage_backend: StorageBackend,
40 pub(crate) snapshot_backend: StorageBackend,
41 pub(crate) cb_size_byte: usize,
42 pub(crate) cb_min_record_size: usize,
43 pub(crate) cb_max_record_size: usize,
44 pub(crate) leaf_page_size: usize,
45 pub(crate) cb_max_key_len: usize,
46 pub(crate) max_fence_len: usize,
47 pub(crate) cb_copy_on_access_ratio: f64,
48 pub(crate) read_record_cache: bool,
49 pub(crate) file_path: PathBuf,
50 pub(crate) use_snapshot: bool,
51 pub(crate) snapshot_version: u64,
52 pub(crate) max_mini_page_size: usize,
53 pub(crate) mini_page_binary_search: bool,
54 pub(crate) write_ahead_log: Option<Arc<WalConfig>>,
55 pub(crate) write_load_full_page: bool,
56 pub(crate) cache_only: bool,
57}
58
59impl Clone for Config {
60 fn clone(&self) -> Self {
61 Self {
62 snapshot_version: self.snapshot_version,
63 read_promotion_rate: AtomicUsize::new(self.read_promotion_rate.load(Ordering::Relaxed)),
64 scan_promotion_rate: AtomicUsize::new(self.scan_promotion_rate.load(Ordering::Relaxed)),
65 storage_backend: self.storage_backend.clone(),
66 snapshot_backend: self.snapshot_backend.clone(),
67 use_snapshot: self.use_snapshot,
68 cb_size_byte: self.cb_size_byte,
69 cb_min_record_size: self.cb_min_record_size,
70 cb_max_record_size: self.cb_max_record_size,
71 leaf_page_size: self.leaf_page_size,
72 cb_max_key_len: self.cb_max_key_len,
73 max_fence_len: self.max_fence_len,
74 cb_copy_on_access_ratio: self.cb_copy_on_access_ratio,
75 read_record_cache: self.read_record_cache,
76 file_path: self.file_path.clone(),
77 max_mini_page_size: self.max_mini_page_size,
78 mini_page_binary_search: self.mini_page_binary_search,
79 write_ahead_log: self.write_ahead_log.clone(),
80 write_load_full_page: self.write_load_full_page,
81 cache_only: self.cache_only,
82 }
83 }
84}
85
86#[derive(Debug, Default, Clone, Eq, PartialEq)]
88pub enum StorageBackend {
89 Memory,
90 #[default]
91 Std,
92 #[cfg(target_os = "linux")]
93 StdDirect,
94 #[cfg(target_os = "linux")]
95 IoUringPolling,
96 #[cfg(target_os = "linux")]
97 IoUringBlocking,
98 #[cfg(all(target_os = "linux", feature = "spdk"))]
99 Spdk,
100}
101
102#[derive(Debug, Deserialize)]
103pub struct ConfigFile {
104 pub(crate) cb_size_byte: usize,
105 pub(crate) cb_min_record_size: usize,
106 pub(crate) cb_max_record_size: usize,
107 pub(crate) cb_max_key_len: usize,
108 pub(crate) leaf_page_size: usize,
109 pub(crate) index_file_path: String,
110 pub(crate) use_snapshot: bool,
111 pub(crate) snapshot_version: u64,
112 pub(crate) backend_storage: String,
113 pub(crate) read_promotion_rate: usize,
114 pub(crate) write_load_full_page: bool,
115 pub(crate) cache_only: bool,
116}
117
118impl Default for Config {
121 fn default() -> Self {
122 let read_promotion_rate = if cfg!(debug_assertions) {
123 DEFAULT_PROMOTION_RATE_DEBUG
124 } else {
125 DEFAULT_PROMOTION_RATE_RLEASE
126 };
127 let scan_promotion_rate = if cfg!(debug_assertions) {
128 DEFAULT_PROMOTION_RATE_DEBUG
129 } else {
130 DEFAULT_PROMOTION_RATE_RLEASE
131 };
132
133 Self {
134 snapshot_version: 0,
135 read_promotion_rate: AtomicUsize::new(read_promotion_rate),
136 scan_promotion_rate: AtomicUsize::new(scan_promotion_rate),
137 use_snapshot: false,
138 cb_size_byte: DEFAULT_CIRCULAR_BUFFER_SIZE,
139 cb_min_record_size: DEFAULT_MIN_RECORD_SIZE,
140 cb_max_record_size: DEFAULT_MAX_RECORD_SIZE,
141 leaf_page_size: DEFAULT_LEAF_PAGE_SIZE,
142 cb_max_key_len: DEFAULT_MAX_KEY_LEN,
143 max_fence_len: DEFAULT_MAX_KEY_LEN * 2,
144 cb_copy_on_access_ratio: DEFAULT_COPY_ON_ACCESS_RATIO,
145 file_path: PathBuf::new(),
146 read_record_cache: true,
147 max_mini_page_size: DEFAULT_MAX_MINI_PAGE_SIZE,
148 mini_page_binary_search: true,
149 storage_backend: StorageBackend::Memory,
150 snapshot_backend: StorageBackend::Std,
151 write_ahead_log: None,
152 write_load_full_page: true,
153 cache_only: false,
154 }
155 }
156}
157impl Config {
158 pub fn new(file_path: impl AsRef<Path>, circular_buffer_size: usize) -> Self {
159 let mut config = Self::default();
160 let mut cache_only = false;
161 let storage_backend = if file_path.as_ref().to_str().unwrap().starts_with(":memory:") {
162 StorageBackend::Memory
163 } else if file_path.as_ref().to_str().unwrap().starts_with(":cache:") {
164 cache_only = true;
165 StorageBackend::Memory
166 } else {
167 StorageBackend::default()
168 };
169
170 config
171 .storage_backend(storage_backend)
172 .cache_only(cache_only)
173 .cb_size_byte(circular_buffer_size)
174 .file_path(file_path);
175
176 config
177 }
178
179 pub fn new_with_config_file<P: AsRef<Path>>(config_file_path: P) -> Self {
182 let config_file_str =
183 fs::read_to_string(config_file_path).expect("couldn't read config file");
184 let config_file: ConfigFile =
185 toml::from_str(&config_file_str).expect("Fail to parse config file");
186 let scan_promotion_rate = if cfg!(debug_assertions) {
187 DEFAULT_PROMOTION_RATE_DEBUG
188 } else {
189 DEFAULT_PROMOTION_RATE_RLEASE
190 };
191 let mut storage = StorageBackend::Memory;
192 if config_file.backend_storage == "disk" {
193 storage = StorageBackend::default();
194 }
195
196 Self {
198 snapshot_version: config_file.snapshot_version,
199 read_promotion_rate: AtomicUsize::new(config_file.read_promotion_rate),
200 scan_promotion_rate: AtomicUsize::new(scan_promotion_rate),
201 use_snapshot: config_file.use_snapshot,
202 cb_size_byte: config_file.cb_size_byte,
203 cb_min_record_size: config_file.cb_min_record_size,
204 cb_max_record_size: config_file.cb_max_record_size,
205 leaf_page_size: config_file.leaf_page_size,
206 cb_max_key_len: config_file.cb_max_key_len,
207 max_fence_len: config_file.cb_max_key_len * 2,
208 cb_copy_on_access_ratio: DEFAULT_COPY_ON_ACCESS_RATIO,
209 file_path: PathBuf::from(config_file.index_file_path),
210 read_record_cache: true,
211 max_mini_page_size: DEFAULT_MAX_MINI_PAGE_SIZE,
212 mini_page_binary_search: true,
213 storage_backend: storage,
214 snapshot_backend: StorageBackend::Std,
215 write_ahead_log: None,
216 write_load_full_page: config_file.write_load_full_page,
217 cache_only: config_file.cache_only,
218 }
219 }
220
221 pub(crate) fn new_from_snapshot(meta: &BfTreeMeta) -> Self {
223 let mut config = Self::default();
224
225 config
227 .snapshot_version(meta.snapshot_version + 1)
228 .cache_only(meta.cache_only)
229 .read_promotion_rate(meta.read_promotion_rate)
230 .scan_promotion_rate(meta.scan_promotion_rate)
231 .cb_min_record_size(meta.cb_min_record_size)
232 .cb_max_record_size(meta.cb_max_record_size)
233 .leaf_page_size(meta.leaf_page_size)
234 .cb_max_key_len(meta.cb_max_key_len)
235 .max_fence_len(meta.max_fence_len)
236 .cb_copy_on_access_ratio(meta.cb_copy_on_access_ratio)
237 .read_record_cache(meta.read_record_cache)
238 .max_mini_page_size(meta.max_mini_page_size)
239 .mini_page_binary_search(meta.mini_page_binary_search)
240 .write_load_full_page(meta.write_load_full_page)
241 .cb_size_byte(meta.cb_size_byte);
242
243 config
244 }
245
246 pub fn storage_backend(&mut self, backend: StorageBackend) -> &mut Self {
251 self.storage_backend = backend;
252 self
253 }
254
255 pub fn write_load_full_page(&mut self, load_full_page: bool) -> &mut Self {
256 self.write_load_full_page = load_full_page;
257 self
258 }
259
260 pub fn scan_promotion_rate(&mut self, prob: usize) -> &mut Self {
264 self.scan_promotion_rate.store(prob, Ordering::Relaxed);
265 self
266 }
267
268 pub fn max_fence_len(&mut self, len: usize) -> &mut Self {
269 self.max_fence_len = len;
270 self
271 }
272
273 pub fn read_record_cache(&mut self, read_full_page_cache: bool) -> &mut Self {
278 self.read_record_cache = read_full_page_cache;
279 self
280 }
281
282 pub fn max_mini_page_size(&mut self, size: usize) -> &mut Self {
286 self.max_mini_page_size = size;
287 self
288 }
289
290 pub fn mini_page_binary_search(&mut self, binary_search: bool) -> &mut Self {
294 self.mini_page_binary_search = binary_search;
295 self
296 }
297
298 pub fn read_promotion_rate(&mut self, prob: usize) -> &mut Self {
302 self.read_promotion_rate.store(prob, Ordering::Relaxed);
303 self
304 }
305
306 pub fn cb_copy_on_access_ratio(&mut self, ratio: f64) -> &mut Self {
314 self.cb_copy_on_access_ratio = ratio;
315 self
316 }
317
318 pub fn enable_write_ahead_log(&mut self, wal_config: Arc<WalConfig>) -> &mut Self {
322 self.write_ahead_log = Some(wal_config);
324 self
325 }
326
327 pub fn enable_write_ahead_log_default(&mut self) -> &mut Self {
335 let wal_config = WalConfig::new(self.file_path.parent().unwrap().join("wal.log"));
336 self.write_ahead_log = Some(Arc::new(wal_config));
337 self
338 }
339
340 pub fn cache_only(&mut self, cache_only: bool) -> &mut Self {
342 self.cache_only = cache_only;
343 self
344 }
345
346 pub fn cb_size_byte(&mut self, cb_size_byte: usize) -> &mut Self {
348 self.cb_size_byte = cb_size_byte;
349 self
350 }
351
352 pub fn file_path<P: AsRef<Path>>(&mut self, file_path: P) -> &mut Self {
353 self.file_path = file_path.as_ref().to_path_buf();
354 self
355 }
356
357 pub fn use_snapshot(&mut self, use_snapshot: bool) -> &mut Self {
359 self.use_snapshot = use_snapshot;
360 self
361 }
362
363 pub fn snapshot_version(&mut self, snapshot_version: u64) -> &mut Self {
365 self.snapshot_version = snapshot_version;
366 self
367 }
368
369 pub fn cb_max_key_len(&mut self, max_key_len: usize) -> &mut Self {
370 self.cb_max_key_len = max_key_len;
371 self.max_fence_len = max_key_len * 2;
372 self
373 }
374
375 pub fn cb_min_record_size(&mut self, min_record_size: usize) -> &mut Self {
376 self.cb_min_record_size = min_record_size;
377 self
378 }
379
380 pub fn cb_max_record_size(&mut self, max_record_size: usize) -> &mut Self {
381 self.cb_max_record_size = max_record_size;
382 self
383 }
384
385 pub fn get_cb_max_record_size(&self) -> usize {
387 self.cb_max_record_size
388 }
389
390 pub fn get_cb_size_byte(&self) -> usize {
391 self.cb_size_byte
392 }
393
394 pub fn leaf_page_size(&mut self, leaf_page_size: usize) -> &mut Self {
395 self.leaf_page_size = leaf_page_size;
396 self
397 }
398
399 pub fn get_leaf_page_size(&self) -> usize {
401 self.leaf_page_size
402 }
403
404 pub fn is_memory_backend(&self) -> bool {
406 self.storage_backend == StorageBackend::Memory
407 }
408
409 pub fn validate(&self) -> Result<(), ConfigError> {
411 if self.cb_min_record_size <= 1 {
413 return Err(ConfigError::MinimumRecordSize(
414 "cb_min_record_size (key + value in bytes) needs to be at least 2 bytes"
415 .to_string(),
416 ));
417 }
418
419 if self.cb_min_record_size > self.cb_max_record_size {
420 return Err(ConfigError::MaximumRecordSize("cb_min_record_size (key + value in bytes) cannot be greater than cb_max_record_size".to_string()));
421 }
422
423 if self.max_fence_len == 0 {
424 return Err(ConfigError::MaxKeyLen(
425 "cb_max_key_len cannot be zero".to_string(),
426 ));
427 }
428
429 if self.max_fence_len / 2 > self.cb_max_record_size {
430 return Err(ConfigError::MaxKeyLen(
431 "cb_max_key_len cannot be greater than cb_max_record_size".to_string(),
432 ));
433 }
434
435 if self.leaf_page_size > MAX_LEAF_PAGE_SIZE {
436 return Err(ConfigError::LeafPageSize(format!(
437 "leaf_page_size cannot be larger than {}",
438 MAX_LEAF_PAGE_SIZE
439 )));
440 }
441
442 if self.max_fence_len / 2 > MAX_KEY_LEN {
443 return Err(ConfigError::MaxKeyLen(format!(
444 "cb_max_key_len cannot be larger than {}",
445 MAX_KEY_LEN
446 )));
447 }
448
449 if !self.cb_size_byte.is_power_of_two() {
450 return Err(ConfigError::CircularBufferSize(
451 "cb_size_byte should be a power of two".to_string(),
452 ));
453 }
454
455 if self.leaf_page_size / self.cb_min_record_size > 4096 {
456 return Err(ConfigError::MinimumRecordSize(
457 "leaf_page_size/min_record_size cannot exceed 2^12.".to_string(),
458 ));
459 }
460
461 if !self.cache_only && !self.leaf_page_size.is_multiple_of(DISK_PAGE_SIZE) {
463 return Err(ConfigError::LeafPageSize(format!(
464 "In non cache-only mode leaf_page_size should be multiple of {}",
465 DISK_PAGE_SIZE
466 )));
467 } else if self.cache_only && !self.leaf_page_size.is_multiple_of(CACHE_LINE_SIZE) {
468 return Err(ConfigError::LeafPageSize(format!(
469 "In cache-only mode leaf_page_size should be multiple of {}",
470 CACHE_LINE_SIZE
471 )));
472 }
473
474 let max_record_size_with_meta = self.cb_max_record_size + std::mem::size_of::<LeafKVMeta>();
476 let mut max_mini_page_size: usize;
477
478 if self.cache_only {
479 if self.leaf_page_size < 2 * max_record_size_with_meta + std::mem::size_of::<LeafNode>()
480 {
481 return Err(ConfigError::MaximumRecordSize(format!(
482 "In cache-only mode, given the leaf_page_size the corresponding cb_max_record_size should be <= {}",
483 (self.leaf_page_size - std::mem::size_of::<LeafNode>()) / 2
484 - std::mem::size_of::<LeafKVMeta>()
485 )));
486 }
487 } else {
488 if max_record_size_with_meta
489 > self.leaf_page_size - self.max_fence_len - 2 * std::mem::size_of::<LeafKVMeta>()
490 {
491 return Err(ConfigError::MaximumRecordSize(format!(
492 "In non cache-only mode, given the leaf_page_size the corresponding cb_max_record_size should be <= {}",
493 self.leaf_page_size
494 - self.max_fence_len
495 - 2 * std::mem::size_of::<LeafKVMeta>()
496 )));
497 }
498 max_mini_page_size = self.leaf_page_size
499 - max_record_size_with_meta
500 - self.max_fence_len
501 - 2 * std::mem::size_of::<LeafKVMeta>();
502 max_mini_page_size = (max_mini_page_size / CACHE_LINE_SIZE) * CACHE_LINE_SIZE;
503
504 if max_mini_page_size < max_record_size_with_meta + std::mem::size_of::<LeafNode>() {
505 return Err(ConfigError::MaximumRecordSize(format!(
506 "In non cache-only mode, given the leaf_page_size the corresponding cb_max_record_size should be <= {}",
507 max_mini_page_size
508 - std::mem::size_of::<LeafNode>()
509 - std::mem::size_of::<LeafKVMeta>()
510 )));
511 }
512 }
513
514 if self.cache_only {
522 if self.cb_size_byte < 4 * self.leaf_page_size {
523 return Err(ConfigError::CircularBufferSize(
524 "In cache-only mode, cb_size_byte should be at least 4 times of leaf_page_size"
525 .to_string(),
526 ));
527 }
528 } else if self.cb_size_byte < 2 * self.leaf_page_size {
529 return Err(ConfigError::CircularBufferSize(
530 "In non cache-only mode, cb_size_byte should be at least 2 times of leaf_page_size"
531 .to_string(),
532 ));
533 }
534
535 Ok(())
536 }
537}
538
539#[derive(Clone, Debug)]
540
541pub struct WalConfig {
542 pub(crate) file_path: PathBuf,
543 pub(crate) flush_interval: Duration,
544 pub(crate) segment_size: usize,
545 pub(crate) storage_backend: StorageBackend,
546}
547
548impl WalConfig {
549 pub fn new(file_path: impl AsRef<Path>) -> Self {
556 Self {
557 file_path: file_path.as_ref().to_path_buf(),
558 flush_interval: Duration::from_millis(1),
559 segment_size: 1024 * 1024 * 1024,
560 storage_backend: StorageBackend::Std,
561 }
562 }
563
564 pub fn flush_interval(&mut self, interval: Duration) -> &mut Self {
566 self.flush_interval = interval;
567 self
568 }
569
570 pub fn segment_size(&mut self, size: usize) -> &mut Self {
572 self.segment_size = size;
573 self
574 }
575
576 pub fn storage_backend(&mut self, backend: StorageBackend) -> &mut Self {
580 self.storage_backend = backend;
581 self
582 }
583}
584
585#[cfg(test)]
586mod tests {
587 use super::*;
588
589 const SAMPLE_CONFIG_FILE: &str = "src/sample_config.toml";
590 #[test]
591 fn test_new_with_config_file() {
592 let config = Config::new_with_config_file(SAMPLE_CONFIG_FILE);
593
594 assert_eq!(config.cb_size_byte, 8192);
595 assert_eq!(config.read_promotion_rate.load(Ordering::Relaxed), 100);
596 assert_eq!(config.write_load_full_page, true);
597 assert_eq!(config.file_path, PathBuf::from("c/d/E"));
598 assert_eq!(config.cache_only, false);
599 }
600
601 #[test]
602 fn test_leaf_page_size_getter_setter() {
603 let mut config = Config::default();
604
605 assert_eq!(config.get_leaf_page_size(), DEFAULT_LEAF_PAGE_SIZE);
607
608 let new_leaf_page_size = 8192;
610 config.leaf_page_size(new_leaf_page_size);
611 assert_eq!(config.get_leaf_page_size(), new_leaf_page_size);
612
613 let another_leaf_page_size = 16384;
615 config.leaf_page_size(another_leaf_page_size);
616 assert_eq!(config.get_leaf_page_size(), another_leaf_page_size);
617 }
618
619 #[test]
620 fn test_cb_max_record_size_getter_setter() {
621 let mut config = Config::default();
622
623 assert_eq!(config.get_cb_max_record_size(), DEFAULT_MAX_RECORD_SIZE);
625
626 let new_max_record_size = 4096;
628 config.cb_max_record_size(new_max_record_size);
629 assert_eq!(config.get_cb_max_record_size(), new_max_record_size);
630
631 let another_max_record_size = 8192;
633 config.cb_max_record_size(another_max_record_size);
634 assert_eq!(config.get_cb_max_record_size(), another_max_record_size);
635 }
636}