1use std::{
5 fs,
6 path::{Path, PathBuf},
7 sync::{atomic::Ordering, Arc},
8 time::Duration,
9};
10
11use crate::{
12 error::ConfigError,
13 nodes::{
14 leaf_node::LeafKVMeta, LeafNode, CACHE_LINE_SIZE, DISK_PAGE_SIZE, MAX_KEY_LEN,
15 MAX_LEAF_PAGE_SIZE,
16 },
17 snapshot::BfTreeMeta,
18};
19use serde::Deserialize;
20use std::sync::atomic::AtomicUsize;
21
22const DEFAULT_PROMOTION_RATE_DEBUG: usize = 50;
23const DEFAULT_PROMOTION_RATE_RLEASE: usize = 30;
24const DEFAULT_MAX_MINI_PAGE_SIZE: usize = 2048; const DEFAULT_COPY_ON_ACCESS_RATIO: f64 = 0.1;
26const DEFAULT_CIRCULAR_BUFFER_SIZE: usize = 1024 * 1024 * 32;
27const DEFAULT_MIN_RECORD_SIZE: usize = 4;
28const DEFAULT_MAX_RECORD_SIZE: usize = 1952;
29const DEFAULT_LEAF_PAGE_SIZE: usize = 4096;
30const DEFAULT_MAX_KEY_LEN: usize = 16;
31
32#[derive(Debug)]
36pub struct Config {
37 pub(crate) read_promotion_rate: AtomicUsize,
38 pub(crate) scan_promotion_rate: AtomicUsize,
39 pub(crate) storage_backend: StorageBackend,
40 pub(crate) snapshot_backend: StorageBackend,
41 pub(crate) cb_size_byte: usize,
42 pub(crate) cb_min_record_size: usize,
43 pub(crate) cb_max_record_size: usize,
44 pub(crate) leaf_page_size: usize,
45 pub(crate) cb_max_key_len: usize,
46 pub(crate) max_fence_len: usize,
47 pub(crate) cb_copy_on_access_ratio: f64,
48 pub(crate) read_record_cache: bool,
49 pub(crate) file_path: PathBuf,
50 pub(crate) use_snapshot: bool,
51 pub(crate) snapshot_version: u64,
52 pub(crate) snapshot_file_path: PathBuf,
53 pub(crate) max_mini_page_size: usize,
54 pub(crate) mini_page_binary_search: bool,
55 pub(crate) write_ahead_log: Option<Arc<WalConfig>>,
56 pub(crate) write_load_full_page: bool,
57 pub(crate) cache_only: bool,
58}
59
60impl Clone for Config {
61 fn clone(&self) -> Self {
62 Self {
63 snapshot_version: self.snapshot_version,
64 read_promotion_rate: AtomicUsize::new(self.read_promotion_rate.load(Ordering::Relaxed)),
65 scan_promotion_rate: AtomicUsize::new(self.scan_promotion_rate.load(Ordering::Relaxed)),
66 storage_backend: self.storage_backend.clone(),
67 snapshot_backend: self.snapshot_backend.clone(),
68 use_snapshot: self.use_snapshot,
69 cb_size_byte: self.cb_size_byte,
70 cb_min_record_size: self.cb_min_record_size,
71 cb_max_record_size: self.cb_max_record_size,
72 leaf_page_size: self.leaf_page_size,
73 cb_max_key_len: self.cb_max_key_len,
74 max_fence_len: self.max_fence_len,
75 cb_copy_on_access_ratio: self.cb_copy_on_access_ratio,
76 read_record_cache: self.read_record_cache,
77 file_path: self.file_path.clone(),
78 snapshot_file_path: self.snapshot_file_path.clone(),
79 max_mini_page_size: self.max_mini_page_size,
80 mini_page_binary_search: self.mini_page_binary_search,
81 write_ahead_log: self.write_ahead_log.clone(),
82 write_load_full_page: self.write_load_full_page,
83 cache_only: self.cache_only,
84 }
85 }
86}
87
88#[derive(Debug, Default, Clone, Eq, PartialEq)]
90pub enum StorageBackend {
91 Memory,
92 #[default]
93 Std,
94 #[cfg(target_os = "linux")]
95 StdDirect,
96 #[cfg(target_os = "linux")]
97 IoUringPolling,
98 #[cfg(target_os = "linux")]
99 IoUringBlocking,
100 #[cfg(all(target_os = "linux", feature = "spdk"))]
101 Spdk,
102}
103
104#[derive(Debug, Deserialize)]
105pub struct ConfigFile {
106 pub(crate) cb_size_byte: usize,
107 pub(crate) cb_min_record_size: usize,
108 pub(crate) cb_max_record_size: usize,
109 pub(crate) cb_max_key_len: usize,
110 pub(crate) leaf_page_size: usize,
111 pub(crate) index_file_path: String,
112 pub(crate) snapshot_file_path: String,
113 pub(crate) use_snapshot: bool,
114 pub(crate) snapshot_version: u64,
115 pub(crate) backend_storage: String,
116 pub(crate) read_promotion_rate: usize,
117 pub(crate) write_load_full_page: bool,
118 pub(crate) cache_only: bool,
119}
120
121impl Default for Config {
124 fn default() -> Self {
125 let read_promotion_rate = if cfg!(debug_assertions) {
126 DEFAULT_PROMOTION_RATE_DEBUG
127 } else {
128 DEFAULT_PROMOTION_RATE_RLEASE
129 };
130 let scan_promotion_rate = if cfg!(debug_assertions) {
131 DEFAULT_PROMOTION_RATE_DEBUG
132 } else {
133 DEFAULT_PROMOTION_RATE_RLEASE
134 };
135
136 Self {
137 snapshot_version: 0,
138 read_promotion_rate: AtomicUsize::new(read_promotion_rate),
139 scan_promotion_rate: AtomicUsize::new(scan_promotion_rate),
140 use_snapshot: false,
141 cb_size_byte: DEFAULT_CIRCULAR_BUFFER_SIZE,
142 cb_min_record_size: DEFAULT_MIN_RECORD_SIZE,
143 cb_max_record_size: DEFAULT_MAX_RECORD_SIZE,
144 leaf_page_size: DEFAULT_LEAF_PAGE_SIZE,
145 cb_max_key_len: DEFAULT_MAX_KEY_LEN,
146 max_fence_len: DEFAULT_MAX_KEY_LEN * 2,
147 cb_copy_on_access_ratio: DEFAULT_COPY_ON_ACCESS_RATIO,
148 file_path: PathBuf::new(),
149 read_record_cache: true,
150 max_mini_page_size: DEFAULT_MAX_MINI_PAGE_SIZE,
151 mini_page_binary_search: true,
152 storage_backend: StorageBackend::Memory,
153 snapshot_backend: StorageBackend::Std,
154 write_ahead_log: None,
155 write_load_full_page: true,
156 cache_only: false,
157 snapshot_file_path: PathBuf::from("targets/snapshot"),
158 }
159 }
160}
161impl Config {
162 pub fn new(file_path: impl AsRef<Path>, circular_buffer_size: usize) -> Self {
163 let mut config = Self::default();
164 let mut cache_only = false;
165 let storage_backend = if file_path.as_ref().to_str().unwrap().starts_with(":memory:") {
166 StorageBackend::Memory
167 } else if file_path.as_ref().to_str().unwrap().starts_with(":cache:") {
168 cache_only = true;
169 StorageBackend::Memory
170 } else {
171 StorageBackend::default()
172 };
173
174 config
175 .storage_backend(storage_backend)
176 .cache_only(cache_only)
177 .cb_size_byte(circular_buffer_size)
178 .file_path(file_path);
179
180 config
181 }
182
183 pub fn new_with_config_file<P: AsRef<Path>>(config_file_path: P) -> Self {
186 let config_file_str =
187 fs::read_to_string(config_file_path).expect("couldn't read config file");
188 let config_file: ConfigFile =
189 toml::from_str(&config_file_str).expect("Fail to parse config file");
190 let scan_promotion_rate = if cfg!(debug_assertions) {
191 DEFAULT_PROMOTION_RATE_DEBUG
192 } else {
193 DEFAULT_PROMOTION_RATE_RLEASE
194 };
195 let mut storage = StorageBackend::Memory;
196 if config_file.backend_storage == "disk" {
197 storage = StorageBackend::default();
198 }
199
200 Self {
202 snapshot_version: config_file.snapshot_version,
203 read_promotion_rate: AtomicUsize::new(config_file.read_promotion_rate),
204 scan_promotion_rate: AtomicUsize::new(scan_promotion_rate),
205 use_snapshot: config_file.use_snapshot,
206 cb_size_byte: config_file.cb_size_byte,
207 cb_min_record_size: config_file.cb_min_record_size,
208 cb_max_record_size: config_file.cb_max_record_size,
209 leaf_page_size: config_file.leaf_page_size,
210 cb_max_key_len: config_file.cb_max_key_len,
211 max_fence_len: config_file.cb_max_key_len * 2,
212 cb_copy_on_access_ratio: DEFAULT_COPY_ON_ACCESS_RATIO,
213 file_path: PathBuf::from(config_file.index_file_path),
214 snapshot_file_path: PathBuf::from(config_file.snapshot_file_path),
215 read_record_cache: true,
216 max_mini_page_size: DEFAULT_MAX_MINI_PAGE_SIZE,
217 mini_page_binary_search: true,
218 storage_backend: storage,
219 snapshot_backend: StorageBackend::Std,
220 write_ahead_log: None,
221 write_load_full_page: config_file.write_load_full_page,
222 cache_only: config_file.cache_only,
223 }
224 }
225
226 pub(crate) fn new_from_snapshot(meta: &BfTreeMeta) -> Self {
228 let mut config = Self::default();
229
230 config
232 .snapshot_version(meta.snapshot_version + 1)
233 .cache_only(meta.cache_only)
234 .read_promotion_rate(meta.read_promotion_rate)
235 .scan_promotion_rate(meta.scan_promotion_rate)
236 .cb_min_record_size(meta.cb_min_record_size)
237 .cb_max_record_size(meta.cb_max_record_size)
238 .leaf_page_size(meta.leaf_page_size)
239 .cb_max_key_len(meta.cb_max_key_len)
240 .max_fence_len(meta.max_fence_len)
241 .cb_copy_on_access_ratio(meta.cb_copy_on_access_ratio)
242 .read_record_cache(meta.read_record_cache)
243 .max_mini_page_size(meta.max_mini_page_size)
244 .mini_page_binary_search(meta.mini_page_binary_search)
245 .write_load_full_page(meta.write_load_full_page)
246 .cb_size_byte(meta.cb_size_byte);
247
248 config
249 }
250
251 pub fn storage_backend(&mut self, backend: StorageBackend) -> &mut Self {
256 self.storage_backend = backend;
257 self
258 }
259
260 pub fn write_load_full_page(&mut self, load_full_page: bool) -> &mut Self {
261 self.write_load_full_page = load_full_page;
262 self
263 }
264
265 pub fn snapshot_backend(&mut self, backend: StorageBackend) -> &mut Self {
269 self.snapshot_backend = backend;
270 self
271 }
272
273 pub fn scan_promotion_rate(&mut self, prob: usize) -> &mut Self {
277 self.scan_promotion_rate.store(prob, Ordering::Relaxed);
278 self
279 }
280
281 pub fn max_fence_len(&mut self, len: usize) -> &mut Self {
282 self.max_fence_len = len;
283 self
284 }
285
286 pub fn read_record_cache(&mut self, read_full_page_cache: bool) -> &mut Self {
291 self.read_record_cache = read_full_page_cache;
292 self
293 }
294
295 pub fn max_mini_page_size(&mut self, size: usize) -> &mut Self {
299 self.max_mini_page_size = size;
300 self
301 }
302
303 pub fn mini_page_binary_search(&mut self, binary_search: bool) -> &mut Self {
307 self.mini_page_binary_search = binary_search;
308 self
309 }
310
311 pub fn read_promotion_rate(&mut self, prob: usize) -> &mut Self {
315 self.read_promotion_rate.store(prob, Ordering::Relaxed);
316 self
317 }
318
319 pub fn cb_copy_on_access_ratio(&mut self, ratio: f64) -> &mut Self {
327 self.cb_copy_on_access_ratio = ratio;
328 self
329 }
330
331 pub fn enable_write_ahead_log(&mut self, wal_config: Arc<WalConfig>) -> &mut Self {
335 self.write_ahead_log = Some(wal_config);
337 self
338 }
339
340 pub fn enable_write_ahead_log_default(&mut self) -> &mut Self {
348 let wal_config = WalConfig::new(self.file_path.parent().unwrap().join("wal.log"));
349 self.write_ahead_log = Some(Arc::new(wal_config));
350 self
351 }
352
353 pub fn cache_only(&mut self, cache_only: bool) -> &mut Self {
355 self.cache_only = cache_only;
356 self
357 }
358
359 pub fn cb_size_byte(&mut self, cb_size_byte: usize) -> &mut Self {
361 self.cb_size_byte = cb_size_byte;
362 self
363 }
364
365 pub fn file_path<P: AsRef<Path>>(&mut self, file_path: P) -> &mut Self {
366 self.file_path = file_path.as_ref().to_path_buf();
367 self
368 }
369
370 pub fn snapshot_file_path<P: AsRef<Path>>(&mut self, file_path: P) -> &mut Self {
371 self.snapshot_file_path = file_path.as_ref().to_path_buf();
372 self
373 }
374
375 pub fn use_snapshot(&mut self, use_snapshot: bool) -> &mut Self {
377 self.use_snapshot = use_snapshot;
378 self
379 }
380
381 pub fn snapshot_version(&mut self, snapshot_version: u64) -> &mut Self {
383 self.snapshot_version = snapshot_version;
384 self
385 }
386
387 pub fn cb_max_key_len(&mut self, max_key_len: usize) -> &mut Self {
388 self.cb_max_key_len = max_key_len;
389 self.max_fence_len = max_key_len * 2;
390 self
391 }
392
393 pub fn cb_min_record_size(&mut self, min_record_size: usize) -> &mut Self {
394 self.cb_min_record_size = min_record_size;
395 self
396 }
397
398 pub fn cb_max_record_size(&mut self, max_record_size: usize) -> &mut Self {
399 self.cb_max_record_size = max_record_size;
400 self
401 }
402
403 pub fn get_cb_max_record_size(&self) -> usize {
405 self.cb_max_record_size
406 }
407
408 pub fn get_cb_size_byte(&self) -> usize {
409 self.cb_size_byte
410 }
411
412 pub fn leaf_page_size(&mut self, leaf_page_size: usize) -> &mut Self {
413 self.leaf_page_size = leaf_page_size;
414 self
415 }
416
417 pub fn get_leaf_page_size(&self) -> usize {
419 self.leaf_page_size
420 }
421
422 pub fn is_memory_backend(&self) -> bool {
424 self.storage_backend == StorageBackend::Memory
425 }
426
427 pub fn validate(&self) -> Result<(), ConfigError> {
429 if self.cb_min_record_size <= 1 {
431 return Err(ConfigError::MinimumRecordSize(
432 "cb_min_record_size (key + value in bytes) needs to be at least 2 bytes"
433 .to_string(),
434 ));
435 }
436
437 if self.cb_min_record_size > self.cb_max_record_size {
438 return Err(ConfigError::MaximumRecordSize("cb_min_record_size (key + value in bytes) cannot be greater than cb_max_record_size".to_string()));
439 }
440
441 if self.max_fence_len == 0 {
442 return Err(ConfigError::MaxKeyLen(
443 "cb_max_key_len cannot be zero".to_string(),
444 ));
445 }
446
447 if self.max_fence_len / 2 > self.cb_max_record_size {
448 return Err(ConfigError::MaxKeyLen(
449 "cb_max_key_len cannot be greater than cb_max_record_size".to_string(),
450 ));
451 }
452
453 if self.leaf_page_size > MAX_LEAF_PAGE_SIZE {
454 return Err(ConfigError::LeafPageSize(format!(
455 "leaf_page_size cannot be larger than {}",
456 MAX_LEAF_PAGE_SIZE
457 )));
458 }
459
460 if self.max_fence_len / 2 > MAX_KEY_LEN {
461 return Err(ConfigError::MaxKeyLen(format!(
462 "cb_max_key_len cannot be larger than {}",
463 MAX_KEY_LEN
464 )));
465 }
466
467 if !self.cb_size_byte.is_power_of_two() {
468 return Err(ConfigError::CircularBufferSize(
469 "cb_size_byte should be a power of two".to_string(),
470 ));
471 }
472
473 if self.leaf_page_size / self.cb_min_record_size > 4096 {
474 return Err(ConfigError::MinimumRecordSize(
475 "leaf_page_size/min_record_size cannot exceed 2^12.".to_string(),
476 ));
477 }
478
479 if !self.cache_only && !self.leaf_page_size.is_multiple_of(DISK_PAGE_SIZE) {
481 return Err(ConfigError::LeafPageSize(format!(
482 "In non cache-only mode leaf_page_size should be multiple of {}",
483 DISK_PAGE_SIZE
484 )));
485 } else if self.cache_only && !self.leaf_page_size.is_multiple_of(CACHE_LINE_SIZE) {
486 return Err(ConfigError::LeafPageSize(format!(
487 "In cache-only mode leaf_page_size should be multiple of {}",
488 CACHE_LINE_SIZE
489 )));
490 }
491
492 let max_record_size_with_meta = self.cb_max_record_size + std::mem::size_of::<LeafKVMeta>();
494 let mut max_mini_page_size: usize;
495
496 if self.cache_only {
497 if self.leaf_page_size < 2 * max_record_size_with_meta + std::mem::size_of::<LeafNode>()
498 {
499 return Err(ConfigError::MaximumRecordSize(format!(
500 "In cache-only mode, given the leaf_page_size the corresponding cb_max_record_size should be <= {}",
501 (self.leaf_page_size - std::mem::size_of::<LeafNode>()) / 2
502 - std::mem::size_of::<LeafKVMeta>()
503 )));
504 }
505 } else {
506 if max_record_size_with_meta
507 > self.leaf_page_size - self.max_fence_len - 2 * std::mem::size_of::<LeafKVMeta>()
508 {
509 return Err(ConfigError::MaximumRecordSize(format!(
510 "In non cache-only mode, given the leaf_page_size the corresponding cb_max_record_size should be <= {}",
511 self.leaf_page_size
512 - self.max_fence_len
513 - 2 * std::mem::size_of::<LeafKVMeta>()
514 )));
515 }
516 max_mini_page_size = self.leaf_page_size
517 - max_record_size_with_meta
518 - self.max_fence_len
519 - 2 * std::mem::size_of::<LeafKVMeta>();
520 max_mini_page_size = (max_mini_page_size / CACHE_LINE_SIZE) * CACHE_LINE_SIZE;
521
522 if max_mini_page_size < max_record_size_with_meta + std::mem::size_of::<LeafNode>() {
523 return Err(ConfigError::MaximumRecordSize(format!(
524 "In non cache-only mode, given the leaf_page_size the corresponding cb_max_record_size should be <= {}",
525 max_mini_page_size
526 - std::mem::size_of::<LeafNode>()
527 - std::mem::size_of::<LeafKVMeta>()
528 )));
529 }
530 }
531
532 if self.cache_only {
540 if self.cb_size_byte < 4 * self.leaf_page_size {
541 return Err(ConfigError::CircularBufferSize(
542 "In cache-only mode, cb_size_byte should be at least 4 times of leaf_page_size"
543 .to_string(),
544 ));
545 }
546 } else if self.cb_size_byte < 2 * self.leaf_page_size {
547 return Err(ConfigError::CircularBufferSize(
548 "In non cache-only mode, cb_size_byte should be at least 2 times of leaf_page_size"
549 .to_string(),
550 ));
551 }
552
553 if self.use_snapshot
554 && self.file_path == self.snapshot_file_path
555 && self.snapshot_file_path != PathBuf::new()
556 {
557 return Err(ConfigError::SnapshotFileInvalid(
558 "snapshot file path should not be the same as the main file path".to_string(),
559 ));
560 }
561
562 Ok(())
563 }
564}
565
566#[derive(Clone, Debug)]
567
568pub struct WalConfig {
569 pub(crate) file_path: PathBuf,
570 pub(crate) flush_interval: Duration,
571 pub(crate) segment_size: usize,
572 pub(crate) storage_backend: StorageBackend,
573}
574
575impl WalConfig {
576 pub fn new(file_path: impl AsRef<Path>) -> Self {
583 Self {
584 file_path: file_path.as_ref().to_path_buf(),
585 flush_interval: Duration::from_millis(1),
586 segment_size: 1024 * 1024 * 1024,
587 storage_backend: StorageBackend::Std,
588 }
589 }
590
591 pub fn flush_interval(&mut self, interval: Duration) -> &mut Self {
593 self.flush_interval = interval;
594 self
595 }
596
597 pub fn segment_size(&mut self, size: usize) -> &mut Self {
599 self.segment_size = size;
600 self
601 }
602
603 pub fn storage_backend(&mut self, backend: StorageBackend) -> &mut Self {
607 self.storage_backend = backend;
608 self
609 }
610}
611
612#[cfg(test)]
613mod tests {
614 use super::*;
615
616 const SAMPLE_CONFIG_FILE: &str = "src/sample_config.toml";
617 #[test]
618 fn test_new_with_config_file() {
619 let config = Config::new_with_config_file(SAMPLE_CONFIG_FILE);
620
621 assert_eq!(config.cb_size_byte, 8192);
622 assert_eq!(config.read_promotion_rate.load(Ordering::Relaxed), 100);
623 assert_eq!(config.write_load_full_page, true);
624 assert_eq!(config.file_path, PathBuf::from("c/d/E"));
625 assert_eq!(config.cache_only, false);
626 }
627
628 #[test]
629 fn test_leaf_page_size_getter_setter() {
630 let mut config = Config::default();
631
632 assert_eq!(config.get_leaf_page_size(), DEFAULT_LEAF_PAGE_SIZE);
634
635 let new_leaf_page_size = 8192;
637 config.leaf_page_size(new_leaf_page_size);
638 assert_eq!(config.get_leaf_page_size(), new_leaf_page_size);
639
640 let another_leaf_page_size = 16384;
642 config.leaf_page_size(another_leaf_page_size);
643 assert_eq!(config.get_leaf_page_size(), another_leaf_page_size);
644 }
645
646 #[test]
647 fn test_cb_max_record_size_getter_setter() {
648 let mut config = Config::default();
649
650 assert_eq!(config.get_cb_max_record_size(), DEFAULT_MAX_RECORD_SIZE);
652
653 let new_max_record_size = 4096;
655 config.cb_max_record_size(new_max_record_size);
656 assert_eq!(config.get_cb_max_record_size(), new_max_record_size);
657
658 let another_max_record_size = 8192;
660 config.cb_max_record_size(another_max_record_size);
661 assert_eq!(config.get_cb_max_record_size(), another_max_record_size);
662 }
663}