shunyadb 0.1.0

A correctness-first embedded storage engine written in Rust, built around WAL, immutable LSM pages, MVCC, and deterministic crash recovery.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

use anyhow::Result;
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use crate::meta::PageMeta;
use crate::storage::page::builder::Page;
use crate::storage::page::reader::read_page;

/// Write page to disk, files must not already exist since pages are immutable.
pub fn write_page(path: impl AsRef<Path>, page: &Page) -> Result<u64> {
  let path = path.as_ref();

  if path.exists() {
    anyhow::bail!("page already exists: {:?}", path);
  }

  let tmp_path: PathBuf = path.with_extension("temp.new");
  let mut file = OpenOptions::new().write(true).create_new(true).open(&tmp_path)?;

  // Write header
  let header_bytes = bincode::serialize(&page.header)?;
  file.write_all(&header_bytes)?;

  // Write payload
  file.write_all(&page.payload)?;

  file.sync_all()?;
  drop(file);

  std::fs::rename(&tmp_path, path)?;

  // 4️⃣ fsync directory
  #[cfg(unix)] {
    let dir = path.parent().unwrap_or(Path::new("."));
    File::open(dir)?.sync_all()?;
  }

  let page_size_bytes = std::fs::metadata(path)?.len();

  Ok (page_size_bytes)
}


/// Read page from disk and validate
pub fn read_page_from_disk(path: impl AsRef<Path>) -> Result<Page> {
  let mut file = File::open(path)?;
  let mut bytes = Vec::new();
  file.read_to_end(&mut bytes)?;

  read_page(&bytes)
}


/// Delete older pages which have been moved to other level
pub fn delete_older_pages(dir: &Path, pages_to_be_removed: Vec<PageMeta>) -> Result<()>{
  for page in pages_to_be_removed {
    let path = dir.join(page.file_name);
    if let Err(e) = fs::remove_file(&path) {
      if e.kind() != std::io::ErrorKind::NotFound {
          return Err(e.into());
      }
    }
  }

  Ok(())
}

#[cfg(test)]
mod tests {
use super::*;

  #[test]
  fn check_duplicate_works() {
    let temp_dir = tempfile::tempdir().unwrap();
    let page_path = temp_dir.path().join("page_1");

    let record = crate::storage::record::Record::new(
      "id1".to_string(),
      1,
      std::collections::BTreeMap::new(),
    );

    let mut builder = crate::storage::page::builder::PageBuilder::new();
    builder.add(record);
    let page = builder.build();

    // First write should succeed
    write_page(&page_path, &page).expect("Failed to write page");

    // Second write should fail due to create_new
    let result = write_page(&page_path, &page);
    assert!(result.is_err());
  }
}