Skip to main content

synapse_store/
lib.rs

1use std::collections::BTreeMap;
2use std::fs;
3use std::io;
4use std::path::{Path, PathBuf};
5use std::sync::{Arc, RwLock};
6
7pub mod wal;
8pub mod sstable;
9pub mod memtable;
10
11use sstable::SSTable;
12use memtable::MemTable;
13
14#[derive(Debug)]
15pub enum StoreError {
16    IO(io::Error),
17    Lock,
18    Other(String),
19}
20
21impl std::fmt::Display for StoreError {
22    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23        match self {
24            StoreError::IO(err) => write!(f, "IO Error: {}", err),
25            StoreError::Lock => write!(f, "Thread Lock Poisoned"),
26            StoreError::Other(msg) => write!(f, "Store Error: {}", msg),
27        }
28    }
29}
30
31impl std::error::Error for StoreError {}
32
33impl From<io::Error> for StoreError {
34    fn from(err: io::Error) -> Self {
35        StoreError::IO(err)
36    }
37}
38
39// 4MB Flush threshold
40const FLUSH_THRESHOLD_BYTES: usize = 4 * 1024 * 1024;
41
42pub struct SynapseStore {
43    db_dir: PathBuf,
44    memtable: Arc<RwLock<MemTable>>,
45    sstables: Arc<RwLock<Vec<(u32, SSTable)>>>, // Sorted (Index, Reader) newest first
46}
47
48impl SynapseStore {
49    // Open database directory and load previous WAL/SSTables
50    pub fn open<P: AsRef<Path>>(dir: P) -> Result<Self, StoreError> {
51        let db_dir = dir.as_ref().to_path_buf();
52        fs::create_dir_all(&db_dir)?;
53
54        // 1. Recover SSTables
55        let mut sst_files = Vec::new();
56        for entry in fs::read_dir(&db_dir)? {
57            let entry = entry?;
58            let path = entry.path();
59            if path.is_file() {
60                let filename = path.file_name().unwrap().to_string_lossy();
61                if filename.starts_with("sst_") && filename.ends_with(".db") {
62                    // Extract index number: e.g. sst_00004.db -> 4
63                    let idx_str = &filename[4..filename.len() - 3];
64                    if let Ok(idx) = idx_str.parse::<u32>() {
65                        sst_files.push((idx, path));
66                    }
67                }
68            }
69        }
70
71        // Sort files numerically, oldest to newest
72        sst_files.sort_by_key(|(idx, _)| *idx);
73
74        let mut sstables = Vec::new();
75        for (idx, path) in sst_files {
76            let table = SSTable::open(path)?;
77            sstables.push((idx, table));
78        }
79
80        // Keep list order: newest first
81        sstables.reverse();
82
83        // 2. Open active MemTable / WAL
84        let wal_path = db_dir.join("wal.log");
85        let memtable = MemTable::open(wal_path)?;
86
87        Ok(Self {
88            db_dir,
89            memtable: Arc::new(RwLock::new(memtable)),
90            sstables: Arc::new(RwLock::new(sstables)),
91        })
92    }
93
94    // Write a key-value pair
95    pub fn put(&self, key: &[u8], val: &[u8]) -> Result<(), StoreError> {
96        let mut mem = self.memtable.write().map_err(|_| StoreError::Lock)?;
97        mem.put(key.to_vec(), val.to_vec())?;
98
99        // Auto-flush if threshold is crossed
100        if mem.size_bytes >= FLUSH_THRESHOLD_BYTES {
101            drop(mem); // Release lock before flushing
102            self.flush()?;
103        }
104        Ok(())
105    }
106
107    // Read value for a key
108    pub fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>, StoreError> {
109        // 1. Search in-memory MemTable
110        {
111            let mem = self.memtable.read().map_err(|_| StoreError::Lock)?;
112            if let Some(val_opt) = mem.get(key) {
113                return Ok(val_opt.clone());
114            }
115        }
116
117        // 2. Search on-disk SSTables (newest to oldest)
118        let mut ssts = self.sstables.write().map_err(|_| StoreError::Lock)?;
119        for (_, sst) in ssts.iter_mut() {
120            if let Some(val_opt) = sst.get(key)? {
121                return Ok(val_opt); // Returns Some(val) or None (deleted)
122            }
123        }
124
125        Ok(None)
126    }
127
128    // Mark key as deleted (tombstone)
129    pub fn delete(&self, key: &[u8]) -> Result<(), StoreError> {
130        let mut mem = self.memtable.write().map_err(|_| StoreError::Lock)?;
131        mem.delete(key.to_vec())?;
132
133        if mem.size_bytes >= FLUSH_THRESHOLD_BYTES {
134            drop(mem);
135            self.flush()?;
136        }
137        Ok(())
138    }
139
140    // Flush active MemTable to disk as an SSTable file
141    pub fn flush(&self) -> Result<(), StoreError> {
142        let mut mem = self.memtable.write().map_err(|_| StoreError::Lock)?;
143        if mem.map.is_empty() {
144            return Ok(());
145        }
146
147        // Reset MemTable state and extract snapshot
148        let snapshot = mem.clear_and_reset()?;
149        drop(mem); // Release MemTable write lock
150
151        let mut ssts = self.sstables.write().map_err(|_| StoreError::Lock)?;
152        
153        // Find next SSTable index number
154        let next_idx = ssts.iter().map(|(idx, _)| *idx).max().unwrap_or(0) + 1;
155        let sst_filename = format!("sst_{:05}.db", next_idx);
156        let sst_path = self.db_dir.join(&sst_filename);
157
158        // Map BTreeMap to slice
159        let sorted_data: Vec<(Vec<u8>, Option<Vec<u8>>)> = snapshot.into_iter().collect();
160        SSTable::write_new(&sst_path, &sorted_data)?;
161
162        // Open newly created reader and insert at index 0 (newest)
163        let sst_reader = SSTable::open(&sst_path)?;
164        ssts.insert(0, (next_idx, sst_reader));
165
166        Ok(())
167    }
168
169    // Consolidates all active SSTables into a single file to improve search speeds
170    pub fn compact(&self) -> Result<(), StoreError> {
171        // Ensure no MemTable flushes occur during compaction
172        let _mem_lock = self.memtable.write().map_err(|_| StoreError::Lock)?;
173        
174        let mut ssts = self.sstables.write().map_err(|_| StoreError::Lock)?;
175        if ssts.len() <= 1 {
176            return Ok(()); // Nothing to compact
177        }
178
179        // Merge-sort all SSTable keys from oldest to newest to preserve overrides
180        let mut consolidated: BTreeMap<Vec<u8>, Option<Vec<u8>>> = BTreeMap::new();
181
182        // Traverse from oldest to newest so newer keys overwrite older ones
183        for (_, sst) in ssts.iter_mut().rev() {
184            // Collect keys first to avoid simultaneous immutable+mutable borrow
185            let keys: Vec<Vec<u8>> = sst.index.iter().map(|r| r.key.clone()).collect();
186            for key in keys {
187                if let Some(val_opt) = sst.get(&key)? {
188                    consolidated.insert(key, val_opt);
189                }
190            }
191        }
192
193        // Clean up: Filter out explicit tombstones (None) during consolidation to free disk space!
194        let compacted_data: Vec<(Vec<u8>, Option<Vec<u8>>)> = consolidated
195            .into_iter()
196            .filter(|(_, val_opt)| val_opt.is_some())
197            .collect();
198
199        // Write to a temporary file
200        let temp_filename = "sst_compaction.tmp";
201        let temp_path = self.db_dir.join(temp_filename);
202        SSTable::write_new(&temp_path, &compacted_data)?;
203
204        // Close all readers to release file locks on Windows
205        let old_indices: Vec<u32> = ssts.iter().map(|(idx, _)| *idx).collect();
206        ssts.clear();
207
208        // Delete old sst files
209        for idx in old_indices {
210            let path = self.db_dir.join(format!("sst_{:05}.db", idx));
211            if path.exists() {
212                fs::remove_file(path)?;
213            }
214        }
215
216        // Rename compacted file to sst_00001.db
217        let final_path = self.db_dir.join("sst_00001.db");
218        fs::rename(&temp_path, &final_path)?;
219
220        // Open consolidated table
221        let sst_reader = SSTable::open(&final_path)?;
222        ssts.push((1, sst_reader));
223
224        Ok(())
225    }
226}