use std::collections::BTreeMap;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
pub mod wal;
pub mod sstable;
pub mod memtable;
use sstable::SSTable;
use memtable::MemTable;
#[derive(Debug)]
pub enum StoreError {
IO(io::Error),
Lock,
Other(String),
}
impl std::fmt::Display for StoreError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
StoreError::IO(err) => write!(f, "IO Error: {}", err),
StoreError::Lock => write!(f, "Thread Lock Poisoned"),
StoreError::Other(msg) => write!(f, "Store Error: {}", msg),
}
}
}
impl std::error::Error for StoreError {}
impl From<io::Error> for StoreError {
fn from(err: io::Error) -> Self {
StoreError::IO(err)
}
}
const FLUSH_THRESHOLD_BYTES: usize = 4 * 1024 * 1024;
pub struct SynapseStore {
db_dir: PathBuf,
memtable: Arc<RwLock<MemTable>>,
sstables: Arc<RwLock<Vec<(u32, SSTable)>>>, }
impl SynapseStore {
pub fn open<P: AsRef<Path>>(dir: P) -> Result<Self, StoreError> {
let db_dir = dir.as_ref().to_path_buf();
fs::create_dir_all(&db_dir)?;
let mut sst_files = Vec::new();
for entry in fs::read_dir(&db_dir)? {
let entry = entry?;
let path = entry.path();
if path.is_file() {
let filename = path.file_name().unwrap().to_string_lossy();
if filename.starts_with("sst_") && filename.ends_with(".db") {
let idx_str = &filename[4..filename.len() - 3];
if let Ok(idx) = idx_str.parse::<u32>() {
sst_files.push((idx, path));
}
}
}
}
sst_files.sort_by_key(|(idx, _)| *idx);
let mut sstables = Vec::new();
for (idx, path) in sst_files {
let table = SSTable::open(path)?;
sstables.push((idx, table));
}
sstables.reverse();
let wal_path = db_dir.join("wal.log");
let memtable = MemTable::open(wal_path)?;
Ok(Self {
db_dir,
memtable: Arc::new(RwLock::new(memtable)),
sstables: Arc::new(RwLock::new(sstables)),
})
}
pub fn put(&self, key: &[u8], val: &[u8]) -> Result<(), StoreError> {
let mut mem = self.memtable.write().map_err(|_| StoreError::Lock)?;
mem.put(key.to_vec(), val.to_vec())?;
if mem.size_bytes >= FLUSH_THRESHOLD_BYTES {
drop(mem); self.flush()?;
}
Ok(())
}
pub fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>, StoreError> {
{
let mem = self.memtable.read().map_err(|_| StoreError::Lock)?;
if let Some(val_opt) = mem.get(key) {
return Ok(val_opt.clone());
}
}
let mut ssts = self.sstables.write().map_err(|_| StoreError::Lock)?;
for (_, sst) in ssts.iter_mut() {
if let Some(val_opt) = sst.get(key)? {
return Ok(val_opt); }
}
Ok(None)
}
pub fn delete(&self, key: &[u8]) -> Result<(), StoreError> {
let mut mem = self.memtable.write().map_err(|_| StoreError::Lock)?;
mem.delete(key.to_vec())?;
if mem.size_bytes >= FLUSH_THRESHOLD_BYTES {
drop(mem);
self.flush()?;
}
Ok(())
}
pub fn flush(&self) -> Result<(), StoreError> {
let mut mem = self.memtable.write().map_err(|_| StoreError::Lock)?;
if mem.map.is_empty() {
return Ok(());
}
let snapshot = mem.clear_and_reset()?;
drop(mem);
let mut ssts = self.sstables.write().map_err(|_| StoreError::Lock)?;
let next_idx = ssts.iter().map(|(idx, _)| *idx).max().unwrap_or(0) + 1;
let sst_filename = format!("sst_{:05}.db", next_idx);
let sst_path = self.db_dir.join(&sst_filename);
let sorted_data: Vec<(Vec<u8>, Option<Vec<u8>>)> = snapshot.into_iter().collect();
SSTable::write_new(&sst_path, &sorted_data)?;
let sst_reader = SSTable::open(&sst_path)?;
ssts.insert(0, (next_idx, sst_reader));
Ok(())
}
pub fn compact(&self) -> Result<(), StoreError> {
let _mem_lock = self.memtable.write().map_err(|_| StoreError::Lock)?;
let mut ssts = self.sstables.write().map_err(|_| StoreError::Lock)?;
if ssts.len() <= 1 {
return Ok(()); }
let mut consolidated: BTreeMap<Vec<u8>, Option<Vec<u8>>> = BTreeMap::new();
for (_, sst) in ssts.iter_mut().rev() {
let keys: Vec<Vec<u8>> = sst.index.iter().map(|r| r.key.clone()).collect();
for key in keys {
if let Some(val_opt) = sst.get(&key)? {
consolidated.insert(key, val_opt);
}
}
}
let compacted_data: Vec<(Vec<u8>, Option<Vec<u8>>)> = consolidated
.into_iter()
.filter(|(_, val_opt)| val_opt.is_some())
.collect();
let temp_filename = "sst_compaction.tmp";
let temp_path = self.db_dir.join(temp_filename);
SSTable::write_new(&temp_path, &compacted_data)?;
let old_indices: Vec<u32> = ssts.iter().map(|(idx, _)| *idx).collect();
ssts.clear();
for idx in old_indices {
let path = self.db_dir.join(format!("sst_{:05}.db", idx));
if path.exists() {
fs::remove_file(path)?;
}
}
let final_path = self.db_dir.join("sst_00001.db");
fs::rename(&temp_path, &final_path)?;
let sst_reader = SSTable::open(&final_path)?;
ssts.push((1, sst_reader));
Ok(())
}
}