use anyhow::Result;
use std::path::{Path, PathBuf};
pub trait StorageBackend: Send + Sync {
fn load_index(&self, index_id: &str) -> Result<Vec<u8>>;
fn save_index(&mut self, index_id: &str, data: &[u8]) -> Result<()>;
fn delete_index(&mut self, index_id: &str) -> Result<()>;
fn exists(&self, index_id: &str) -> bool;
fn get_size(&self, index_id: &str) -> Result<u64>;
fn list_indices(&self) -> Result<Vec<String>>;
fn backend_type(&self) -> &'static str;
}
pub struct HotTierStorage {
cache: std::sync::Arc<std::sync::Mutex<std::collections::HashMap<String, Vec<u8>>>>,
}
impl HotTierStorage {
pub fn new() -> Self {
Self {
cache: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
}
}
pub fn memory_usage(&self) -> u64 {
let cache = self.cache.lock().expect("lock should not be poisoned");
cache.values().map(|v| v.len() as u64).sum()
}
pub fn cache_size(&self) -> usize {
let cache = self.cache.lock().expect("lock should not be poisoned");
cache.len()
}
}
impl Default for HotTierStorage {
fn default() -> Self {
Self::new()
}
}
impl StorageBackend for HotTierStorage {
fn load_index(&self, index_id: &str) -> Result<Vec<u8>> {
let cache = self.cache.lock().expect("lock should not be poisoned");
cache
.get(index_id)
.cloned()
.ok_or_else(|| anyhow::anyhow!("Index {} not found in hot tier", index_id))
}
fn save_index(&mut self, index_id: &str, data: &[u8]) -> Result<()> {
let mut cache = self.cache.lock().expect("lock should not be poisoned");
cache.insert(index_id.to_string(), data.to_vec());
Ok(())
}
fn delete_index(&mut self, index_id: &str) -> Result<()> {
let mut cache = self.cache.lock().expect("lock should not be poisoned");
cache
.remove(index_id)
.ok_or_else(|| anyhow::anyhow!("Index {} not found in hot tier", index_id))?;
Ok(())
}
fn exists(&self, index_id: &str) -> bool {
let cache = self.cache.lock().expect("lock should not be poisoned");
cache.contains_key(index_id)
}
fn get_size(&self, index_id: &str) -> Result<u64> {
let cache = self.cache.lock().expect("lock should not be poisoned");
cache
.get(index_id)
.map(|v| v.len() as u64)
.ok_or_else(|| anyhow::anyhow!("Index {} not found in hot tier", index_id))
}
fn list_indices(&self) -> Result<Vec<String>> {
let cache = self.cache.lock().expect("lock should not be poisoned");
Ok(cache.keys().cloned().collect())
}
fn backend_type(&self) -> &'static str {
"HotTier (In-Memory)"
}
}
pub struct WarmTierStorage {
base_path: PathBuf,
compression_enabled: bool,
compression_level: i32,
}
impl WarmTierStorage {
pub fn new<P: AsRef<Path>>(
base_path: P,
compression_enabled: bool,
compression_level: i32,
) -> Result<Self> {
let base_path = base_path.as_ref().to_path_buf();
std::fs::create_dir_all(&base_path)?;
Ok(Self {
base_path,
compression_enabled,
compression_level,
})
}
fn get_index_path(&self, index_id: &str) -> PathBuf {
let filename = if self.compression_enabled {
format!("{}.idx.zst", index_id)
} else {
format!("{}.idx", index_id)
};
self.base_path.join(filename)
}
}
impl StorageBackend for WarmTierStorage {
fn load_index(&self, index_id: &str) -> Result<Vec<u8>> {
let path = self.get_index_path(index_id);
let data = std::fs::read(&path)?;
if self.compression_enabled {
Ok(oxiarc_zstd::decode_all(&data)
.map_err(|e| anyhow::anyhow!("Zstd decompression failed: {}", e))?)
} else {
Ok(data)
}
}
fn save_index(&mut self, index_id: &str, data: &[u8]) -> Result<()> {
let path = self.get_index_path(index_id);
let final_data = if self.compression_enabled {
oxiarc_zstd::encode_all(data, self.compression_level)
.map_err(|e| anyhow::anyhow!("Zstd compression failed: {}", e))?
} else {
data.to_vec()
};
std::fs::write(&path, final_data)?;
Ok(())
}
fn delete_index(&mut self, index_id: &str) -> Result<()> {
let path = self.get_index_path(index_id);
std::fs::remove_file(&path)?;
Ok(())
}
fn exists(&self, index_id: &str) -> bool {
self.get_index_path(index_id).exists()
}
fn get_size(&self, index_id: &str) -> Result<u64> {
let path = self.get_index_path(index_id);
Ok(std::fs::metadata(&path)?.len())
}
fn list_indices(&self) -> Result<Vec<String>> {
let mut indices = Vec::new();
for entry in std::fs::read_dir(&self.base_path)? {
let entry = entry?;
if let Some(filename) = entry.file_name().to_str() {
if filename.ends_with(".idx") || filename.ends_with(".idx.zst") {
let index_id = filename
.trim_end_matches(".idx.zst")
.trim_end_matches(".idx")
.to_string();
indices.push(index_id);
}
}
}
Ok(indices)
}
fn backend_type(&self) -> &'static str {
"WarmTier (Memory-Mapped)"
}
}
pub struct ColdTierStorage {
base_path: PathBuf,
compression_level: i32,
}
impl ColdTierStorage {
pub fn new<P: AsRef<Path>>(base_path: P, compression_level: i32) -> Result<Self> {
let base_path = base_path.as_ref().to_path_buf();
std::fs::create_dir_all(&base_path)?;
Ok(Self {
base_path,
compression_level,
})
}
fn get_index_path(&self, index_id: &str) -> PathBuf {
self.base_path.join(format!("{}.idx.zst", index_id))
}
}
impl StorageBackend for ColdTierStorage {
fn load_index(&self, index_id: &str) -> Result<Vec<u8>> {
let path = self.get_index_path(index_id);
let compressed_data = std::fs::read(&path)?;
oxiarc_zstd::decode_all(&compressed_data)
.map_err(|e| anyhow::anyhow!("Zstd decompression failed: {}", e))
}
fn save_index(&mut self, index_id: &str, data: &[u8]) -> Result<()> {
let path = self.get_index_path(index_id);
let compressed_data = oxiarc_zstd::encode_all(data, self.compression_level)
.map_err(|e| anyhow::anyhow!("Zstd compression failed: {}", e))?;
std::fs::write(&path, compressed_data)?;
Ok(())
}
fn delete_index(&mut self, index_id: &str) -> Result<()> {
let path = self.get_index_path(index_id);
std::fs::remove_file(&path)?;
Ok(())
}
fn exists(&self, index_id: &str) -> bool {
self.get_index_path(index_id).exists()
}
fn get_size(&self, index_id: &str) -> Result<u64> {
let path = self.get_index_path(index_id);
Ok(std::fs::metadata(&path)?.len())
}
fn list_indices(&self) -> Result<Vec<String>> {
let mut indices = Vec::new();
for entry in std::fs::read_dir(&self.base_path)? {
let entry = entry?;
if let Some(filename) = entry.file_name().to_str() {
if filename.ends_with(".idx.zst") {
let index_id = filename.trim_end_matches(".idx.zst").to_string();
indices.push(index_id);
}
}
}
Ok(indices)
}
fn backend_type(&self) -> &'static str {
"ColdTier (Compressed Disk)"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_hot_tier_storage() -> Result<()> {
let mut storage = HotTierStorage::new();
let data = vec![1, 2, 3, 4, 5];
storage.save_index("test", &data)?;
assert!(storage.exists("test"));
let __val = storage.get_size("test")?;
assert_eq!(__val, 5);
let loaded = storage.load_index("test")?;
assert_eq!(loaded, data);
storage.delete_index("test")?;
assert!(!storage.exists("test"));
Ok(())
}
#[test]
fn test_warm_tier_storage() -> Result<()> {
use std::env;
let temp_dir = env::temp_dir().join("oxirs_warm_tier_test");
std::fs::create_dir_all(&temp_dir)?;
let mut storage = WarmTierStorage::new(&temp_dir, true, 6)?;
let data = vec![1, 2, 3, 4, 5];
storage.save_index("test", &data)?;
assert!(storage.exists("test"));
let loaded = storage.load_index("test")?;
assert_eq!(loaded, data);
storage.delete_index("test")?;
assert!(!storage.exists("test"));
std::fs::remove_dir_all(&temp_dir).ok();
Ok(())
}
#[test]
fn test_cold_tier_storage() -> Result<()> {
use std::env;
let temp_dir = env::temp_dir().join("oxirs_cold_tier_test");
std::fs::create_dir_all(&temp_dir)?;
let mut storage = ColdTierStorage::new(&temp_dir, 19)?;
let data = vec![1, 2, 3, 4, 5];
storage.save_index("test", &data)?;
assert!(storage.exists("test"));
let loaded = storage.load_index("test")?;
assert_eq!(loaded, data);
storage.delete_index("test")?;
assert!(!storage.exists("test"));
std::fs::remove_dir_all(&temp_dir).ok();
Ok(())
}
}