use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use parking_lot::RwLock;
use crate::types::*;
use crate::cid::*;
use crate::graph::*;
use crate::storage::{lsm::*, merkle::*, mvcc::*};
#[derive(Debug, Clone)]
pub struct PersistentStorageConfig {
pub data_dir: PathBuf,
pub memtable_size: usize,
pub sstable_max_size: usize,
pub compaction_interval: u64,
pub snapshot_interval: u64,
}
impl Default for PersistentStorageConfig {
fn default() -> Self {
Self {
data_dir: PathBuf::from("./data"),
memtable_size: 1000,
sstable_max_size: 10 * 1024 * 1024, compaction_interval: 3600, snapshot_interval: 86400, }
}
}
#[derive(Debug)]
pub struct PersistentStorage {
cid_manager: Arc<RwLock<CidManager>>,
lsm_tree: Arc<RwLock<LSMTree>>,
merkle_dag: Arc<RwLock<MerkleDAG>>,
mvcc_manager: Arc<RwLock<MVCCManager>>,
config: PersistentStorageConfig,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum StorageResult<T> {
Success(T),
NotFound,
VersionConflict,
IntegrityError(String),
IOError(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PersistedGraph {
pub cid: Cid,
pub vertex_cids: Vec<Cid>,
pub edge_cids: Vec<Cid>,
pub metadata_cid: Option<Cid>,
pub timestamp: u64,
}
impl PersistentStorage {
pub fn new(config: PersistentStorageConfig) -> Result<Self> {
std::fs::create_dir_all(&config.data_dir)?;
let cid_manager = Arc::new(RwLock::new(CidManager::new()));
let lsm_tree = Arc::new(RwLock::new(LSMTree::new(
config.data_dir.join("lsm"),
config.memtable_size,
config.sstable_max_size,
)?));
let merkle_dag = Arc::new(RwLock::new(MerkleDAG::new()));
let mvcc_manager = Arc::new(RwLock::new(MVCCManager::new()));
Ok(Self {
cid_manager,
lsm_tree,
merkle_dag,
mvcc_manager,
config,
})
}
pub fn store_graph(&self, graph: &Graph) -> Result<Cid> {
let mut cid_manager = self.cid_manager.write();
let mut merkle_dag = self.merkle_dag.write();
let graph_cid = cid_manager.compute_graph_cid(&GraphCore {
nodes: graph.vertices.values().cloned().collect(),
edges: graph.edges.values().cloned().collect(),
boundary: None,
attrs: None,
})?;
let mut vertex_cids = Vec::new();
for vertex in graph.vertices.values() {
let vertex_cid = cid_manager.compute_node_cid(vertex)?;
let vertex_key = format!("vertex:{}", vertex_cid.as_str());
let vertex_data = serde_json::to_vec(vertex)?;
self.store_data(&vertex_key, &vertex_data)?;
vertex_cids.push(vertex_cid);
}
let mut edge_cids = Vec::new();
for edge in graph.edges.values() {
let edge_cid = cid_manager.compute_edge_cid(edge)?;
let edge_key = format!("edge:{}", edge_cid.as_str());
let edge_data = serde_json::to_vec(edge)?;
self.store_data(&edge_key, &edge_data)?;
edge_cids.push(edge_cid);
}
let persisted_graph = PersistedGraph {
cid: graph_cid.clone(),
vertex_cids,
edge_cids,
metadata_cid: None,
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs(),
};
let metadata_key = format!("graph:{}", graph_cid.as_str());
let metadata_data = serde_json::to_vec(&persisted_graph)?;
self.store_data(&metadata_key, &metadata_data)?;
Ok(graph_cid)
}
pub fn load_graph(&self, cid: &Cid) -> Result<Graph> {
let metadata_key = format!("graph:{}", cid.as_str());
let metadata_data = match self.load_data(&metadata_key)? {
StorageResult::Success(data) => data,
StorageResult::NotFound => return Err(KotobaError::Storage("Graph not found".to_string())),
_ => return Err(KotobaError::Storage("Failed to load graph metadata".to_string())),
};
let persisted_graph: PersistedGraph = serde_json::from_slice(&metadata_data)?;
let mut vertices = HashMap::new();
for vertex_cid in &persisted_graph.vertex_cids {
let vertex_key = format!("vertex:{}", vertex_cid.as_str());
let vertex_data = match self.load_data(&vertex_key)? {
StorageResult::Success(data) => data,
_ => continue, };
let vertex: VertexData = serde_json::from_slice(&vertex_data)?;
vertices.insert(vertex.id, vertex);
}
let mut edges = HashMap::new();
for edge_cid in &persisted_graph.edge_cids {
let edge_key = format!("edge:{}", edge_cid.as_str());
let edge_data = match self.load_data(&edge_key)? {
StorageResult::Success(data) => data,
_ => continue, };
let edge: EdgeData = serde_json::from_slice(&edge_data)?;
edges.insert(edge.id, edge);
}
let mut graph = Graph::empty();
for vertex in vertices.values() {
graph.add_vertex(vertex.clone());
}
for edge in edges.values() {
graph.add_edge(edge.clone());
}
Ok(graph)
}
pub fn store_data(&self, key: &str, data: &[u8]) -> Result<()> {
let mut lsm_tree = self.lsm_tree.write();
lsm_tree.put(key.to_string(), data.to_vec());
Ok(())
}
pub fn load_data(&self, key: &str) -> Result<StorageResult<Vec<u8>>> {
let lsm_tree = self.lsm_tree.read();
match lsm_tree.get(key)? {
Some(data) => Ok(StorageResult::Success(data)),
None => Ok(StorageResult::NotFound),
}
}
pub fn delete_data(&self, key: &str) -> Result<()> {
let mut lsm_tree = self.lsm_tree.write();
lsm_tree.delete(key.to_string());
Ok(())
}
pub fn get_merkle_root(&self) -> ContentHash {
let merkle_dag = self.merkle_dag.read();
let mut hasher = sha2::Sha256::new();
let mut sorted_hashes: Vec<_> = merkle_dag.nodes.keys().collect();
sorted_hashes.sort();
for hash in sorted_hashes {
hasher.update(hash.0.as_bytes());
}
ContentHash(format!("{:x}", hasher.finalize()))
}
pub fn verify_integrity(&self) -> Result<bool> {
let merkle_dag = self.merkle_dag.read();
for node in merkle_dag.nodes.values() {
let mut hasher = sha2::Sha256::new();
hasher.update(&node.data);
for child in &node.children {
hasher.update(child.0.as_bytes());
}
let computed_hash = format!("{:x}", hasher.finalize());
if computed_hash != node.hash.0 {
return Ok(false);
}
}
Ok(true)
}
pub fn create_snapshot(&self) -> Result<String> {
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let snapshot_id = format!("snapshot_{}", timestamp);
let lsm_tree = self.lsm_tree.read();
lsm_tree.create_snapshot(&snapshot_id)?;
let merkle_dag = self.merkle_dag.read();
let merkle_snapshot_path = self.config.data_dir.join(format!("merkle_{}", snapshot_id));
let merkle_data = serde_json::to_vec(&merkle_dag.nodes)?;
std::fs::write(merkle_snapshot_path, merkle_data)?;
Ok(snapshot_id)
}
pub fn restore_from_snapshot(&self, snapshot_id: &str) -> Result<()> {
let mut lsm_tree = self.lsm_tree.write();
lsm_tree.restore_from_snapshot(snapshot_id)?;
let merkle_snapshot_path = self.config.data_dir.join(format!("merkle_{}", snapshot_id));
if merkle_snapshot_path.exists() {
let merkle_data = std::fs::read(merkle_snapshot_path)?;
let nodes: HashMap<ContentHash, MerkleNode> = serde_json::from_slice(&merkle_data)?;
let mut merkle_dag = self.merkle_dag.write();
merkle_dag.nodes = nodes;
}
Ok(())
}
pub fn compact(&self) -> Result<()> {
let mut lsm_tree = self.lsm_tree.write();
lsm_tree.compact()?;
Ok(())
}
pub fn get_stats(&self) -> StorageStats {
let lsm_tree = self.lsm_tree.read();
let merkle_dag = self.merkle_dag.read();
let mvcc_manager = self.mvcc_manager.read();
StorageStats {
lsm_entries: lsm_tree.stats().total_entries,
merkle_nodes: merkle_dag.len(),
active_transactions: mvcc_manager.active_transactions(),
data_size: lsm_tree.stats().total_size,
}
}
pub fn cleanup(&self, max_age_days: u64) -> Result<()> {
let cutoff_time = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs()
.saturating_sub(max_age_days * 24 * 3600);
let mut lsm_tree = self.lsm_tree.write();
lsm_tree.cleanup(cutoff_time)?;
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StorageStats {
pub lsm_entries: usize,
pub merkle_nodes: usize,
pub active_transactions: usize,
pub data_size: u64,
}
#[derive(Debug)]
pub struct DistributedStorageManager {
local_storage: Arc<PersistentStorage>,
nodes: HashMap<NodeId, NodeStorageInfo>,
consistency_config: ConsistencyConfig,
}
#[derive(Debug, Clone)]
pub struct ConsistencyConfig {
pub check_interval_secs: u64,
pub replication_factor: usize,
pub read_consistency: ConsistencyLevel,
pub write_consistency: ConsistencyLevel,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ConsistencyLevel {
One,
Quorum,
All,
}
#[derive(Debug, Clone)]
pub struct ConsistencyCheck {
pub cid: Cid,
pub available_nodes: usize,
pub required_replicas: usize,
pub is_consistent: bool,
pub missing_nodes: Vec<NodeId>,
pub corrupted_nodes: Vec<NodeId>,
}
#[derive(Debug, Clone)]
pub struct ConflictResolution {
pub cid: Cid,
pub resolved_version: Cid,
pub conflicting_versions: Vec<Cid>,
pub resolution_method: ConflictResolutionMethod,
}
#[derive(Debug, Clone)]
pub enum ConflictResolutionMethod {
LatestTimestamp,
OldestTimestamp,
Merge,
Manual,
}
#[derive(Debug, Clone)]
pub struct NodeStorageInfo {
pub node_id: NodeId,
pub address: String,
pub cid_ranges: Vec<CidRange>,
pub last_sync: u64,
}
impl DistributedStorageManager {
pub fn new(local_storage: Arc<PersistentStorage>) -> Self {
Self {
local_storage,
nodes: HashMap::new(),
consistency_config: ConsistencyConfig {
check_interval_secs: 300, replication_factor: 3,
read_consistency: ConsistencyLevel::Quorum,
write_consistency: ConsistencyLevel::Quorum,
},
}
}
pub fn with_consistency_config(mut self, config: ConsistencyConfig) -> Self {
self.consistency_config = config;
self
}
pub async fn check_consistency(&self, cid: &Cid) -> Result<ConsistencyCheck> {
let responsible_nodes = self.get_responsible_nodes(cid);
let mut available_nodes = 0;
let mut missing_nodes = Vec::new();
let mut corrupted_nodes = Vec::new();
let mut reference_data: Option<Vec<u8>> = None;
let mut data_found = false;
for node_info in &responsible_nodes {
match self.fetch_data_from_node(node_info, cid).await {
Ok(data) => {
available_nodes += 1;
data_found = true;
if let Some(ref ref_data) = reference_data {
if *ref_data != data {
corrupted_nodes.push(node_info.node_id.clone());
}
} else {
reference_data = Some(data);
}
}
Err(_) => {
missing_nodes.push(node_info.node_id.clone());
}
}
}
let required_replicas = self.consistency_config.replication_factor;
let is_consistent = data_found &&
available_nodes >= required_replicas &&
corrupted_nodes.is_empty() &&
missing_nodes.len() <= (responsible_nodes.len() - required_replicas);
Ok(ConsistencyCheck {
cid: cid.clone(),
available_nodes,
required_replicas,
is_consistent,
missing_nodes,
corrupted_nodes,
})
}
pub async fn replicate_data(&self, cid: &Cid, data: &[u8], replication_factor: usize) -> Result<()> {
let responsible_nodes = self.get_responsible_nodes(cid);
for node_info in responsible_nodes.iter().take(replication_factor) {
self.send_data_to_node(node_info, cid, data).await?;
}
Ok(())
}
pub async fn resolve_conflicts(&self, cid: &Cid, versions: &[Cid]) -> Result<ConflictResolution> {
if versions.is_empty() {
return Err(KotobaError::Storage("No versions provided".to_string()));
}
if versions.len() == 1 {
return Ok(ConflictResolution {
cid: cid.clone(),
resolved_version: versions[0].clone(),
conflicting_versions: vec![],
resolution_method: ConflictResolutionMethod::LatestTimestamp,
});
}
let mut version_info = Vec::new();
for version in versions {
if let Ok(data) = self.local_storage.load_data(&format!("cid:{}", version.as_str())) {
if let StorageResult::Success(data_bytes) = data {
version_info.push((version.clone(), data_bytes.len() as u64));
}
}
}
version_info.sort_by_key(|(_, timestamp)| *timestamp);
let resolved_version = version_info.last().unwrap().0.clone();
let conflicting_versions = versions.iter()
.filter(|v| *v != &resolved_version)
.cloned()
.collect();
Ok(ConflictResolution {
cid: cid.clone(),
resolved_version,
conflicting_versions,
resolution_method: ConflictResolutionMethod::LatestTimestamp,
})
}
pub async fn ensure_read_consistency(&self, cid: &Cid) -> Result<Option<Vec<u8>>> {
match self.consistency_config.read_consistency {
ConsistencyLevel::One => {
self.local_storage.load_data(&format!("cid:{}", cid.as_str()))
.map(|result| match result {
StorageResult::Success(data) => Some(data),
_ => None,
})
}
ConsistencyLevel::Quorum => {
let check = self.check_consistency(cid).await?;
if check.is_consistent && check.available_nodes >= check.required_replicas {
self.local_storage.load_data(&format!("cid:{}", cid.as_str()))
.map(|result| match result {
StorageResult::Success(data) => Some(data),
_ => None,
})
} else {
Ok(None)
}
}
ConsistencyLevel::All => {
let responsible_nodes = self.get_responsible_nodes(cid);
let total_nodes = responsible_nodes.len();
let check = self.check_consistency(cid).await?;
if check.available_nodes == total_nodes && check.is_consistent {
self.local_storage.load_data(&format!("cid:{}", cid.as_str()))
.map(|result| match result {
StorageResult::Success(data) => Some(data),
_ => None,
})
} else {
Ok(None)
}
}
}
}
pub async fn ensure_write_consistency(&self, cid: &Cid, data: &[u8]) -> Result<()> {
self.local_storage.store_data(&format!("cid:{}", cid.as_str()), data)?;
match self.consistency_config.write_consistency {
ConsistencyLevel::One => {
Ok(())
}
ConsistencyLevel::Quorum | ConsistencyLevel::All => {
let replication_count = match self.consistency_config.write_consistency {
ConsistencyLevel::Quorum => (self.nodes.len() / 2) + 1,
ConsistencyLevel::All => self.nodes.len(),
_ => unreachable!(),
};
self.replicate_data(cid, data, replication_count.max(1)).await?;
Ok(())
}
}
}
fn get_responsible_nodes(&self, cid: &Cid) -> Vec<&NodeStorageInfo> {
let mut responsible = Vec::new();
for node_info in self.nodes.values() {
for range in &node_info.cid_ranges {
let hash = cid.as_str().as_bytes();
let hash_value = hash.iter().fold(0u64, |acc, &b| acc.wrapping_add(b as u64));
if hash_value >= range.start && hash_value <= range.end {
responsible.push(node_info);
break;
}
}
}
responsible
}
async fn fetch_data_from_node(&self, node_info: &NodeStorageInfo, cid: &Cid) -> Result<Vec<u8>> {
if node_info.node_id.0 == "local" {
match self.local_storage.load_data(&format!("cid:{}", cid.as_str()))? {
StorageResult::Success(data) => Ok(data),
_ => Err(KotobaError::Storage("Data not found".to_string())),
}
} else {
Err(KotobaError::Storage("Remote node communication not implemented".to_string()))
}
}
async fn send_data_to_node(&self, node_info: &NodeStorageInfo, cid: &Cid, data: &[u8]) -> Result<()> {
if node_info.node_id.0 == "local" {
self.local_storage.store_data(&format!("cid:{}", cid.as_str()), data)?;
Ok(())
} else {
Err(KotobaError::Storage("Remote node communication not implemented".to_string()))
}
}
pub fn get_responsible_node(&self, cid: &Cid) -> Option<&NodeStorageInfo> {
let hash = cid.as_str().as_bytes();
let hash_value = hash.iter().fold(0u64, |acc, &b| acc.wrapping_add(b as u64));
for node_info in self.nodes.values() {
for range in &node_info.cid_ranges {
if hash_value >= range.start && hash_value <= range.end {
return Some(node_info);
}
}
}
None
}
pub async fn replicate_data(&self, cid: &Cid, data: &[u8], replication_factor: usize) -> Result<()> {
let responsible_nodes: Vec<_> = (0..replication_factor)
.filter_map(|_| self.get_responsible_node(cid))
.collect();
for node_info in responsible_nodes {
println!("Replicating {} to node {}", cid.as_str(), node_info.node_id.0);
}
Ok(())
}
pub async fn verify_consistency(&self, cid: &Cid) -> Result<bool> {
match self.local_storage.load_data(&format!("cid:{}", cid.as_str()))? {
StorageResult::Success(_) => Ok(true),
_ => Ok(false),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
fn create_test_config() -> PersistentStorageConfig {
let temp_dir = tempdir().unwrap();
PersistentStorageConfig {
data_dir: temp_dir.path().to_path_buf(),
memtable_size: 10,
sstable_max_size: 1024,
compaction_interval: 3600,
snapshot_interval: 86400,
}
}
fn create_test_graph() -> Graph {
let mut graph = Graph::empty();
let v1 = graph.add_vertex(VertexData {
id: VertexId::new("v1").unwrap(),
labels: vec!["Person".to_string()],
props: HashMap::new(),
});
let v2 = graph.add_vertex(VertexData {
id: VertexId::new("v2").unwrap(),
labels: vec!["Person".to_string()],
props: HashMap::new(),
});
graph.add_edge(EdgeData {
id: EdgeId::new("e1").unwrap(),
src: v1,
dst: v2,
label: "FOLLOWS".to_string(),
props: HashMap::new(),
});
graph
}
#[test]
fn test_store_and_load_graph() {
let config = create_test_config();
let storage = PersistentStorage::new(config).unwrap();
let test_graph = create_test_graph();
let cid = storage.store_graph(&test_graph).unwrap();
let loaded_graph = storage.load_graph(&cid).unwrap();
assert_eq!(loaded_graph.vertices.len(), test_graph.vertices.len());
assert_eq!(loaded_graph.edges.len(), test_graph.edges.len());
}
#[test]
fn test_data_operations() {
let config = create_test_config();
let storage = PersistentStorage::new(config).unwrap();
let key = "test_key";
let data = b"test_data";
storage.store_data(key, data).unwrap();
match storage.load_data(key).unwrap() {
StorageResult::Success(loaded_data) => assert_eq!(loaded_data, data),
_ => panic!("Data not found"),
}
storage.delete_data(key).unwrap();
match storage.load_data(key).unwrap() {
StorageResult::NotFound => {} _ => panic!("Data should be deleted"),
}
}
#[test]
fn test_integrity_verification() {
let config = create_test_config();
let storage = PersistentStorage::new(config).unwrap();
assert!(storage.verify_integrity().unwrap());
}
#[test]
fn test_storage_stats() {
let config = create_test_config();
let storage = PersistentStorage::new(config).unwrap();
let stats = storage.get_stats();
assert_eq!(stats.lsm_entries, 0);
assert_eq!(stats.merkle_nodes, 0);
assert_eq!(stats.active_transactions, 0);
}
}