use crate::error::{Result, TdbError};
use anyhow::Context;
use chrono::{DateTime, Duration, Utc};
use parking_lot::{Mutex, RwLock};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ReplicationMode {
MasterSlave,
MasterMaster,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum SyncMode {
Async,
Sync,
SemiSync,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ReplicaRole {
Master,
Slave,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ReplicationStatus {
Healthy,
Lagging,
Unreachable,
Error,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicaNode {
pub node_id: String,
pub endpoint: String,
pub role: ReplicaRole,
pub status: ReplicationStatus,
pub last_sync: DateTime<Utc>,
pub lag_ms: i64,
pub last_lsn: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicationChange {
pub lsn: u64,
pub txn_id: String,
pub timestamp: DateTime<Utc>,
pub data: Vec<u8>,
pub source_node: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ConflictResolution {
LastWriteWins,
FirstWriteWins,
Manual,
Custom,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicationConfig {
pub mode: ReplicationMode,
pub sync_mode: SyncMode,
pub conflict_resolution: ConflictResolution,
pub max_lag_ms: i64,
pub heartbeat_interval: Duration,
pub auto_failover: bool,
pub quorum_size: usize,
pub change_buffer_size: usize,
}
impl Default for ReplicationConfig {
fn default() -> Self {
Self {
mode: ReplicationMode::MasterSlave,
sync_mode: SyncMode::Async,
conflict_resolution: ConflictResolution::LastWriteWins,
max_lag_ms: 1000,
heartbeat_interval: Duration::seconds(5),
auto_failover: true,
quorum_size: 2,
change_buffer_size: 10000,
}
}
}
pub struct ReplicationManager {
node_id: String,
config: ReplicationConfig,
role: Arc<RwLock<ReplicaRole>>,
replicas: Arc<RwLock<HashMap<String, ReplicaNode>>>,
change_buffer: Arc<Mutex<VecDeque<ReplicationChange>>>,
current_lsn: Arc<Mutex<u64>>,
stats: Arc<Mutex<ReplicationStats>>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReplicationStats {
pub total_changes: u64,
pub total_bytes_replicated: u64,
pub successful_replications: u64,
pub failed_replications: u64,
pub conflicts_detected: u64,
pub conflicts_resolved: u64,
pub failovers: u64,
pub avg_replication_latency_ms: f64,
total_latency_ms: f64,
}
impl ReplicationManager {
pub fn new(node_id: String, config: ReplicationConfig) -> Self {
let initial_role = match config.mode {
ReplicationMode::MasterSlave => ReplicaRole::Master,
ReplicationMode::MasterMaster => ReplicaRole::Master,
};
Self {
node_id,
config,
role: Arc::new(RwLock::new(initial_role)),
replicas: Arc::new(RwLock::new(HashMap::new())),
change_buffer: Arc::new(Mutex::new(VecDeque::new())),
current_lsn: Arc::new(Mutex::new(0)),
stats: Arc::new(Mutex::new(ReplicationStats::default())),
}
}
pub async fn add_replica(&mut self, node_id: String, endpoint: String) -> Result<()> {
let replica = ReplicaNode {
node_id: node_id.clone(),
endpoint,
role: ReplicaRole::Slave,
status: ReplicationStatus::Healthy,
last_sync: Utc::now(),
lag_ms: 0,
last_lsn: 0,
};
self.replicas.write().insert(node_id, replica);
Ok(())
}
pub async fn remove_replica(&mut self, node_id: &str) -> Result<()> {
self.replicas.write().remove(node_id);
Ok(())
}
pub async fn promote_to_master(&mut self, node_id: &str) -> Result<()> {
let mut replicas = self.replicas.write();
if let Some(replica) = replicas.get_mut(node_id) {
replica.role = ReplicaRole::Master;
replica.status = ReplicationStatus::Healthy;
let mut stats = self.stats.lock();
stats.failovers += 1;
}
Ok(())
}
pub async fn record_change(&mut self, txn_id: String, data: Vec<u8>) -> Result<u64> {
let lsn = {
let mut current_lsn = self.current_lsn.lock();
*current_lsn += 1;
*current_lsn
};
let change = ReplicationChange {
lsn,
txn_id,
timestamp: Utc::now(),
data: data.clone(),
source_node: self.node_id.clone(),
};
let mut buffer = self.change_buffer.lock();
buffer.push_back(change);
while buffer.len() > self.config.change_buffer_size {
buffer.pop_front();
}
let mut stats = self.stats.lock();
stats.total_changes += 1;
stats.total_bytes_replicated += data.len() as u64;
Ok(lsn)
}
pub async fn replicate_changes(&mut self, changes: Vec<ReplicationChange>) -> Result<()> {
let start_time = Utc::now();
match self.config.sync_mode {
SyncMode::Async => self.replicate_async(changes).await?,
SyncMode::Sync => self.replicate_sync(changes).await?,
SyncMode::SemiSync => self.replicate_semi_sync(changes).await?,
}
let latency = (Utc::now() - start_time).num_milliseconds() as f64;
let mut stats = self.stats.lock();
stats.total_latency_ms += latency;
stats.avg_replication_latency_ms = stats.total_latency_ms / stats.total_changes as f64;
Ok(())
}
async fn replicate_async(&self, changes: Vec<ReplicationChange>) -> Result<()> {
let replicas = self.replicas.read().clone();
for (node_id, _replica) in replicas.iter() {
self.send_changes_to_replica(node_id, &changes).await?;
}
let mut stats = self.stats.lock();
stats.successful_replications += replicas.len() as u64;
Ok(())
}
async fn replicate_sync(&self, changes: Vec<ReplicationChange>) -> Result<()> {
let replicas = self.replicas.read().clone();
let mut success_count = 0;
for (node_id, _replica) in replicas.iter() {
match self.send_changes_to_replica(node_id, &changes).await {
Ok(_) => success_count += 1,
Err(_) => {
let mut stats = self.stats.lock();
stats.failed_replications += 1;
}
}
}
if success_count != replicas.len() {
return Err(TdbError::Other(format!(
"Synchronous replication failed: {}/{} replicas acknowledged",
success_count,
replicas.len()
)));
}
let mut stats = self.stats.lock();
stats.successful_replications += success_count as u64;
Ok(())
}
async fn replicate_semi_sync(&self, changes: Vec<ReplicationChange>) -> Result<()> {
let replicas = self.replicas.read().clone();
let mut success_count = 0;
for (node_id, _replica) in replicas.iter() {
match self.send_changes_to_replica(node_id, &changes).await {
Ok(_) => success_count += 1,
Err(_) => {
let mut stats = self.stats.lock();
stats.failed_replications += 1;
}
}
}
if success_count == 0 {
return Err(TdbError::Other(
"Semi-sync replication failed: no replicas acknowledged".to_string(),
));
}
let mut stats = self.stats.lock();
stats.successful_replications += success_count as u64;
Ok(())
}
async fn send_changes_to_replica(
&self,
_node_id: &str,
_changes: &[ReplicationChange],
) -> Result<()> {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
Ok(())
}
pub async fn check_replica_health(&self) -> Result<()> {
let now = Utc::now();
let max_lag = self.config.max_lag_ms;
let mut replicas = self.replicas.write();
for replica in replicas.values_mut() {
let lag = (now - replica.last_sync).num_milliseconds();
replica.lag_ms = lag;
if lag > max_lag {
replica.status = ReplicationStatus::Lagging;
} else {
replica.status = ReplicationStatus::Healthy;
}
}
Ok(())
}
pub async fn resolve_conflict(
&mut self,
change1: &ReplicationChange,
change2: &ReplicationChange,
) -> Result<ReplicationChange> {
let mut stats = self.stats.lock();
stats.conflicts_detected += 1;
let resolved = match self.config.conflict_resolution {
ConflictResolution::LastWriteWins => {
if change1.timestamp > change2.timestamp {
change1.clone()
} else {
change2.clone()
}
}
ConflictResolution::FirstWriteWins => {
if change1.timestamp < change2.timestamp {
change1.clone()
} else {
change2.clone()
}
}
ConflictResolution::Manual | ConflictResolution::Custom => {
if change1.timestamp > change2.timestamp {
change1.clone()
} else {
change2.clone()
}
}
};
stats.conflicts_resolved += 1;
Ok(resolved)
}
pub async fn check_and_failover(&mut self) -> Result<bool> {
if !self.config.auto_failover {
return Ok(false);
}
let replicas = self.replicas.read().clone();
let unhealthy_masters: Vec<_> = replicas
.values()
.filter(|r| r.role == ReplicaRole::Master && r.status != ReplicationStatus::Healthy)
.collect();
if unhealthy_masters.is_empty() {
return Ok(false);
}
let best_slave = replicas
.values()
.filter(|r| r.role == ReplicaRole::Slave && r.status == ReplicationStatus::Healthy)
.max_by_key(|r| r.last_lsn);
if let Some(slave) = best_slave {
self.promote_to_master(&slave.node_id).await?;
return Ok(true);
}
Ok(false)
}
pub fn node_id(&self) -> &str {
&self.node_id
}
pub fn role(&self) -> ReplicaRole {
*self.role.read()
}
pub fn stats(&self) -> ReplicationStats {
self.stats.lock().clone()
}
pub fn replica_count(&self) -> usize {
self.replicas.read().len()
}
pub fn healthy_replica_count(&self) -> usize {
self.replicas
.read()
.values()
.filter(|r| r.status == ReplicationStatus::Healthy)
.count()
}
pub fn current_lsn(&self) -> u64 {
*self.current_lsn.lock()
}
pub fn change_buffer_size(&self) -> usize {
self.change_buffer.lock().len()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_replication_manager_creation() {
let config = ReplicationConfig::default();
let manager = ReplicationManager::new("node1".to_string(), config);
assert_eq!(manager.node_id(), "node1");
assert_eq!(manager.role(), ReplicaRole::Master);
assert_eq!(manager.replica_count(), 0);
}
#[tokio::test]
async fn test_add_replicas() {
let config = ReplicationConfig::default();
let mut manager = ReplicationManager::new("node1".to_string(), config);
manager
.add_replica("replica1".to_string(), "http://replica1:8080".to_string())
.await
.unwrap();
manager
.add_replica("replica2".to_string(), "http://replica2:8080".to_string())
.await
.unwrap();
assert_eq!(manager.replica_count(), 2);
}
#[tokio::test]
async fn test_record_change() {
let config = ReplicationConfig::default();
let mut manager = ReplicationManager::new("node1".to_string(), config);
let lsn = manager
.record_change("txn-001".to_string(), vec![1, 2, 3, 4])
.await
.unwrap();
assert_eq!(lsn, 1);
assert_eq!(manager.current_lsn(), 1);
let stats = manager.stats();
assert_eq!(stats.total_changes, 1);
assert_eq!(stats.total_bytes_replicated, 4);
}
#[tokio::test]
async fn test_async_replication() {
let config = ReplicationConfig {
sync_mode: SyncMode::Async,
..Default::default()
};
let mut manager = ReplicationManager::new("node1".to_string(), config);
manager
.add_replica("replica1".to_string(), "http://replica1:8080".to_string())
.await
.unwrap();
let changes = vec![ReplicationChange {
lsn: 1,
txn_id: "txn-001".to_string(),
timestamp: Utc::now(),
data: vec![1, 2, 3],
source_node: "node1".to_string(),
}];
manager.replicate_changes(changes).await.unwrap();
let stats = manager.stats();
assert_eq!(stats.successful_replications, 1);
}
#[tokio::test]
async fn test_sync_replication() {
let config = ReplicationConfig {
sync_mode: SyncMode::Sync,
..Default::default()
};
let mut manager = ReplicationManager::new("node1".to_string(), config);
manager
.add_replica("replica1".to_string(), "http://replica1:8080".to_string())
.await
.unwrap();
manager
.add_replica("replica2".to_string(), "http://replica2:8080".to_string())
.await
.unwrap();
let changes = vec![ReplicationChange {
lsn: 1,
txn_id: "txn-001".to_string(),
timestamp: Utc::now(),
data: vec![1, 2, 3],
source_node: "node1".to_string(),
}];
manager.replicate_changes(changes).await.unwrap();
let stats = manager.stats();
assert_eq!(stats.successful_replications, 2);
}
#[tokio::test]
async fn test_semi_sync_replication() {
let config = ReplicationConfig {
sync_mode: SyncMode::SemiSync,
..Default::default()
};
let mut manager = ReplicationManager::new("node1".to_string(), config);
manager
.add_replica("replica1".to_string(), "http://replica1:8080".to_string())
.await
.unwrap();
let changes = vec![ReplicationChange {
lsn: 1,
txn_id: "txn-001".to_string(),
timestamp: Utc::now(),
data: vec![1, 2, 3],
source_node: "node1".to_string(),
}];
manager.replicate_changes(changes).await.unwrap();
let stats = manager.stats();
assert!(stats.successful_replications >= 1);
}
#[tokio::test]
async fn test_promote_to_master() {
let config = ReplicationConfig::default();
let mut manager = ReplicationManager::new("node1".to_string(), config);
manager
.add_replica("replica1".to_string(), "http://replica1:8080".to_string())
.await
.unwrap();
manager.promote_to_master("replica1").await.unwrap();
let stats = manager.stats();
assert_eq!(stats.failovers, 1);
}
#[tokio::test]
async fn test_conflict_resolution_last_write_wins() {
let config = ReplicationConfig {
conflict_resolution: ConflictResolution::LastWriteWins,
mode: ReplicationMode::MasterMaster,
..Default::default()
};
let mut manager = ReplicationManager::new("node1".to_string(), config);
let change1 = ReplicationChange {
lsn: 1,
txn_id: "txn-001".to_string(),
timestamp: Utc::now(),
data: vec![1, 2, 3],
source_node: "node1".to_string(),
};
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
let change2 = ReplicationChange {
lsn: 2,
txn_id: "txn-001".to_string(),
timestamp: Utc::now(),
data: vec![4, 5, 6],
source_node: "node2".to_string(),
};
let resolved = manager.resolve_conflict(&change1, &change2).await.unwrap();
assert_eq!(resolved.data, vec![4, 5, 6]);
let stats = manager.stats();
assert_eq!(stats.conflicts_detected, 1);
assert_eq!(stats.conflicts_resolved, 1);
}
#[tokio::test]
async fn test_conflict_resolution_first_write_wins() {
let config = ReplicationConfig {
conflict_resolution: ConflictResolution::FirstWriteWins,
mode: ReplicationMode::MasterMaster,
..Default::default()
};
let mut manager = ReplicationManager::new("node1".to_string(), config);
let change1 = ReplicationChange {
lsn: 1,
txn_id: "txn-001".to_string(),
timestamp: Utc::now(),
data: vec![1, 2, 3],
source_node: "node1".to_string(),
};
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
let change2 = ReplicationChange {
lsn: 2,
txn_id: "txn-001".to_string(),
timestamp: Utc::now(),
data: vec![4, 5, 6],
source_node: "node2".to_string(),
};
let resolved = manager.resolve_conflict(&change1, &change2).await.unwrap();
assert_eq!(resolved.data, vec![1, 2, 3]);
}
#[tokio::test]
async fn test_replica_health_check() {
let config = ReplicationConfig {
max_lag_ms: 100,
..Default::default()
};
let manager = ReplicationManager::new("node1".to_string(), config);
manager.replicas.write().insert(
"replica1".to_string(),
ReplicaNode {
node_id: "replica1".to_string(),
endpoint: "http://replica1:8080".to_string(),
role: ReplicaRole::Slave,
status: ReplicationStatus::Healthy,
last_sync: Utc::now() - Duration::seconds(1),
lag_ms: 0,
last_lsn: 10,
},
);
manager.check_replica_health().await.unwrap();
let replicas = manager.replicas.read();
let replica = replicas.get("replica1").unwrap();
assert!(replica.lag_ms > 100);
assert_eq!(replica.status, ReplicationStatus::Lagging);
}
#[tokio::test]
async fn test_change_buffer() {
let config = ReplicationConfig {
change_buffer_size: 3,
..Default::default()
};
let mut manager = ReplicationManager::new("node1".to_string(), config);
for i in 0..5 {
manager
.record_change(format!("txn-{:03}", i), vec![i as u8])
.await
.unwrap();
}
assert_eq!(manager.change_buffer_size(), 3);
}
#[tokio::test]
async fn test_replication_stats() {
let config = ReplicationConfig::default();
let mut manager = ReplicationManager::new("node1".to_string(), config);
manager
.add_replica("replica1".to_string(), "http://replica1:8080".to_string())
.await
.unwrap();
for i in 0..5 {
manager
.record_change(format!("txn-{:03}", i), vec![1, 2, 3])
.await
.unwrap();
}
let stats = manager.stats();
assert_eq!(stats.total_changes, 5);
assert_eq!(stats.total_bytes_replicated, 15);
}
#[tokio::test]
async fn test_remove_replica() {
let config = ReplicationConfig::default();
let mut manager = ReplicationManager::new("node1".to_string(), config);
manager
.add_replica("replica1".to_string(), "http://replica1:8080".to_string())
.await
.unwrap();
assert_eq!(manager.replica_count(), 1);
manager.remove_replica("replica1").await.unwrap();
assert_eq!(manager.replica_count(), 0);
}
}