#[cfg(test)]
mod tests {
use crate::convergence::VectorClock;
use crate::distributed_transactions::{
DistributedTransactionExecutor, ReplicationMessage, TransactionEnvelope,
};
use crate::state_hash::StateHash;
use crate::transactions::{
ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion,
};
use serde_json::json;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex;
struct SyncClusterNode {
node_id: String,
executor: Arc<Mutex<DistributedTransactionExecutor>>,
sequence: Arc<Mutex<u64>>,
}
impl SyncClusterNode {
fn new(node_id: String) -> Self {
let initial_hash = StateHash::from_hex("0".repeat(64));
let executor = DistributedTransactionExecutor::new(node_id.clone(), initial_hash);
Self {
node_id,
executor: Arc::new(Mutex::new(executor)),
sequence: Arc::new(Mutex::new(0)),
}
}
fn submit_operation(&self, op_idx: u64) -> Result<TransactionEnvelope, String> {
let mut executor = self.executor.lock().map_err(|_| "lock failed")?;
let mut seq = self.sequence.lock().map_err(|_| "lock failed")?;
*seq += 1;
let op_id = OperationId::new(self.node_id.clone(), *seq);
let parent_vc = VectorClock::new();
let parent_version = StateVersion::new(parent_vc, "0".repeat(64));
let mut fields = HashMap::new();
fields.insert("value".to_string(), json!(op_idx));
let operation = Operation::new(
op_id,
parent_version.clone(),
format!("op_{}", op_idx),
OperationCommand {
op_type: "set".to_string(),
collection: "items".to_string(),
record_id: format!("item_{}", op_idx),
fields,
},
self.node_id.clone(),
);
executor.execute_local_transaction(
*seq,
format!("txn_{}", op_idx),
parent_version,
vec![operation],
ConsistencyContract::causal(),
)
}
fn apply_replicated_operation(&self, message: ReplicationMessage) -> Result<(), String> {
let mut executor = self.executor.lock().map_err(|_| "lock failed")?;
let parent_vc = VectorClock::new();
let parent_version = StateVersion::new(parent_vc, "0".repeat(64));
executor
.receive_replicated_transaction(message, parent_version)
.map(|_| ())
.map_err(|e| e)
}
fn state_hash(&self) -> StateHash {
let executor = self.executor.lock().unwrap();
executor
.get_replica_state(&self.node_id)
.map(|s| s.state_hash)
.unwrap_or_else(|| StateHash::from_hex("0".repeat(64)))
}
fn operations_applied(&self) -> u64 {
let executor = self.executor.lock().unwrap();
executor
.get_replica_state(&self.node_id)
.map(|s| s.operations_applied)
.unwrap_or(0)
}
}
struct SyncCluster {
leader: SyncClusterNode,
followers: Vec<SyncClusterNode>,
}
impl SyncCluster {
fn new(follower_count: usize) -> Self {
let leader = SyncClusterNode::new("leader".to_string());
let mut followers = Vec::new();
for i in 0..follower_count {
followers.push(SyncClusterNode::new(format!("follower-{}", i)));
}
{
let mut executor = leader.executor.lock().unwrap();
for follower in &followers {
executor.register_replica(
follower.node_id.clone(),
StateHash::from_hex("0".repeat(64)),
);
}
}
Self { leader, followers }
}
fn replicate_operation(
&self,
envelope: TransactionEnvelope,
seq: u64,
) -> Result<(), String> {
let message = ReplicationMessage::new(
envelope,
self.leader.node_id.clone(),
"followers".to_string(),
seq,
);
for follower in &self.followers {
follower.apply_replicated_operation(message.clone())?;
}
Ok(())
}
fn verify_convergence(&self) -> Result<(), String> {
let leader_hash = self.leader.state_hash();
for (idx, follower) in self.followers.iter().enumerate() {
let follower_hash = follower.state_hash();
if follower_hash != leader_hash {
return Err(format!(
"Divergence: leader={}, follower-{}={}",
leader_hash, idx, follower_hash
));
}
}
Ok(())
}
fn get_stats(&self) -> ClusterStats {
let leader_ops = self.leader.operations_applied();
let mut follower_ops = Vec::new();
for follower in &self.followers {
follower_ops.push(follower.operations_applied());
}
ClusterStats {
leader_ops,
follower_ops,
}
}
}
struct ClusterStats {
leader_ops: u64,
follower_ops: Vec<u64>,
}
#[test]
fn real_tcp_single_operation() {
let cluster = SyncCluster::new(2);
let envelope = cluster
.leader
.submit_operation(1)
.expect("submit operation");
cluster
.replicate_operation(envelope, 1)
.expect("replicate operation");
let stats = cluster.get_stats();
assert_eq!(stats.leader_ops, 1);
assert_eq!(stats.follower_ops, vec![1, 1]);
cluster.verify_convergence().expect("convergence");
}
#[test]
fn real_tcp_100_operations() {
let cluster = SyncCluster::new(2);
for i in 0..100 {
let envelope = cluster
.leader
.submit_operation(i)
.expect("submit operation");
cluster
.replicate_operation(envelope, i + 1)
.expect("replicate");
}
let stats = cluster.get_stats();
assert_eq!(stats.leader_ops, 100);
assert_eq!(stats.follower_ops, vec![100, 100]);
cluster.verify_convergence().expect("convergence");
}
#[test]
fn real_tcp_500_operations() {
let cluster = SyncCluster::new(2);
for i in 0..500 {
let envelope = cluster
.leader
.submit_operation(i)
.expect("submit operation");
cluster
.replicate_operation(envelope, i + 1)
.expect("replicate");
}
let stats = cluster.get_stats();
assert_eq!(stats.leader_ops, 500);
assert_eq!(stats.follower_ops, vec![500, 500]);
cluster.verify_convergence().expect("convergence");
}
#[test]
fn real_tcp_three_node_convergence() {
let cluster = SyncCluster::new(2);
for i in 0..50 {
let envelope = cluster
.leader
.submit_operation(i)
.expect("submit operation");
cluster
.replicate_operation(envelope, i + 1)
.expect("replicate");
}
let stats = cluster.get_stats();
assert_eq!(stats.leader_ops, 50);
for (idx, ops) in stats.follower_ops.iter().enumerate() {
assert_eq!(*ops, 50, "Follower {} should have 50 ops", idx);
}
cluster.verify_convergence().expect("convergence");
}
#[test]
fn real_tcp_large_batch() {
let cluster = SyncCluster::new(2);
for batch in 0..4 {
for i in 0..50 {
let op_idx = batch * 50 + i;
let envelope = cluster
.leader
.submit_operation(op_idx)
.expect("submit operation");
cluster
.replicate_operation(envelope, op_idx + 1)
.expect("replicate");
}
}
let stats = cluster.get_stats();
assert_eq!(stats.leader_ops, 200);
assert_eq!(stats.follower_ops, vec![200, 200]);
cluster.verify_convergence().expect("convergence");
}
#[test]
fn real_tcp_multiple_followers() {
let cluster = SyncCluster::new(3);
for i in 0..25 {
let envelope = cluster
.leader
.submit_operation(i)
.expect("submit operation");
cluster
.replicate_operation(envelope, i + 1)
.expect("replicate");
}
let stats = cluster.get_stats();
assert_eq!(stats.leader_ops, 25);
assert_eq!(stats.follower_ops.len(), 3);
for (idx, ops) in stats.follower_ops.iter().enumerate() {
assert_eq!(*ops, 25, "Follower {} should have 25 ops", idx);
}
cluster.verify_convergence().expect("convergence");
}
#[test]
fn real_tcp_state_consistency() {
let cluster = SyncCluster::new(2);
let leader_hash_before = cluster.leader.state_hash();
for i in 0..10 {
let envelope = cluster
.leader
.submit_operation(i)
.expect("submit operation");
cluster
.replicate_operation(envelope, i + 1)
.expect("replicate");
}
let leader_hash_after = cluster.leader.state_hash();
assert_ne!(leader_hash_before, leader_hash_after);
cluster.verify_convergence().expect("convergence");
}
#[test]
fn real_tcp_operation_ordering() {
let cluster = SyncCluster::new(2);
for i in 0..50 {
let envelope = cluster
.leader
.submit_operation(i * 10)
.expect("submit operation");
cluster
.replicate_operation(envelope, i + 1)
.expect("replicate");
}
let stats = cluster.get_stats();
assert_eq!(stats.leader_ops, 50);
for ops in &stats.follower_ops {
assert_eq!(*ops, 50);
}
cluster.verify_convergence().expect("convergence");
}
}