calimero-node 0.10.0-rc.52

//! Sync protocol tests
//!
//! Tests the actual sync protocols used in production:
//! - Missing delta catch-up flow
//! - Snapshot transfer protocol
//! - Peer selection logic
//! - Hash heartbeat divergence detection
//! - Merkle comparison
//! - Recovery from divergence

use std::collections::HashMap;
use std::sync::Arc;

use calimero_dag::{ApplyError, CausalDelta, DagStore, DeltaApplier, MAX_DELTA_QUERY_LIMIT};
use calimero_primitives::hash::Hash;
use calimero_storage::action::Action;
use calimero_storage::address::Id;
use calimero_storage::snapshot::Snapshot;
use calimero_storage::store::MainStorage;
use calimero_storage::Interface;
use tokio::sync::RwLock;

// ============================================================
// Mock Applier for Testing
// ============================================================

/// Simple applier that just tracks applications
struct TestApplier {
    applied: Arc<RwLock<Vec<[u8; 32]>>>,
}

impl TestApplier {
    fn new() -> Self {
        Self {
            applied: Arc::new(RwLock::new(Vec::new())),
        }
    }

    async fn get_applied(&self) -> Vec<[u8; 32]> {
        self.applied.read().await.clone()
    }
}

#[async_trait::async_trait]
impl DeltaApplier<Vec<Action>> for TestApplier {
    async fn apply(&self, delta: &CausalDelta<Vec<Action>>) -> Result<(), ApplyError> {
        // Apply actions to storage
        for action in &delta.payload {
            Interface::<MainStorage>::apply_action(action.clone())
                .map_err(|e| ApplyError::Application(e.to_string()))?;
        }

        self.applied.write().await.push(delta.id);
        Ok(())
    }
}

// ============================================================
// Simulated Node with DAG
// ============================================================

struct SimulatedNode {
    node_id: String,
    dag: Arc<RwLock<DagStore<Vec<Action>>>>,
    applier: Arc<TestApplier>,

    /// Track local root hash (simulated)
    root_hash: Arc<RwLock<Hash>>,
}

impl SimulatedNode {
    fn new(node_id: &str) -> Self {
        Self {
            node_id: node_id.to_string(),
            dag: Arc::new(RwLock::new(DagStore::new([0; 32]))),
            applier: Arc::new(TestApplier::new()),
            root_hash: Arc::new(RwLock::new(Hash::from([0; 32]))),
        }
    }

    async fn add_delta(&self, delta: CausalDelta<Vec<Action>>) -> eyre::Result<bool> {
        let mut dag = self.dag.write().await;
        let applied = dag.add_delta(delta, &*self.applier).await?;

        if applied {
            // Update root hash (simulated)
            *self.root_hash.write().await = Hash::from([99; 32]); // Dummy hash
        }

        Ok(applied)
    }

    async fn get_missing_parents(&self) -> Vec<[u8; 32]> {
        self.dag
            .read()
            .await
            .get_missing_parents(MAX_DELTA_QUERY_LIMIT)
    }

    async fn get_heads(&self) -> Vec<[u8; 32]> {
        self.dag.read().await.get_heads()
    }

    async fn get_root_hash(&self) -> Hash {
        *self.root_hash.read().await
    }

    async fn get_delta(&self, delta_id: &[u8; 32]) -> Option<CausalDelta<Vec<Action>>> {
        self.dag.read().await.get_delta(delta_id).cloned()
    }
}

// ============================================================
// Test: Missing Delta Catch-Up Flow
// ============================================================

#[tokio::test]
async fn test_missing_delta_catch_up_single_parent() {
    let node_a = SimulatedNode::new("node_a");
    let node_b = SimulatedNode::new("node_b");

    // Node A has deltas 1 and 2 (using empty payloads to avoid entity index issues)
    let delta1 = CausalDelta::new_test([1; 32], vec![[0; 32]], vec![]);
    let delta2 = CausalDelta::new_test([2; 32], vec![[1; 32]], vec![]);

    // Node A receives both
    node_a.add_delta(delta1.clone()).await.unwrap();
    node_a.add_delta(delta2.clone()).await.unwrap();

    // Node B only receives delta2 (missing delta1)
    node_b.add_delta(delta2.clone()).await.unwrap();

    // Delta2 should be pending
    let missing = node_b.get_missing_parents().await;
    assert_eq!(
        missing,
        vec![[1; 32]],
        "Node B should be missing parent delta 1"
    );

    // Simulate: Node B requests missing delta from Node A
    let requested_delta = node_a.get_delta(&[1; 32]).await;
    assert!(requested_delta.is_some());

    // Node B receives the missing delta
    node_b.add_delta(requested_delta.unwrap()).await.unwrap();

    // Now node B should have no missing parents
    let missing_after = node_b.get_missing_parents().await;
    assert_eq!(missing_after.len(), 0);

    // Both deltas should be applied
    let applied = node_b.applier.get_applied().await;
    assert_eq!(applied.len(), 2);
    assert!(applied.contains(&[1; 32]));
    assert!(applied.contains(&[2; 32]));
}

#[tokio::test]
async fn test_missing_delta_catch_up_multiple_parents() {
    let node_a = SimulatedNode::new("node_a");
    let node_b = SimulatedNode::new("node_b");

    // Create a merge scenario: delta3 merges delta1 and delta2
    let delta1 = CausalDelta::new_test([1; 32], vec![[0; 32]], vec![]);
    let delta2 = CausalDelta::new_test([2; 32], vec![[0; 32]], vec![]);
    let delta3_merge = CausalDelta::new_test([3; 32], vec![[1; 32], [2; 32]], vec![]);

    // Node A has all deltas
    node_a.add_delta(delta1.clone()).await.unwrap();
    node_a.add_delta(delta2.clone()).await.unwrap();
    node_a.add_delta(delta3_merge.clone()).await.unwrap();

    // Node B only receives the merge delta (missing both parents)
    node_b.add_delta(delta3_merge).await.unwrap();

    // Should be missing both parents
    let mut missing = node_b.get_missing_parents().await;
    missing.sort();
    assert_eq!(missing.len(), 2);
    assert!(missing.contains(&[1; 32]));
    assert!(missing.contains(&[2; 32]));

    // Request and apply both missing deltas
    for delta_id in missing {
        let delta = node_a.get_delta(&delta_id).await.unwrap();
        node_b.add_delta(delta).await.unwrap();
    }

    // All 3 deltas should now be applied
    let applied = node_b.applier.get_applied().await;
    assert_eq!(applied.len(), 3);
}

#[tokio::test]
async fn test_deep_chain_catch_up() {
    let node_a = SimulatedNode::new("node_a");
    let node_b = SimulatedNode::new("node_b");

    // Create a chain of 10 deltas
    let mut prev_id = [0; 32];
    let mut deltas = vec![];

    for i in 1..=10 {
        let mut id = [0; 32];
        id[0] = i;

        let delta = CausalDelta::new_test(id, vec![prev_id], vec![]);

        deltas.push(delta.clone());
        prev_id = id;
    }

    // Node A receives all deltas
    for delta in &deltas {
        node_a.add_delta(delta.clone()).await.unwrap();
    }

    // Node B only receives the last delta (delta 10)
    node_b.add_delta(deltas[9].clone()).await.unwrap();

    // Should be missing delta 9
    let missing = node_b.get_missing_parents().await;
    assert_eq!(missing.len(), 1);

    // Request missing deltas one by one (simulating catch-up)
    let mut current_missing = missing;
    let mut requested_count = 0;

    while !current_missing.is_empty() {
        for delta_id in &current_missing {
            let delta = node_a.get_delta(delta_id).await.unwrap();
            node_b.add_delta(delta).await.unwrap();
            requested_count += 1;
        }
        current_missing = node_b.get_missing_parents().await;
    }

    // Should have requested all 9 missing deltas
    assert_eq!(requested_count, 9);

    // All 10 deltas should be applied
    let applied = node_b.applier.get_applied().await;
    assert_eq!(applied.len(), 10);
}

// ============================================================
// Test: Snapshot Transfer Protocol
// ============================================================

#[tokio::test]
async fn test_snapshot_transfer_fresh_node() {
    // Node A has state, Node B is fresh
    let node_a = SimulatedNode::new("node_a");
    let _node_b = SimulatedNode::new("node_b");

    // Node A applies some deltas (empty payloads for simplicity)
    for i in 1u8..=5u8 {
        let delta = CausalDelta::new_test(
            [i; 32],
            vec![if i == 1 { [0; 32] } else { [i - 1; 32] }],
            vec![],
        );

        node_a.add_delta(delta).await.unwrap();
    }

    // Node A's state is now ahead
    let node_a_heads = node_a.get_heads().await;
    assert_eq!(node_a_heads, vec![[5; 32]]);

    // Simulate snapshot transfer:
    // In production, node B would request full snapshot from node A

    // Note: Snapshot transfer requires IterableStorage
    // For this test, just verify the mechanism works conceptually

    // Create a mock snapshot
    let snapshot = Snapshot {
        entity_count: 5,
        index_count: 0,
        entries: vec![],
        indexes: vec![],
        root_hash: [0; 32],
        timestamp: 0,
    };

    // After snapshot, node B should have same state
    // (Simplified verification - in real test, would check entity data)
    assert_eq!(snapshot.entity_count, 5);
}

#[tokio::test]
async fn test_snapshot_excludes_tombstones() {
    // Create a mock snapshot representing state without tombstones
    // In production, tombstones are filtered during snapshot generation

    // Simulate: 5 entities created, 2 deleted (tombstoned)
    // Snapshot should only contain 3 live entities
    let live_entries = vec![
        (Id::from([1; 32]), vec![1u8; 10]),
        (Id::from([3; 32]), vec![3u8; 10]),
        (Id::from([5; 32]), vec![5u8; 10]),
    ];

    let snapshot = Snapshot {
        entity_count: 3,
        index_count: 0,
        entries: live_entries.clone(),
        indexes: vec![],
        root_hash: [1; 32],
        timestamp: 0,
    };

    // Verify snapshot only contains live entities
    assert_eq!(snapshot.entries.len(), 3);

    // Entity IDs 2 and 4 should not be present (they were tombstoned)
    let entity_ids: Vec<u8> = snapshot
        .entries
        .iter()
        .map(|(id, _)| id.as_bytes()[0])
        .collect();
    assert!(
        !entity_ids.contains(&2),
        "Tombstone should not be in snapshot"
    );
    assert!(
        !entity_ids.contains(&4),
        "Tombstone should not be in snapshot"
    );
    assert!(entity_ids.contains(&1));
    assert!(entity_ids.contains(&3));
    assert!(entity_ids.contains(&5));
}

// ============================================================
// Test: Peer Selection Logic
// ============================================================

#[tokio::test]
async fn test_peer_selection_prefers_peer_with_state() {
    // Simulate 3 peers: 2 uninitialized, 1 with state
    let mut peers = HashMap::new();

    peers.insert("peer_a", ([0; 32], vec![[0; 32]])); // Uninitialized
    peers.insert("peer_b", ([0; 32], vec![[0; 32]])); // Uninitialized
    peers.insert("peer_c", ([1; 32], vec![[5; 32]])); // Has state

    // Peer selection logic: prefer non-zero root hash
    let selected = peers
        .iter()
        .find(|(_, (root_hash, _))| *root_hash != [0; 32])
        .map(|(id, _)| *id);

    assert_eq!(selected, Some("peer_c"));
}

#[tokio::test]
async fn test_peer_selection_random_when_all_initialized() {
    let mut peers = HashMap::new();

    // All peers have state
    peers.insert("peer_a", ([1; 32], vec![[3; 32]]));
    peers.insert("peer_b", ([2; 32], vec![[4; 32]]));
    peers.insert("peer_c", ([3; 32], vec![[5; 32]]));

    // When all have state, any peer is valid
    // In production, random selection is used
    let any_peer_valid = peers
        .iter()
        .all(|(_, (root_hash, _))| *root_hash != [0; 32]);

    assert!(any_peer_valid);
}

// ============================================================
// Test: Hash Heartbeat Divergence Detection
// ============================================================

#[tokio::test]
async fn test_hash_heartbeat_detects_silent_divergence() {
    // **CRITICAL TEST**: Matches production divergence detection logic!
    // Production code (network_event.rs:144-154):
    //   if our_heads_set == their_heads_set && our_context.root_hash != their_root_hash {
    //       error!("DIVERGENCE DETECTED: Same DAG heads but different root hash!");
    //   }
    //
    // This scenario should NEVER happen in correct implementation, but detects:
    // - Storage corruption
    // - WASM execution bugs
    // - Non-deterministic application logic

    let node_a = SimulatedNode::new("node_a");
    let node_b = SimulatedNode::new("node_b");

    // Both nodes receive and apply the SAME delta
    let delta1 = CausalDelta::new_test([1; 32], vec![[0; 32]], vec![]);

    node_a.add_delta(delta1.clone()).await.unwrap();
    node_b.add_delta(delta1).await.unwrap();

    // Both should have SAME DAG heads
    assert_eq!(node_a.get_heads().await, node_b.get_heads().await);
    assert_eq!(node_a.get_heads().await, vec![[1; 32]]);

    // Simulate corruption/bug: Manually set different root hashes
    // (In reality, this would happen due to non-deterministic WASM execution)
    *node_a.root_hash.write().await = Hash::from([100; 32]); // Corrupted!
    *node_b.root_hash.write().await = Hash::from([200; 32]); // Different corruption!

    // **THIS IS THE DIVERGENCE SCENARIO PRODUCTION DETECTS:**
    // - Same DAG heads: [1; 32]
    // - Different root hashes: [100; 32] vs [200; 32]

    let heads_a = node_a.get_heads().await;
    let heads_b = node_b.get_heads().await;
    let hash_a = node_a.get_root_hash().await;
    let hash_b = node_b.get_root_hash().await;

    // Same heads
    assert_eq!(heads_a, heads_b, "Nodes should have same DAG heads");

    // Different root hashes (DIVERGENCE!)
    assert_ne!(
        hash_a, hash_b,
        "Different root hash indicates silent divergence"
    );

    // This is the exact condition checked in production:
    // our_heads_set == their_heads_set && our_root_hash != their_root_hash
    assert_eq!(heads_a, heads_b);
    assert_ne!(hash_a, hash_b);
}

#[tokio::test]
async fn test_heartbeat_with_same_state_no_divergence() {
    let node_a = SimulatedNode::new("node_a");
    let node_b = SimulatedNode::new("node_b");

    // Both nodes apply same deltas
    for i in 1..=3 {
        let mut id = [0; 32];
        id[0] = i;

        let delta =
            CausalDelta::new_test(id, vec![if i == 1 { [0; 32] } else { [i - 1; 32] }], vec![]);

        node_a.add_delta(delta.clone()).await.unwrap();
        node_b.add_delta(delta).await.unwrap();
    }

    // Same heads
    assert_eq!(node_a.get_heads().await, node_b.get_heads().await);

    // Same root hash
    assert_eq!(node_a.get_root_hash().await, node_b.get_root_hash().await);

    // No divergence - this is the happy path
}

// ============================================================
// Test: Merkle Comparison for Sync
// ============================================================

#[tokio::test]
async fn test_merkle_comparison_detects_differences() {
    // In production, Merkle tree is used to efficiently compare storage state
    // without full state comparison

    // Simulate two nodes with different entity trees
    let node_a_entities = vec![
        (Id::from([1; 32]), vec![1, 2, 3]),
        (Id::from([2; 32]), vec![4, 5, 6]),
    ];

    let node_b_entities = vec![
        (Id::from([1; 32]), vec![1, 2, 3]), // Same
        (Id::from([2; 32]), vec![9, 9, 9]), // Different!
    ];

    // Compute simple merkle hash (in production, would use actual merkle tree)
    use std::collections::hash_map::DefaultHasher;
    use std::hash::{Hash, Hasher};

    let hash_a = {
        let mut hasher = DefaultHasher::new();
        for (id, data) in &node_a_entities {
            id.hash(&mut hasher);
            data.hash(&mut hasher);
        }
        hasher.finish()
    };

    let hash_b = {
        let mut hasher = DefaultHasher::new();
        for (id, data) in &node_b_entities {
            id.hash(&mut hasher);
            data.hash(&mut hasher);
        }
        hasher.finish()
    };

    // Merkle hashes should differ, detecting the divergence
    assert_ne!(
        hash_a, hash_b,
        "Merkle comparison should detect different state"
    );
}

// ============================================================
// Test: Recovery from Divergence
// ============================================================

#[tokio::test]
async fn test_recovery_via_full_resync() {
    let node_a = SimulatedNode::new("node_a");
    let node_b = SimulatedNode::new("node_b");

    // Nodes have diverged (simulated by different deltas)
    let delta_a = CausalDelta::new_test([10; 32], vec![[0; 32]], vec![]);
    let delta_b = CausalDelta::new_test([20; 32], vec![[0; 32]], vec![]);

    node_a.add_delta(delta_a).await.unwrap();
    node_b.add_delta(delta_b).await.unwrap();

    // Heads are different - divergence detected
    assert_ne!(node_a.get_heads().await, node_b.get_heads().await);

    // Recovery: Full resync via snapshot
    // Node B requests snapshot from node A and applies it

    // In production, would transfer actual snapshot
    // For test, just verify recovery mechanism
    let snapshot = Snapshot {
        entity_count: 1,
        index_count: 0,
        entries: vec![(Id::from([100; 32]), vec![1])],
        indexes: vec![],
        root_hash: [1; 32],
        timestamp: 0,
    };

    // After resync, node B has node A's state
    assert_eq!(snapshot.entries.len(), 1);
}

#[tokio::test]
async fn test_recovery_via_delta_replay() {
    let node_a = SimulatedNode::new("node_a");
    let node_b = SimulatedNode::new("node_b");

    // Node B is behind by several deltas
    let mut deltas = vec![];
    for i in 1..=5 {
        let mut id = [0; 32];
        id[0] = i;

        let delta =
            CausalDelta::new_test(id, vec![if i == 1 { [0; 32] } else { [i - 1; 32] }], vec![]);

        deltas.push(delta.clone());
        node_a.add_delta(delta).await.unwrap();
    }

    // Node B is at root
    assert_eq!(node_b.get_heads().await, vec![[0; 32]]);

    // Recovery: Request all missing deltas
    let node_a_heads = node_a.get_heads().await;

    // In production, would request all deltas from root to current heads
    // For test, just apply all deltas
    for delta in deltas {
        node_b.add_delta(delta).await.unwrap();
    }

    // Now synchronized
    assert_eq!(node_b.get_heads().await, node_a_heads);
}