use std::sync::{Arc, Mutex};
use dashmap::DashMap;
use tracing::{debug, warn};
use crate::a2a::A2ARouter;
use crate::error::{KernelError, KernelResult};
use crate::ipc::{KernelMessage, MessageTarget};
use crate::mesh_chain::{ChainSyncRequest, ChainSyncResponse};
use crate::mesh_heartbeat::{HeartbeatConfig, HeartbeatTracker};
use crate::mesh_ipc::MeshIpcEnvelope;
use crate::mesh_kad::KademliaTable;
pub struct PeerConnection {
pub node_id: String,
pub connected_at: chrono::DateTime<chrono::Utc>,
pub sender: tokio::sync::mpsc::Sender<Vec<u8>>,
}
pub struct DiscoveryState {
pub kademlia: Mutex<KademliaTable>,
pub peer_addresses: DashMap<String, String>,
pub heartbeat: Mutex<HeartbeatTracker>,
}
pub struct MeshRuntime {
node_id: String,
peers: DashMap<String, PeerConnection>,
local_router: Option<Arc<A2ARouter>>,
discovery: Option<DiscoveryState>,
}
impl MeshRuntime {
pub fn new(node_id: String) -> Self {
Self {
node_id,
peers: DashMap::new(),
local_router: None,
discovery: None,
}
}
pub fn with_discovery(node_id: String, kademlia_id: [u8; 32]) -> Self {
Self {
node_id,
peers: DashMap::new(),
local_router: None,
discovery: Some(DiscoveryState {
kademlia: Mutex::new(KademliaTable::new(kademlia_id)),
peer_addresses: DashMap::new(),
heartbeat: Mutex::new(HeartbeatTracker::new(HeartbeatConfig::default())),
}),
}
}
pub fn node_id(&self) -> &str {
&self.node_id
}
pub fn set_local_router(&mut self, router: Arc<A2ARouter>) {
self.local_router = Some(router);
}
pub fn add_peer(&self, node_id: String, sender: tokio::sync::mpsc::Sender<Vec<u8>>) {
debug!(peer = %node_id, "adding peer connection");
self.peers.insert(
node_id.clone(),
PeerConnection {
node_id,
connected_at: chrono::Utc::now(),
sender,
},
);
}
pub async fn send_to_peer(
&self,
node_id: &str,
envelope: MeshIpcEnvelope,
) -> KernelResult<()> {
let peer = self
.peers
.get(node_id)
.ok_or_else(|| KernelError::Mesh(format!("peer not connected: {node_id}")))?;
let bytes = envelope
.to_bytes()
.map_err(|e| KernelError::Mesh(format!("serialization error: {e}")))?;
peer.sender
.send(bytes)
.await
.map_err(|_| KernelError::Mesh(format!("send channel closed for peer {node_id}")))?;
debug!(peer = %node_id, "sent envelope to peer");
Ok(())
}
pub async fn route_to_remote(
&self,
node_id: &str,
message: KernelMessage,
) -> KernelResult<()> {
let envelope =
MeshIpcEnvelope::new(self.node_id.clone(), node_id.to_string(), message);
self.send_to_peer(node_id, envelope).await
}
pub async fn handle_incoming(&self, data: &[u8]) -> KernelResult<()> {
let envelope = MeshIpcEnvelope::from_bytes(data)
.map_err(|e| KernelError::Mesh(format!("deserialization error: {e}")))?;
debug!(
from_node = %envelope.source_node,
dest_node = %envelope.dest_node,
"received mesh envelope"
);
let router = self.local_router.as_ref().ok_or_else(|| {
KernelError::Mesh("no local router attached to mesh runtime".into())
})?;
let mut message = envelope.message;
if let MessageTarget::RemoteNode { target, .. } = message.target {
message.target = *target;
}
router.send(message).await
}
pub fn peer_count(&self) -> usize {
self.peers.len()
}
pub fn peer_ids(&self) -> Vec<String> {
self.peers
.iter()
.map(|entry| entry.key().clone())
.collect()
}
pub fn disconnect_peer(&self, node_id: &str) {
if self.peers.remove(node_id).is_some() {
debug!(peer = %node_id, "disconnected peer");
} else {
warn!(peer = %node_id, "disconnect_peer: peer not found");
}
}
pub fn register_peer_address(&self, node_id: &str, addr: &str) {
if let Some(ref disc) = self.discovery {
disc.peer_addresses
.insert(node_id.to_string(), addr.to_string());
}
}
pub fn discover_peers(&self) -> Vec<(String, String)> {
match self.discovery.as_ref() {
Some(disc) => disc
.peer_addresses
.iter()
.map(|entry| (entry.key().clone(), entry.value().clone()))
.collect(),
None => Vec::new(),
}
}
pub fn record_heartbeat(&self, node_id: &str) {
if let Some(ref disc) = self.discovery {
let mut hb = disc.heartbeat.lock().unwrap();
if hb.peer_state(node_id).is_none() {
hb.add_peer(node_id.to_string());
}
hb.record_alive(node_id);
}
}
pub fn check_peer_health(&self) -> Vec<String> {
match self.discovery.as_ref() {
Some(disc) => {
let hb = disc.heartbeat.lock().unwrap();
let mut unhealthy: Vec<String> = hb
.suspect_peers()
.into_iter()
.map(|s| s.to_string())
.collect();
unhealthy.extend(
hb.dead_peers()
.into_iter()
.map(|s| s.to_string()),
);
unhealthy
}
None => Vec::new(),
}
}
pub fn remove_dead_peers(&self) {
if let Some(ref disc) = self.discovery {
let dead: Vec<String> = {
let hb = disc.heartbeat.lock().unwrap();
hb.dead_peers()
.into_iter()
.map(|s| s.to_string())
.collect()
};
for node_id in &dead {
self.disconnect_peer(node_id);
disc.peer_addresses.remove(node_id.as_str());
}
}
}
pub fn discovery_mut(&mut self) -> Option<&mut DiscoveryState> {
self.discovery.as_mut()
}
pub fn discovery(&self) -> Option<&DiscoveryState> {
self.discovery.as_ref()
}
pub fn build_chain_sync_request(&self, from_seq: u64) -> Vec<u8> {
let req = ChainSyncRequest {
chain_id: 0,
after_sequence: from_seq,
after_hash: String::new(),
max_events: 256,
};
serde_json::to_vec(&req).unwrap_or_default()
}
pub fn handle_chain_sync_response(&self, data: &[u8]) -> KernelResult<usize> {
let resp: ChainSyncResponse = serde_json::from_slice(data)
.map_err(|e| KernelError::Mesh(format!("chain sync deserialize error: {e}")))?;
Ok(resp.events.len())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::capability::{AgentCapabilities, CapabilityChecker};
use crate::ipc::{KernelMessage, MessagePayload, MessageTarget};
use crate::mesh_heartbeat::HeartbeatState;
use crate::mesh_ipc::MeshIpcEnvelope;
use crate::process::{ProcessEntry, ProcessState, ProcessTable, ResourceUsage};
use crate::topic::TopicRouter;
use tokio_util::sync::CancellationToken;
fn make_router_with_process() -> (
Arc<A2ARouter>,
crate::process::Pid,
tokio::sync::mpsc::Receiver<KernelMessage>,
) {
let table = Arc::new(ProcessTable::new(64));
let entry = ProcessEntry {
pid: 0,
agent_id: "test-agent".into(),
state: ProcessState::Running,
capabilities: AgentCapabilities::default(),
resource_usage: ResourceUsage::default(),
cancel_token: CancellationToken::new(),
parent_pid: None,
};
let pid = table.insert(entry).unwrap();
let checker = Arc::new(CapabilityChecker::new(table.clone()));
let topics = Arc::new(TopicRouter::new(table.clone()));
let router = Arc::new(A2ARouter::new(table, checker, topics));
let rx = router.create_inbox(pid);
(router, pid, rx)
}
#[test]
fn new_runtime_has_no_peers() {
let rt = MeshRuntime::new("node-local".into());
assert_eq!(rt.peer_count(), 0);
assert!(rt.peer_ids().is_empty());
assert_eq!(rt.node_id(), "node-local");
}
#[test]
fn add_peer_increases_count() {
let rt = MeshRuntime::new("local".into());
let (tx, _rx) = tokio::sync::mpsc::channel(16);
rt.add_peer("peer-1".into(), tx);
assert_eq!(rt.peer_count(), 1);
assert_eq!(rt.peer_ids(), vec!["peer-1".to_string()]);
}
#[tokio::test]
async fn send_to_peer_delivers_serialized_bytes() {
let rt = MeshRuntime::new("local".into());
let (tx, mut rx) = tokio::sync::mpsc::channel(16);
rt.add_peer("peer-1".into(), tx);
let msg = KernelMessage::text(0, MessageTarget::Broadcast, "hello");
let envelope = MeshIpcEnvelope::new("local".into(), "peer-1".into(), msg);
let expected_id = envelope.envelope_id.clone();
rt.send_to_peer("peer-1", envelope).await.unwrap();
let received = rx.recv().await.unwrap();
let decoded = MeshIpcEnvelope::from_bytes(&received).unwrap();
assert_eq!(decoded.envelope_id, expected_id);
assert_eq!(decoded.source_node, "local");
assert_eq!(decoded.dest_node, "peer-1");
}
#[tokio::test]
async fn handle_incoming_injects_into_local_router() {
let (router, pid, mut inbox_rx) = make_router_with_process();
let mut rt = MeshRuntime::new("node-b".into());
rt.set_local_router(router);
let inner_target = MessageTarget::Process(pid);
let remote_target = MessageTarget::RemoteNode {
node_id: "node-b".into(),
target: Box::new(inner_target),
};
let msg = KernelMessage::text(pid, remote_target, "from-remote");
let envelope = MeshIpcEnvelope::new("node-a".into(), "node-b".into(), msg);
let bytes = envelope.to_bytes().unwrap();
rt.handle_incoming(&bytes).await.unwrap();
let delivered = inbox_rx.try_recv().unwrap();
assert!(matches!(delivered.target, MessageTarget::Process(p) if p == pid));
match &delivered.payload {
MessagePayload::Text(s) => assert_eq!(s, "from-remote"),
other => panic!("expected Text payload, got: {other:?}"),
}
}
#[test]
fn disconnect_peer_removes_it() {
let rt = MeshRuntime::new("local".into());
let (tx, _rx) = tokio::sync::mpsc::channel(16);
rt.add_peer("peer-1".into(), tx);
assert_eq!(rt.peer_count(), 1);
rt.disconnect_peer("peer-1");
assert_eq!(rt.peer_count(), 0);
}
#[tokio::test]
async fn round_trip_send_receive() {
let (router, pid, mut inbox_rx) = make_router_with_process();
let rt_a = MeshRuntime::new("node-a".into());
let (tx, mut peer_rx) = tokio::sync::mpsc::channel(16);
rt_a.add_peer("node-b".into(), tx);
let msg = KernelMessage::text(
pid,
MessageTarget::RemoteNode {
node_id: "node-b".into(),
target: Box::new(MessageTarget::Process(pid)),
},
"round-trip",
);
rt_a.route_to_remote("node-b", msg).await.unwrap();
let wire_bytes = peer_rx.recv().await.unwrap();
let mut rt_b = MeshRuntime::new("node-b".into());
rt_b.set_local_router(router);
rt_b.handle_incoming(&wire_bytes).await.unwrap();
let delivered = inbox_rx.try_recv().unwrap();
match &delivered.payload {
MessagePayload::Text(s) => assert_eq!(s, "round-trip"),
other => panic!("expected Text payload, got: {other:?}"),
}
}
#[test]
fn peer_connection_stores_node_id_and_timestamp() {
let rt = MeshRuntime::new("local".into());
let (tx, _rx) = tokio::sync::mpsc::channel(16);
rt.add_peer("remote-42".into(), tx);
let entry = rt.peers.get("remote-42").unwrap();
assert_eq!(entry.node_id, "remote-42");
let age = chrono::Utc::now() - entry.connected_at;
assert!(age.num_seconds() < 2);
}
#[test]
fn multiple_peers_connected() {
let rt = MeshRuntime::new("local".into());
for i in 0..5 {
let (tx, _rx) = tokio::sync::mpsc::channel(16);
rt.add_peer(format!("peer-{i}"), tx);
}
assert_eq!(rt.peer_count(), 5);
let mut ids = rt.peer_ids();
ids.sort();
assert_eq!(ids, vec!["peer-0", "peer-1", "peer-2", "peer-3", "peer-4"]);
}
#[tokio::test]
async fn send_to_unknown_peer_errors() {
let rt = MeshRuntime::new("local".into());
let msg = KernelMessage::text(0, MessageTarget::Broadcast, "hi");
let envelope = MeshIpcEnvelope::new("local".into(), "ghost".into(), msg);
let err = rt.send_to_peer("ghost", envelope).await.unwrap_err();
let msg = err.to_string();
assert!(msg.contains("peer not connected"), "got: {msg}");
}
#[tokio::test]
async fn handle_incoming_malformed_data_errors() {
let mut rt = MeshRuntime::new("local".into());
let (router, _pid, _rx) = make_router_with_process();
rt.set_local_router(router);
let err = rt.handle_incoming(b"not valid json").await.unwrap_err();
let msg = err.to_string();
assert!(msg.contains("deserialization"), "got: {msg}");
}
#[tokio::test]
async fn handle_incoming_without_router_errors() {
let rt = MeshRuntime::new("local".into());
let msg = KernelMessage::text(0, MessageTarget::Broadcast, "hi");
let envelope = MeshIpcEnvelope::new("a".into(), "local".into(), msg);
let bytes = envelope.to_bytes().unwrap();
let err = rt.handle_incoming(&bytes).await.unwrap_err();
let msg = err.to_string();
assert!(msg.contains("no local router"), "got: {msg}");
}
#[test]
fn disconnect_nonexistent_peer_is_noop() {
let rt = MeshRuntime::new("local".into());
rt.disconnect_peer("ghost"); assert_eq!(rt.peer_count(), 0);
}
#[tokio::test]
async fn two_nodes_exchange_messages() {
let (router_b, pid_b, mut inbox_b) = make_router_with_process();
let rt_a = MeshRuntime::new("node-a".into());
let mut rt_b = MeshRuntime::new("node-b".into());
rt_b.set_local_router(router_b);
let (a_to_b_tx, mut a_to_b_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(64);
let (b_to_a_tx, mut b_to_a_rx) = tokio::sync::mpsc::channel::<Vec<u8>>(64);
rt_a.add_peer("node-b".into(), a_to_b_tx);
rt_b.add_peer("node-a".into(), b_to_a_tx);
let msg_a = KernelMessage::text(
pid_b,
MessageTarget::RemoteNode {
node_id: "node-b".into(),
target: Box::new(MessageTarget::Process(pid_b)),
},
"hello-from-a",
);
rt_a.route_to_remote("node-b", msg_a).await.unwrap();
let wire_bytes = a_to_b_rx.recv().await.unwrap();
rt_b.handle_incoming(&wire_bytes).await.unwrap();
let delivered = inbox_b.try_recv().unwrap();
match &delivered.payload {
MessagePayload::Text(s) => assert_eq!(s, "hello-from-a"),
other => panic!("expected Text, got: {other:?}"),
}
let msg_b = KernelMessage::text(0, MessageTarget::Broadcast, "reply-from-b");
rt_b.route_to_remote("node-a", msg_b).await.unwrap();
let reply_bytes = b_to_a_rx.recv().await.unwrap();
let reply_env = MeshIpcEnvelope::from_bytes(&reply_bytes).unwrap();
assert_eq!(reply_env.source_node, "node-b");
assert_eq!(reply_env.dest_node, "node-a");
}
#[test]
fn discover_peers_returns_registered_addresses() {
let rt = MeshRuntime::with_discovery("node-x".into(), [0u8; 32]);
rt.register_peer_address("peer-1", "10.0.0.1:9470");
rt.register_peer_address("peer-2", "10.0.0.2:9470");
rt.register_peer_address("peer-3", "10.0.0.3:9470");
let mut peers = rt.discover_peers();
peers.sort_by(|a, b| a.0.cmp(&b.0));
assert_eq!(peers.len(), 3);
assert_eq!(peers[0], ("peer-1".into(), "10.0.0.1:9470".into()));
assert_eq!(peers[1], ("peer-2".into(), "10.0.0.2:9470".into()));
assert_eq!(peers[2], ("peer-3".into(), "10.0.0.3:9470".into()));
}
#[test]
fn heartbeat_tracking_detects_dead_peer() {
let mut rt = MeshRuntime::with_discovery("local".into(), [0u8; 32]);
{
let disc = rt.discovery_mut().unwrap();
let mut hb = disc.heartbeat.lock().unwrap();
*hb = HeartbeatTracker::new(crate::mesh_heartbeat::HeartbeatConfig {
suspect_timeout: std::time::Duration::from_secs(0),
..crate::mesh_heartbeat::HeartbeatConfig::default()
});
hb.add_peer("healthy".into());
hb.add_peer("dying".into());
}
rt.record_heartbeat("healthy");
{
let disc = rt.discovery_mut().unwrap();
let mut hb = disc.heartbeat.lock().unwrap();
hb.record_miss("dying");
hb.record_miss("dying");
}
let unhealthy = rt.check_peer_health();
assert!(unhealthy.contains(&"dying".to_string()));
assert!(!unhealthy.contains(&"healthy".to_string()));
}
#[test]
fn remove_dead_peers_cleans_connections() {
let mut rt = MeshRuntime::with_discovery("local".into(), [0u8; 32]);
let (tx, _rx) = tokio::sync::mpsc::channel(16);
rt.add_peer("dead-peer".into(), tx);
rt.register_peer_address("dead-peer", "10.0.0.99:9470");
{
let disc = rt.discovery_mut().unwrap();
let mut hb = disc.heartbeat.lock().unwrap();
*hb = HeartbeatTracker::new(crate::mesh_heartbeat::HeartbeatConfig {
suspect_timeout: std::time::Duration::from_secs(0),
..crate::mesh_heartbeat::HeartbeatConfig::default()
});
hb.add_peer("dead-peer".into());
hb.record_miss("dead-peer");
hb.record_miss("dead-peer");
assert_eq!(
hb.peer_state("dead-peer"),
Some(HeartbeatState::Dead)
);
}
assert_eq!(rt.peer_count(), 1);
rt.remove_dead_peers();
assert_eq!(rt.peer_count(), 0);
assert!(rt.discover_peers().is_empty());
}
#[test]
fn kademlia_routing_finds_closest_peers() {
let mut rt = MeshRuntime::with_discovery("local".into(), [0u8; 32]);
let disc = rt.discovery_mut().unwrap();
let mut kad = disc.kademlia.lock().unwrap();
for i in 1..=5u8 {
let mut peer_key = [0u8; 32];
peer_key[0] = i;
kad.add_peer(
peer_key,
crate::mesh_kad::DhtEntry {
key: format!("peer-{i}"),
node_id: format!("peer-{i}"),
address: format!("10.0.0.{i}:9470"),
platform: "linux".into(),
last_seen: 1000,
governance_genesis_prefix: "0000000000000000".into(),
},
);
}
let closest = kad.find_closest(&[0u8; 32], 3);
assert_eq!(closest.len(), 3);
}
#[test]
fn with_discovery_initializes_state() {
let rt = MeshRuntime::with_discovery("disc-node".into(), [0xAB; 32]);
assert_eq!(rt.node_id(), "disc-node");
assert!(rt.discovery().is_some());
let disc = rt.discovery().unwrap();
assert_eq!(*disc.kademlia.lock().unwrap().local_key(), [0xAB; 32]);
assert_eq!(disc.heartbeat.lock().unwrap().peer_count(), 0);
}
#[test]
fn discover_peers_empty_without_discovery() {
let rt = MeshRuntime::new("plain-node".into());
assert!(rt.discover_peers().is_empty());
rt.register_peer_address("x", "y");
rt.record_heartbeat("x");
assert!(rt.check_peer_health().is_empty());
}
#[test]
fn chain_sync_request_round_trip() {
let rt = MeshRuntime::new("sync-node".into());
let bytes = rt.build_chain_sync_request(42);
let req: crate::mesh_chain::ChainSyncRequest =
serde_json::from_slice(&bytes).unwrap();
assert_eq!(req.chain_id, 0);
assert_eq!(req.after_sequence, 42);
assert_eq!(req.max_events, 256);
}
#[test]
fn chain_sync_response_handling() {
let rt = MeshRuntime::new("sync-node".into());
let resp = crate::mesh_chain::ChainSyncResponse {
chain_id: 0,
events: vec![
serde_json::json!({"type": "write"}),
serde_json::json!({"type": "delete"}),
],
has_more: false,
tip_sequence: 100,
tip_hash: "abc".into(),
};
let data = serde_json::to_vec(&resp).unwrap();
let count = rt.handle_chain_sync_response(&data).unwrap();
assert_eq!(count, 2);
}
#[test]
fn chain_sync_response_malformed_errors() {
let rt = MeshRuntime::new("sync-node".into());
let err = rt.handle_chain_sync_response(b"bad json").unwrap_err();
assert!(err.to_string().contains("chain sync deserialize"));
}
}