use crate::guardian::error::{GuardianError, Result};
use crate::p2p::messaging::{CONNECTION_TIMEOUT, HEARTBEAT_INTERVAL, MAX_MESSAGE_SIZE, PROTOCOL};
use crate::p2p::network::core::IrohBackend;
use crate::traits::{
DirectChannelEmitter, DirectChannelFactory, DirectChannelOptions, EventPubSubPayload,
};
use async_trait::async_trait;
use futures::StreamExt;
use iroh::EndpointId as NodeId;
use iroh_gossip::net::Gossip;
use iroh_gossip::proto::TopicId;
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
sync::Arc,
time::{Duration, Instant},
};
use tokio::sync::{Mutex, RwLock, broadcast, mpsc};
use tokio::task::JoinHandle;
use tracing::Span;
type TopicMessageChannels = Arc<RwLock<HashMap<TopicId, broadcast::Sender<(NodeId, Vec<u8>)>>>>;
const BEACON_TIMEOUT: Duration = Duration::from_secs(CONNECTION_TIMEOUT.as_secs() / 6);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DirectChannelMessage {
pub message_type: MessageType,
pub payload: Vec<u8>,
pub timestamp: u64,
pub sender: String, }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum MessageType {
Data,
Heartbeat,
Ack,
}
#[async_trait]
pub trait DirectChannelNetwork: Send + Sync {
async fn publish_message(&self, topic: &TopicId, message: &[u8]) -> Result<()>;
async fn subscribe_topic(&self, topic: &TopicId, bootstrap_peers: Vec<NodeId>) -> Result<()>;
async fn get_connected_peers(&self) -> Vec<NodeId>;
async fn get_topic_peers(&self, topic: &TopicId) -> Vec<NodeId>;
fn as_any(&self) -> &dyn std::any::Any;
}
pub struct IrohBridge {
span: Span,
#[allow(dead_code)] backend: Arc<IrohBackend>,
gossip: Gossip,
connected_peers: Arc<RwLock<Vec<NodeId>>>,
topic_peers: Arc<RwLock<HashMap<TopicId, Vec<NodeId>>>>,
subscribed_topics: Arc<RwLock<HashMap<TopicId, bool>>>,
own_node_id: NodeId,
topic_message_channels: TopicMessageChannels,
topic_event_loops: Arc<RwLock<HashMap<TopicId, JoinHandle<()>>>>,
}
impl IrohBridge {
pub async fn new(span: Span, backend: Arc<IrohBackend>) -> Result<Self> {
let endpoint_arc = backend.get_endpoint().await?;
let endpoint_lock = endpoint_arc.read().await;
let endpoint = endpoint_lock
.as_ref()
.ok_or_else(|| GuardianError::Other("Endpoint not available".to_string()))?
.clone();
let own_node_id = endpoint.id();
drop(endpoint_lock);
let gossip = Gossip::builder()
.max_message_size(backend.config().gossip.max_message_size)
.spawn(endpoint);
Ok(Self {
span,
backend,
gossip,
connected_peers: Arc::new(RwLock::new(Vec::new())),
topic_peers: Arc::new(RwLock::new(HashMap::new())),
subscribed_topics: Arc::new(RwLock::new(HashMap::new())),
own_node_id,
topic_message_channels: Arc::new(RwLock::new(HashMap::new())),
topic_event_loops: Arc::new(RwLock::new(HashMap::new())),
})
}
pub fn span(&self) -> &Span {
&self.span
}
pub fn node_id(&self) -> NodeId {
self.own_node_id
}
pub async fn start(&self) -> Result<()> {
let _entered = self.span.enter();
tracing::info!("IrohBridge started with iroh-gossip");
Ok(())
}
pub async fn update_connected_peers(&self, peers: Vec<NodeId>) {
let _entered = self.span.enter();
let mut connected = self.connected_peers.write().await;
*connected = peers.clone();
tracing::debug!("Connected peers updated: {}", connected.len());
}
pub async fn update_topic_peers(&self, topic: TopicId, peers: Vec<NodeId>) {
let mut topic_peers = self.topic_peers.write().await;
topic_peers.insert(topic, peers.clone());
tracing::debug!(
"Peers of topic {} updated: {}",
topic.fmt_short(),
peers.len()
);
}
async fn publish(&self, topic: &TopicId, message: &[u8]) -> Result<()> {
let subscribed_topics = self.subscribed_topics.read().await;
if !subscribed_topics.contains_key(topic) {
return Err(GuardianError::Other(format!(
"Topic {} is not subscribed for publication",
topic.fmt_short()
)));
}
drop(subscribed_topics);
let mut gossip_topic = self.gossip.subscribe(*topic, vec![]).await.map_err(|e| {
GuardianError::Other(format!("Error accessing topic for publication: {}", e))
})?;
gossip_topic
.broadcast(bytes::Bytes::copy_from_slice(message))
.await
.map_err(|e| {
GuardianError::Other(format!("Error publishing message via iroh-gossip: {}", e))
})?;
tracing::debug!(
"Message published via iroh-gossip on topic: {}",
topic.fmt_short()
);
Ok(())
}
pub async fn stop(&self) -> Result<()> {
tracing::info!("IrohBridge stopped");
Ok(())
}
pub async fn get_interface_stats(&self) -> HashMap<String, u64> {
let mut stats = HashMap::new();
let connected = self.connected_peers.read().await;
stats.insert(
"interface_connected_peers".to_string(),
connected.len() as u64,
);
let topics = self.topic_peers.read().await;
stats.insert("interface_tracked_topics".to_string(), topics.len() as u64);
stats
}
fn topic_id_from_str(topic: &str) -> TopicId {
let hash = blake3::hash(topic.as_bytes());
TopicId::from_bytes(hash.into())
}
pub async fn get_topic_receiver(
&self,
topic: &TopicId,
) -> Option<broadcast::Receiver<(NodeId, Vec<u8>)>> {
let channels = self.topic_message_channels.read().await;
channels.get(topic).map(|sender| sender.subscribe())
}
}
#[async_trait]
impl DirectChannelNetwork for IrohBridge {
async fn publish_message(&self, topic: &TopicId, message: &[u8]) -> Result<()> {
tracing::debug!(
"Publicando mensagem no tópico: {}, {} bytes",
topic.fmt_short(),
message.len()
);
self.publish(topic, message).await?;
tracing::info!(
"Mensagem publicada com sucesso no tópico via iroh-gossip: {}",
topic.fmt_short()
);
Ok(())
}
async fn subscribe_topic(&self, topic: &TopicId, bootstrap_peers: Vec<NodeId>) -> Result<()> {
tracing::debug!(
"Subscribing to topic: {} with {} bootstrap peers",
topic.fmt_short(),
bootstrap_peers.len()
);
{
let topics = self.subscribed_topics.read().await;
if topics.contains_key(topic) {
tracing::debug!(
"Topic {} is already subscribed, re-subscribing with new peers",
topic.fmt_short()
);
if !bootstrap_peers.is_empty() {
let gossip_topic_new = self
.gossip
.subscribe(*topic, bootstrap_peers.clone())
.await
.map_err(|e| {
GuardianError::Other(format!("Error re-subscribing to topic: {}", e))
})?;
let message_tx = {
let channels = self.topic_message_channels.read().await;
channels.get(topic).cloned()
};
if let Some(tx) = message_tx {
let topic_id = *topic;
let topic_peers_map = self.topic_peers.clone();
let span = self.span.clone();
tokio::spawn(async move {
let _entered = span.enter();
let mut gossip_topic = gossip_topic_new;
tracing::info!(
"[PEER_MESH] Additional event loop started for topic {} with new peers",
topic_id.fmt_short()
);
while let Some(event_result) = gossip_topic.next().await {
match event_result {
Ok(iroh_gossip::api::Event::Received(msg)) => {
tracing::info!(
"[PEER_MESH] Message received via the new mesh on topic {}: {} bytes from peer {}",
topic_id.fmt_short(),
msg.content.len(),
msg.delivered_from
);
let _ = tx.send((msg.delivered_from, msg.content.to_vec()));
}
Ok(iroh_gossip::api::Event::NeighborUp(peer_id)) => {
tracing::info!(
"[PEER_MESH] Peer {} connected to topic {} via the new mesh",
peer_id,
topic_id.fmt_short()
);
let mut peers = topic_peers_map.write().await;
peers
.entry(topic_id)
.or_insert_with(Vec::new)
.push(peer_id);
}
Ok(iroh_gossip::api::Event::NeighborDown(peer_id)) => {
tracing::debug!(
"[PEER_MESH] Peer {} disconnected from topic {} via the new mesh",
peer_id,
topic_id.fmt_short()
);
let mut peers = topic_peers_map.write().await;
if let Some(peer_list) = peers.get_mut(&topic_id) {
peer_list.retain(|p| *p != peer_id);
}
}
Ok(iroh_gossip::api::Event::Lagged) => {
tracing::warn!(
"[PEER_MESH] Event loop lagging on topic {} (new mesh)",
topic_id.fmt_short()
);
}
Err(e) => {
tracing::error!(
"[PEER_MESH] Error in the event stream of topic {} (new mesh): {}",
topic_id.fmt_short(),
e
);
break;
}
}
}
tracing::debug!(
"[PEER_MESH] Event loop ended for topic {} (new mesh)",
topic_id.fmt_short()
);
});
tracing::info!(
"Re-subscription with an additional event loop performed for topic: {}",
topic.fmt_short()
);
} else {
tracing::warn!(
"Message channel not found for topic {} - discarding the re-subscription",
topic.fmt_short()
);
}
}
return Ok(());
}
}
{
let mut topics = self.subscribed_topics.write().await;
topics.insert(*topic, true);
let mut topic_peers = self.topic_peers.write().await;
topic_peers.entry(*topic).or_insert_with(Vec::new);
}
let (message_tx, _message_rx) = broadcast::channel::<(NodeId, Vec<u8>)>(100);
{
let mut channels = self.topic_message_channels.write().await;
channels.insert(*topic, message_tx.clone());
}
let mut gossip_topic = self
.gossip
.subscribe(*topic, bootstrap_peers.clone())
.await
.map_err(|e| GuardianError::Other(format!("Error subscribing to topic: {}", e)))?;
let topic_id = *topic;
let topic_peers_map = self.topic_peers.clone();
let span = self.span.clone();
let event_loop = tokio::spawn(async move {
let _entered = span.enter();
tracing::info!("Event loop started for topic: {}", topic_id.fmt_short());
while let Some(event_result) = gossip_topic.next().await {
match event_result {
Ok(iroh_gossip::api::Event::Received(msg)) => {
tracing::debug!(
"Message received on topic {}: {} bytes from peer {}",
topic_id.fmt_short(),
msg.content.len(),
msg.delivered_from
);
let _ = message_tx.send((msg.delivered_from, msg.content.to_vec()));
}
Ok(iroh_gossip::api::Event::NeighborUp(peer_id)) => {
tracing::debug!(
"Peer {} connected to topic {}",
peer_id,
topic_id.fmt_short()
);
let mut peers = topic_peers_map.write().await;
peers.entry(topic_id).or_insert_with(Vec::new).push(peer_id);
}
Ok(iroh_gossip::api::Event::NeighborDown(peer_id)) => {
tracing::debug!(
"Peer {} disconnected from topic {}",
peer_id,
topic_id.fmt_short()
);
let mut peers = topic_peers_map.write().await;
if let Some(peer_list) = peers.get_mut(&topic_id) {
peer_list.retain(|p| *p != peer_id);
}
}
Ok(iroh_gossip::api::Event::Lagged) => {
tracing::warn!("Event loop lagging on topic {}", topic_id.fmt_short());
}
Err(e) => {
tracing::error!(
"Error in the event stream of topic {}: {}",
topic_id.fmt_short(),
e
);
break;
}
}
}
tracing::info!("Event loop ended for topic: {}", topic_id.fmt_short());
});
{
let mut loops = self.topic_event_loops.write().await;
loops.insert(*topic, event_loop);
}
tracing::info!(
"Successfully subscribed to topic via iroh-gossip: {} with {} peers",
topic.fmt_short(),
bootstrap_peers.len()
);
Ok(())
}
async fn get_connected_peers(&self) -> Vec<NodeId> {
let peers = self.connected_peers.read().await;
let peer_list = peers.clone();
tracing::debug!("Returning {} connected peers", peer_list.len());
peer_list
}
async fn get_topic_peers(&self, topic: &TopicId) -> Vec<NodeId> {
tracing::debug!("Getting peers of topic: {}", topic.fmt_short());
let topic_peers = self.topic_peers.read().await;
let peers = topic_peers.get(topic).cloned().unwrap_or_default();
tracing::debug!(
"Topic {} has {} connected peers",
topic.fmt_short(),
peers.len()
);
peers
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
#[derive(Debug, Clone)]
struct ChannelState {
#[allow(dead_code)]
node_id: NodeId,
topic: TopicId,
connection_status: ConnectionStatus,
last_activity: Instant,
message_count: u64,
last_heartbeat: Instant,
}
#[derive(Debug, Clone)]
enum ConnectionStatus {
Disconnected,
Connecting,
Connected,
#[allow(dead_code)]
Error(String),
}
#[derive(Debug)]
enum DirectChannelEvent {
PeerConnected(NodeId),
PeerDisconnected(NodeId),
MessageReceived {
peer: NodeId,
payload: Vec<u8>,
},
MessageSent {
peer: NodeId,
success: bool,
error: Option<String>,
},
HeartbeatReceived(NodeId),
HeartbeatTimeout(NodeId),
}
pub struct DirectChannel {
span: Span,
iroh_network: Arc<dyn DirectChannelNetwork>,
emitter: Arc<dyn DirectChannelEmitter<Error = GuardianError>>,
channels: Arc<RwLock<HashMap<NodeId, ChannelState>>>,
event_sender: mpsc::UnboundedSender<DirectChannelEvent>,
_event_receiver: Arc<Mutex<Option<mpsc::UnboundedReceiver<DirectChannelEvent>>>>,
own_node_id: NodeId,
running: Arc<Mutex<bool>>,
}
impl DirectChannel {
pub fn new(
span: Span,
iroh_network: Arc<dyn DirectChannelNetwork>,
emitter: Arc<dyn DirectChannelEmitter<Error = GuardianError>>,
own_node_id: NodeId,
) -> Self {
let (event_sender, event_receiver) = mpsc::unbounded_channel();
Self {
span,
iroh_network,
emitter,
channels: Arc::new(RwLock::new(HashMap::new())),
event_sender,
_event_receiver: Arc::new(Mutex::new(Some(event_receiver))),
own_node_id,
running: Arc::new(Mutex::new(false)),
}
}
fn get_channel_topic(&self, peer: NodeId) -> TopicId {
let (first, second) = if self.own_node_id.as_bytes() < peer.as_bytes() {
(self.own_node_id, peer)
} else {
(peer, self.own_node_id)
};
let topic_string = format!("{}/channel/{}/{}", PROTOCOL, first, second);
IrohBridge::topic_id_from_str(&topic_string)
}
pub async fn start(&self) -> Result<()> {
let mut running = self.running.lock().await;
if *running {
return Ok(());
}
*running = true;
let mut receiver = self
._event_receiver
.lock()
.await
.take()
.ok_or_else(|| GuardianError::Other("Event receiver already taken".to_string()))?;
let emitter = self.emitter.clone();
let span = self.span.clone();
let channels = self.channels.clone();
let running_flag = self.running.clone();
tokio::spawn(async move {
while let Some(event) = receiver.recv().await {
let running = *running_flag.lock().await;
if !running {
break;
}
if let Err(e) = Self::handle_event(event, &emitter, &span, &channels).await {
tracing::error!("Error processing event: {}", e);
}
}
tracing::info!("Event processing loop terminated");
});
self.start_heartbeat_loop().await;
Ok(())
}
async fn start_heartbeat_loop(&self) {
let channels = self.channels.clone();
let event_sender = self.event_sender.clone();
let span = self.span.clone();
let running_flag = self.running.clone();
let iroh_network = self.iroh_network.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(HEARTBEAT_INTERVAL);
loop {
interval.tick().await;
let running = *running_flag.lock().await;
if !running {
break;
}
let peers_to_heartbeat: Vec<(NodeId, TopicId)> = {
let channels_map = channels.read().await;
channels_map
.iter()
.filter_map(|(node_id, state)| {
match state.connection_status {
ConnectionStatus::Connected => {
if state.last_heartbeat.elapsed() > HEARTBEAT_INTERVAL {
Some((*node_id, state.topic))
} else {
None
}
}
_ => None,
}
})
.collect()
};
for (peer, topic) in peers_to_heartbeat {
if let Err(e) = Self::send_heartbeat(&iroh_network, &topic, &span).await {
tracing::warn!("Failed to send heartbeat to {}: {}", peer, e);
let _ = event_sender.send(DirectChannelEvent::HeartbeatTimeout(peer));
} else {
tracing::trace!(peer = %peer, "Heartbeat sent to peer");
}
}
let peers_to_reconnect: Vec<NodeId> = {
let channels_map = channels.read().await;
channels_map
.iter()
.filter_map(|(node_id, state)| {
match &state.connection_status {
ConnectionStatus::Error(err) => {
if state.last_activity.elapsed() > Duration::from_secs(30) {
tracing::debug!(
"Attempting to reconnect with peer {} after error: {}",
node_id,
err
);
Some(*node_id)
} else {
None
}
}
ConnectionStatus::Disconnected => {
if state.last_activity.elapsed() > Duration::from_secs(60) {
tracing::debug!(
"Attempting to reconnect with disconnected peer: {}",
node_id
);
Some(*node_id)
} else {
None
}
}
_ => None,
}
})
.collect()
};
for peer in peers_to_reconnect {
let mut channels_map = channels.write().await;
if let Some(state) = channels_map.get_mut(&peer) {
state.connection_status = ConnectionStatus::Connecting;
state.last_activity = Instant::now();
if let Err(e) =
Self::send_heartbeat(&iroh_network, &state.topic, &span).await
{
tracing::warn!("Reconnection attempt failed with {}: {}", peer, e);
} else {
tracing::info!("Reconnection attempt started for peer: {}", peer);
}
}
}
}
});
}
async fn send_heartbeat(
iroh_network: &Arc<dyn DirectChannelNetwork>,
topic: &TopicId,
_span: &Span,
) -> Result<()> {
let heartbeat_msg = DirectChannelMessage {
message_type: MessageType::Heartbeat,
payload: vec![],
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
sender: "heartbeat".to_string(),
};
let serialized = serde_cbor::to_vec(&heartbeat_msg)
.map_err(|e| GuardianError::Other(format!("Heartbeat serialization error: {}", e)))?;
iroh_network.publish_message(topic, &serialized).await?;
tracing::trace!(topic = %topic.fmt_short(), "Heartbeat sent on topic");
Ok(())
}
async fn handle_event(
event: DirectChannelEvent,
emitter: &Arc<dyn DirectChannelEmitter<Error = GuardianError>>,
_span: &Span,
channels: &Arc<RwLock<HashMap<NodeId, ChannelState>>>,
) -> Result<()> {
match event {
DirectChannelEvent::MessageReceived { peer, payload } => {
tracing::debug!("Message received from {}: {} bytes", peer, payload.len());
if payload.len() > MAX_MESSAGE_SIZE {
tracing::warn!("Message too large from {}: {} bytes", peer, payload.len());
return Ok(());
}
{
let mut channels_map = channels.write().await;
if let Some(state) = channels_map.get_mut(&peer) {
state.last_activity = Instant::now();
state.message_count += 1;
}
}
let event_payload = EventPubSubPayload { payload, peer };
emitter
.emit(event_payload)
.await
.map_err(|e| GuardianError::Other(format!("Error emitting event: {}", e)))?;
}
DirectChannelEvent::PeerConnected(peer) => {
tracing::info!("Peer connected: {}", peer);
let mut channels_map = channels.write().await;
if let Some(state) = channels_map.get_mut(&peer) {
state.connection_status = ConnectionStatus::Connected;
state.last_activity = Instant::now();
state.last_heartbeat = Instant::now();
}
}
DirectChannelEvent::PeerDisconnected(peer) => {
tracing::info!("Peer disconnected: {}", peer);
let mut channels_map = channels.write().await;
if let Some(state) = channels_map.get_mut(&peer) {
state.connection_status = ConnectionStatus::Disconnected;
}
}
DirectChannelEvent::MessageSent {
peer,
success,
error,
} => {
if success {
tracing::debug!("Message sent successfully to: {}", peer);
} else {
tracing::warn!("Failed to send message to {}: {:?}", peer, error);
}
}
DirectChannelEvent::HeartbeatReceived(peer) => {
tracing::trace!(peer = %peer, "Heartbeat received from");
let mut channels_map = channels.write().await;
if let Some(state) = channels_map.get_mut(&peer) {
state.last_activity = Instant::now();
state.last_heartbeat = Instant::now();
}
}
DirectChannelEvent::HeartbeatTimeout(peer) => {
tracing::warn!("Heartbeat timeout for peer: {}", peer);
let mut channels_map = channels.write().await;
if let Some(state) = channels_map.get_mut(&peer) {
state.connection_status =
ConnectionStatus::Error("Heartbeat timeout".to_string());
}
}
}
Ok(())
}
pub async fn send_data(&self, peer: NodeId, payload: Vec<u8>) -> Result<()> {
if payload.len() > MAX_MESSAGE_SIZE {
return Err(GuardianError::Other(format!(
"Message too large: {} bytes (maximum: {})",
payload.len(),
MAX_MESSAGE_SIZE
)));
}
let topic = self.get_channel_topic(peer);
let message = DirectChannelMessage {
message_type: MessageType::Data,
payload,
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
sender: self.own_node_id.to_string(),
};
let serialized = serde_cbor::to_vec(&message)
.map_err(|e| GuardianError::Other(format!("Serialization error: {}", e)))?;
match self.iroh_network.publish_message(&topic, &serialized).await {
Ok(()) => {
let _ = self.event_sender.send(DirectChannelEvent::MessageSent {
peer,
success: true,
error: None,
});
tracing::debug!("Data sent to {}: {} bytes", peer, message.payload.len());
Ok(())
}
Err(e) => {
let error_msg = format!("Error publishing message: {}", e);
let _ = self.event_sender.send(DirectChannelEvent::MessageSent {
peer,
success: false,
error: Some(error_msg.clone()),
});
Err(GuardianError::Other(error_msg))
}
}
}
pub async fn connect_to_peer(&self, peer: NodeId) -> Result<()> {
let topic = self.get_channel_topic(peer);
let mut channels_map = self.channels.write().await;
if let Some(state) = channels_map.get(&peer) {
match state.connection_status {
ConnectionStatus::Connected => {
tracing::debug!("Already connected to peer: {}", peer);
return Ok(());
}
ConnectionStatus::Connecting => {
tracing::debug!("Connection in progress with peer: {}", peer);
return Ok(());
}
_ => {}
}
}
channels_map.insert(
peer,
ChannelState {
node_id: peer,
topic,
connection_status: ConnectionStatus::Connecting,
last_activity: Instant::now(),
message_count: 0,
last_heartbeat: Instant::now(),
},
);
drop(channels_map);
self.iroh_network
.subscribe_topic(&topic, vec![peer])
.await?;
self.start_message_consumer_for_topic(topic).await?;
tracing::info!(
"Connecting to peer {} on topic: {}",
peer,
topic.fmt_short()
);
self.establish_peer_connection(peer, topic).await?;
Ok(())
}
async fn start_message_consumer_for_topic(&self, topic: TopicId) -> Result<()> {
let iroh_bridge = self
.iroh_network
.as_any()
.downcast_ref::<IrohBridge>()
.ok_or_else(|| GuardianError::Other("Cannot downcast to IrohBridge".to_string()))?;
let Some(mut receiver): Option<broadcast::Receiver<(NodeId, Vec<u8>)>> =
iroh_bridge.get_topic_receiver(&topic).await
else {
return Err(GuardianError::Other(format!(
"Could not get a receiver for topic: {}",
topic.fmt_short()
)));
};
let event_sender = self.event_sender.clone();
let span = self.span.clone();
tokio::spawn(async move {
let _entered = span.enter();
tracing::debug!("Consumer loop started for topic: {}", topic.fmt_short());
loop {
match receiver.recv().await {
Ok((peer, data)) => {
tracing::debug!(
"Message received from peer {} on topic {}: {} bytes",
peer,
topic.fmt_short(),
data.len()
);
match serde_cbor::from_slice::<DirectChannelMessage>(&data) {
Ok(decoded_msg) => {
match decoded_msg.message_type {
MessageType::Data => {
let _ = event_sender.send(
DirectChannelEvent::MessageReceived {
peer,
payload: decoded_msg.payload,
},
);
}
MessageType::Heartbeat => {
let _ = event_sender
.send(DirectChannelEvent::HeartbeatReceived(peer));
}
MessageType::Ack => {
tracing::trace!("ACK received from: {}", peer);
}
}
}
Err(e) => {
tracing::warn!(
"Error decoding message from {} on topic {}: {}",
peer,
topic.fmt_short(),
e
);
}
}
}
Err(broadcast::error::RecvError::Lagged(n)) => {
tracing::warn!(
"Consumer loop lagging on topic {}: {} messages lost",
topic.fmt_short(),
n
);
}
Err(broadcast::error::RecvError::Closed) => {
tracing::info!("Channel closed for topic: {}", topic.fmt_short());
break;
}
}
}
tracing::debug!("Consumer loop ended for topic: {}", topic.fmt_short());
});
Ok(())
}
async fn establish_peer_connection(&self, peer: NodeId, topic: TopicId) -> Result<()> {
tracing::debug!("Establishing connection with peer: {}", peer);
let connected_peers = self.iroh_network.get_connected_peers().await;
let is_peer_connected = connected_peers.contains(&peer);
if is_peer_connected {
tracing::debug!("Peer {} is already connected globally", peer);
let _ = self
.event_sender
.send(DirectChannelEvent::PeerConnected(peer));
return Ok(());
}
let discovery_timeout = CONNECTION_TIMEOUT;
let start_time = Instant::now();
while start_time.elapsed() < discovery_timeout {
let topic_peers = self.iroh_network.get_topic_peers(&topic).await;
if topic_peers.contains(&peer) {
tracing::info!("Peer {} discovered on topic: {}", peer, topic.fmt_short());
if self.send_handshake_message(&topic, peer).await.is_ok() {
tracing::info!("Handshake successful with peer: {}", peer);
let _ = self
.event_sender
.send(DirectChannelEvent::PeerConnected(peer));
return Ok(());
}
}
let updated_peers = self.iroh_network.get_connected_peers().await;
if updated_peers.contains(&peer) {
tracing::info!("Peer {} connected via global discovery", peer);
let _ = self
.event_sender
.send(DirectChannelEvent::PeerConnected(peer));
return Ok(());
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
tracing::warn!(
"Peer {} not found directly, sending a discovery beacon",
peer
);
if let Err(e) = self.send_discovery_beacon(&topic, peer).await {
tracing::error!("Failed to send discovery beacon to {}: {}", peer, e);
let mut channels_map = self.channels.write().await;
if let Some(state) = channels_map.get_mut(&peer) {
state.connection_status =
ConnectionStatus::Error(format!("Discovery timeout: {}", e));
}
return Err(GuardianError::Other(format!(
"Timeout discovering peer {} after {}s",
peer,
discovery_timeout.as_secs()
)));
}
let beacon_timeout = BEACON_TIMEOUT;
let beacon_start = Instant::now();
while beacon_start.elapsed() < beacon_timeout {
let topic_peers = self.iroh_network.get_topic_peers(&topic).await;
if topic_peers.contains(&peer) {
tracing::info!("Peer {} responded to the discovery beacon", peer);
let _ = self
.event_sender
.send(DirectChannelEvent::PeerConnected(peer));
return Ok(());
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
tracing::warn!(
"Connection with peer {} could not be established at the moment",
peer
);
Ok(())
}
async fn send_handshake_message(&self, topic: &TopicId, target_peer: NodeId) -> Result<()> {
let handshake_msg = DirectChannelMessage {
message_type: MessageType::Ack, payload: format!("handshake:{}", self.own_node_id).into_bytes(),
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
sender: self.own_node_id.to_string(),
};
let serialized = serde_cbor::to_vec(&handshake_msg)
.map_err(|e| GuardianError::Other(format!("Handshake serialization error: {}", e)))?;
self.iroh_network
.publish_message(topic, &serialized)
.await?;
tracing::debug!("Handshake sent to peer: {}", target_peer);
Ok(())
}
async fn send_discovery_beacon(&self, topic: &TopicId, target_peer: NodeId) -> Result<()> {
let beacon_msg = DirectChannelMessage {
message_type: MessageType::Heartbeat, payload: format!("discovery_beacon:{}:{}", self.own_node_id, target_peer).into_bytes(),
timestamp: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
sender: self.own_node_id.to_string(),
};
let serialized = serde_cbor::to_vec(&beacon_msg)
.map_err(|e| GuardianError::Other(format!("Beacon serialization error: {}", e)))?;
self.iroh_network
.publish_message(topic, &serialized)
.await?;
tracing::debug!("Discovery beacon sent on topic: {}", topic.fmt_short());
Ok(())
}
pub async fn handle_iroh_message(
&self,
message_data: &[u8],
sender_peer: NodeId,
) -> Result<()> {
let decoded_msg: DirectChannelMessage = serde_cbor::from_slice(message_data)
.map_err(|e| GuardianError::Other(format!("Error decoding message: {}", e)))?;
match decoded_msg.message_type {
MessageType::Data => {
let _ = self.event_sender.send(DirectChannelEvent::MessageReceived {
peer: sender_peer,
payload: decoded_msg.payload,
});
}
MessageType::Heartbeat => {
if let Ok(payload_str) = String::from_utf8(decoded_msg.payload.clone()) {
if payload_str.starts_with("discovery_beacon:") {
self.handle_discovery_beacon(sender_peer, payload_str)
.await?;
} else {
let _ = self
.event_sender
.send(DirectChannelEvent::HeartbeatReceived(sender_peer));
}
} else {
let _ = self
.event_sender
.send(DirectChannelEvent::HeartbeatReceived(sender_peer));
}
}
MessageType::Ack => {
if let Ok(payload_str) = String::from_utf8(decoded_msg.payload.clone()) {
if payload_str.starts_with("handshake:") {
self.handle_handshake_response(sender_peer, payload_str)
.await?;
} else {
tracing::trace!(sender_peer = %sender_peer, "ACK received from");
}
} else {
tracing::trace!(sender_peer = %sender_peer, "ACK received from");
}
}
}
Ok(())
}
async fn handle_discovery_beacon(
&self,
sender_peer: NodeId,
beacon_payload: String,
) -> Result<()> {
tracing::debug!(
"Discovery beacon received from: {} - {}",
sender_peer,
beacon_payload
);
let parts: Vec<&str> = beacon_payload.split(':').collect();
if parts.len() >= 3 {
let _beacon_sender = parts[1]; let beacon_target = parts[2];
if beacon_target == self.own_node_id.to_string() {
tracing::info!("Discovery beacon directed at us from: {}", sender_peer);
let channels_map = self.channels.read().await;
if let Some(state) = channels_map.get(&sender_peer)
&& matches!(
state.connection_status,
ConnectionStatus::Connecting | ConnectionStatus::Disconnected
)
{
drop(channels_map);
let topic = self.get_channel_topic(sender_peer);
if let Err(e) = self.send_handshake_message(&topic, sender_peer).await {
tracing::warn!("Failed to respond to beacon from {}: {}", sender_peer, e);
} else {
tracing::info!("Response handshake sent to: {}", sender_peer);
}
}
}
}
Ok(())
}
async fn handle_handshake_response(
&self,
sender_peer: NodeId,
handshake_payload: String,
) -> Result<()> {
tracing::debug!(
"Handshake received from: {} - {}",
sender_peer,
handshake_payload
);
let parts: Vec<&str> = handshake_payload.split(':').collect();
if parts.len() >= 2 {
let handshake_peer = parts[1];
tracing::info!(
"Valid handshake received from peer: {} (id: {})",
sender_peer,
handshake_peer
);
let mut channels_map = self.channels.write().await;
if let Some(state) = channels_map.get_mut(&sender_peer) {
match state.connection_status {
ConnectionStatus::Connecting => {
state.connection_status = ConnectionStatus::Connected;
state.last_activity = Instant::now();
state.last_heartbeat = Instant::now();
let _ = self
.event_sender
.send(DirectChannelEvent::PeerConnected(sender_peer));
tracing::info!("Connection established with peer: {}", sender_peer);
}
ConnectionStatus::Connected => {
state.last_activity = Instant::now();
state.last_heartbeat = Instant::now();
tracing::trace!("Maintenance handshake received from: {}", sender_peer);
}
_ => {
tracing::debug!(
"Handshake received from peer in state: {:?}",
state.connection_status
);
}
}
}
}
Ok(())
}
pub async fn stop(&self) -> Result<()> {
let mut running = self.running.lock().await;
*running = false;
let peers: Vec<NodeId> = {
let channels_map = self.channels.read().await;
channels_map.keys().cloned().collect()
};
for peer in peers {
let mut channels_map = self.channels.write().await;
if let Some(state) = channels_map.remove(&peer) {
tracing::info!(
"Peer removed: {} (topic: {})",
peer,
state.topic.fmt_short()
);
let _ = self
.event_sender
.send(DirectChannelEvent::PeerDisconnected(peer));
}
}
tracing::info!("DirectChannel stopped");
Ok(())
}
pub async fn list_connected_peers(&self) -> Vec<NodeId> {
let channels_map = self.channels.read().await;
channels_map
.iter()
.filter_map(|(node_id, state)| match state.connection_status {
ConnectionStatus::Connected => Some(*node_id),
_ => None,
})
.collect()
}
pub async fn get_channel_stats(&self) -> HashMap<NodeId, (u64, Duration)> {
let channels_map = self.channels.read().await;
channels_map
.iter()
.map(|(node_id, state)| {
(
*node_id,
(state.message_count, state.last_activity.elapsed()),
)
})
.collect()
}
async fn close_internal(&self) -> Result<()> {
tracing::info!("Closing DirectChannel...");
self.stop().await?;
if let Err(e) = self.emitter.close().await {
tracing::warn!("Error closing emitter: {}", e);
}
tracing::info!("DirectChannel closed successfully");
Ok(())
}
}
#[async_trait]
impl crate::traits::DirectChannel for DirectChannel {
type Error = GuardianError;
async fn connect(&mut self, peer: NodeId) -> std::result::Result<(), Self::Error> {
tracing::info!("Connecting to peer: {}", peer);
self.connect_to_peer(peer).await
}
async fn send(&mut self, peer: NodeId, data: Vec<u8>) -> std::result::Result<(), Self::Error> {
tracing::debug!("Sending {} bytes to {}", data.len(), peer);
self.send_data(peer, data).await
}
async fn close(&mut self) -> std::result::Result<(), Self::Error> {
self.close_internal().await
}
async fn close_shared(&self) -> std::result::Result<(), Self::Error> {
self.close_internal().await
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
pub struct HolderChannels {
iroh_network: Arc<dyn DirectChannelNetwork>,
span: Span,
own_node_id: NodeId,
}
impl HolderChannels {
pub fn new(
span: Span,
iroh_network: Arc<dyn DirectChannelNetwork>,
own_node_id: NodeId,
) -> Self {
Self {
iroh_network,
span,
own_node_id,
}
}
pub async fn new_channel(
&self,
emitter: Box<dyn DirectChannelEmitter<Error = GuardianError>>,
opts: Option<DirectChannelOptions>,
) -> Result<Box<dyn crate::traits::DirectChannel<Error = GuardianError>>> {
let resolved_opts = opts.unwrap_or_default();
let span = resolved_opts.span.unwrap_or_else(|| self.span.clone());
let dc = DirectChannel::new(
span.clone(),
self.iroh_network.clone(),
Arc::from(emitter),
self.own_node_id,
);
dc.start().await?;
tracing::info!(protocol = PROTOCOL, "DirectChannel created with protocol");
Ok(Box::new(dc))
}
}
pub fn init_direct_channel_factory(
span: Span,
own_node_id: NodeId,
backend: Arc<IrohBackend>,
) -> DirectChannelFactory {
Arc::new(
move |emitter: Arc<dyn DirectChannelEmitter<Error = GuardianError>>,
opts: Option<DirectChannelOptions>| {
let span = span.clone();
let own_node_id = own_node_id;
let backend = backend.clone();
Box::pin(async move {
tracing::info!(
"Initializing DirectChannel factory for node: {}",
own_node_id
);
let iroh_interface = Arc::new(
create_unified_iroh_interface(span.clone(), backend.clone())
.await
.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)?,
);
let holder = HolderChannels::new(span.clone(), iroh_interface, own_node_id);
let emitter_box = Box::new(EmitterWrapper(emitter));
let channel = holder
.new_channel(emitter_box, opts)
.await
.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)?;
Ok(Arc::from(channel)
as Arc<
dyn crate::traits::DirectChannel<Error = GuardianError>,
>)
})
},
)
}
struct EmitterWrapper(Arc<dyn DirectChannelEmitter<Error = GuardianError>>);
#[async_trait]
impl DirectChannelEmitter for EmitterWrapper {
type Error = GuardianError;
async fn emit(&self, payload: EventPubSubPayload) -> std::result::Result<(), Self::Error> {
self.0.emit(payload).await
}
async fn close(&self) -> std::result::Result<(), Self::Error> {
self.0.close().await
}
}
pub async fn create_direct_channel_with_iroh(
iroh_network: Arc<dyn DirectChannelNetwork>,
emitter: Arc<dyn DirectChannelEmitter<Error = GuardianError>>,
span: Span,
own_node_id: NodeId,
) -> Result<DirectChannel> {
let channel = DirectChannel::new(span.clone(), iroh_network, emitter, own_node_id);
channel.start().await?;
tracing::info!("DirectChannel created with an integrated Iroh interface");
Ok(channel)
}
pub async fn create_unified_iroh_interface(
span: Span,
backend: Arc<IrohBackend>,
) -> Result<IrohBridge> {
let interface = IrohBridge::new(span.clone(), backend).await?;
interface.start().await?;
tracing::info!("Unified Iroh interface initialized with integrated iroh-gossip");
Ok(interface)
}
pub fn create_test_node_id() -> NodeId {
let secret_key = iroh::SecretKey::generate();
secret_key.public()
}
#[cfg(test)]
mod tests {
use super::*;
fn roundtrip(msg: &DirectChannelMessage) -> DirectChannelMessage {
let bytes = serde_cbor::to_vec(msg).expect("serialize");
serde_cbor::from_slice::<DirectChannelMessage>(&bytes).expect("deserialize")
}
#[test]
fn direct_channel_message_roundtrip_data() {
let msg = DirectChannelMessage {
message_type: MessageType::Data,
payload: b"hello world".to_vec(),
timestamp: 1_725_000_000,
sender: "node-abc".to_string(),
};
let back = roundtrip(&msg);
assert!(matches!(back.message_type, MessageType::Data));
assert_eq!(back.payload, msg.payload);
assert_eq!(back.timestamp, msg.timestamp);
assert_eq!(back.sender, msg.sender);
}
#[test]
fn direct_channel_message_roundtrip_all_types() {
for mt in [MessageType::Data, MessageType::Heartbeat, MessageType::Ack] {
let msg = DirectChannelMessage {
message_type: mt.clone(),
payload: vec![1, 2, 3, 4],
timestamp: 42,
sender: "peer".to_string(),
};
let back = roundtrip(&msg);
assert_eq!(
std::mem::discriminant(&back.message_type),
std::mem::discriminant(&mt)
);
assert_eq!(back.payload, vec![1, 2, 3, 4]);
}
}
#[test]
fn direct_channel_message_empty_payload_roundtrip() {
let msg = DirectChannelMessage {
message_type: MessageType::Heartbeat,
payload: vec![],
timestamp: 0,
sender: String::new(),
};
let back = roundtrip(&msg);
assert!(back.payload.is_empty());
assert!(matches!(back.message_type, MessageType::Heartbeat));
}
#[test]
fn corrupt_bytes_fail_to_deserialize() {
let garbage = [0xff, 0x00, 0x13, 0x37, 0x42];
assert!(serde_cbor::from_slice::<DirectChannelMessage>(&garbage).is_err());
}
#[test]
fn topic_id_is_deterministic_for_same_name() {
let a = IrohBridge::topic_id_from_str("shared-kv");
let b = IrohBridge::topic_id_from_str("shared-kv");
assert_eq!(a, b);
}
#[test]
fn topic_id_differs_for_different_names() {
let a = IrohBridge::topic_id_from_str("topic-a");
let b = IrohBridge::topic_id_from_str("topic-b");
assert_ne!(a, b);
}
}