use std::collections::HashSet;
use std::sync::Arc;
use futures::future::BoxFuture;
use futures::stream::{FuturesUnordered, StreamExt as _};
use iroh_base::EndpointAddr;
use rostra_client_db::Database;
use rostra_core::Timestamp;
use rostra_core::event::IrohNodeId;
use rostra_core::id::{RostraId, ToShort as _};
use rostra_p2p::ConnectionSnafu;
use rostra_p2p::connection::Connection;
use rostra_p2p_api::ROSTRA_P2P_V0_ALPN;
use rostra_util_error::FmtCompact as _;
use rostra_util_fmt::AsFmtOption as _;
use snafu::{OptionExt as _, ResultExt as _};
use tracing::{debug, trace};
use super::{RRECORD_HEAD_KEY, RRECORD_P2P_KEY, get_rrecord_typed};
use crate::client::{NodeSource, P2PState};
use crate::connection_cache::ConnectionCache;
use crate::error::{
ConnectError, ConnectIrohSnafu, ConnectResult, IdResolveResult, InvalidIdSnafu,
MissingTicketSnafu, NodeInBackoffSnafu, PkarrResolveSnafu, RRecordSnafu, ResolveSnafu,
};
use crate::id::{CompactTicket, IdPublishedData, IdResolvedData};
const LOG_TARGET: &str = "rostra::client-net";
pub trait IdEndpointLookup: Send + Sync {
fn get_node_ids(&self, id: RostraId) -> BoxFuture<'_, HashSet<IrohNodeId>>;
}
impl IdEndpointLookup for Database {
fn get_node_ids(&self, id: RostraId) -> BoxFuture<'_, HashSet<IrohNodeId>> {
Box::pin(async move {
self.get_id_endpoints(id)
.await
.into_keys()
.map(|(_ts, node_id)| node_id)
.collect()
})
}
}
#[derive(Debug)]
pub enum EndpointConnectResult {
Success(Connection),
Failed(ConnectError),
InBackoff,
Skipped,
}
pub struct ClientNetworking {
pub(crate) endpoint: iroh::Endpoint,
pub(crate) pkarr_client: Arc<pkarr::Client>,
pub(crate) p2p_state: P2PState,
pub(crate) connection_cache: ConnectionCache,
id_endpoint_lookup: Arc<dyn IdEndpointLookup>,
}
impl ClientNetworking {
pub fn new(
endpoint: iroh::Endpoint,
pkarr_client: Arc<pkarr::Client>,
id_endpoint_lookup: Arc<dyn IdEndpointLookup>,
) -> Self {
Self {
endpoint,
pkarr_client,
p2p_state: P2PState::new(),
connection_cache: ConnectionCache::new(),
id_endpoint_lookup,
}
}
pub fn p2p_state(&self) -> &P2PState {
&self.p2p_state
}
pub fn connection_cache(&self) -> &ConnectionCache {
&self.connection_cache
}
pub async fn connect_cached(&self, id: RostraId) -> ConnectResult<Connection> {
self.connection_cache.get_or_connect(self, id).await
}
pub async fn connect_to_endpoint(
&self,
endpoint_addr: EndpointAddr,
source: NodeSource,
rostra_id: Option<RostraId>,
) -> EndpointConnectResult {
let node_id = IrohNodeId::from_bytes(*endpoint_addr.id.as_bytes());
if endpoint_addr.id == self.endpoint.id() {
return EndpointConnectResult::Skipped;
}
if self.p2p_state.is_node_in_backoff(node_id).await {
if let Some(remaining) = self.p2p_state.get_node_backoff_remaining(node_id).await {
trace!(
target: LOG_TARGET,
%node_id,
remaining_secs = remaining.as_secs(),
"Node is in backoff, skipping connection attempt"
);
}
return EndpointConnectResult::InBackoff;
}
let now = Timestamp::now();
self.p2p_state
.update_node(node_id, |state| {
state.last_attempt = Some(now);
state.source = source;
state.rostra_id = rostra_id;
})
.await;
debug!(
target: LOG_TARGET,
iroh_id = %endpoint_addr.id,
rostra_id = %rostra_id.map(|id| id.to_short().to_string()).unwrap_or_default(),
"Connecting to endpoint"
);
let conn_result = self
.endpoint
.connect(endpoint_addr, ROSTRA_P2P_V0_ALPN)
.await;
trace!(
target: LOG_TARGET,
%node_id,
err = %conn_result.as_ref().err().fmt_option(),
"Iroh connect result"
);
match conn_result {
Ok(conn) => {
let conn = Connection::from(conn);
let ping_result = conn.ping(0).await;
trace!(
target: LOG_TARGET,
%node_id,
err = %ping_result.as_ref().err().fmt_option(),
"Ping result"
);
match ping_result {
Ok(_) => {
let now = Timestamp::now();
self.p2p_state
.update_node(node_id, |state| state.record_success(now))
.await;
if let Some(id) = rostra_id {
self.p2p_state
.update(id, |state| state.last_success = Some(now))
.await;
}
debug!(
target: LOG_TARGET,
%node_id,
rostra_id = %rostra_id.fmt_option(),
"Connected to endpoint successfully"
);
EndpointConnectResult::Success(conn)
}
Err(err) => {
let now = Timestamp::now();
self.p2p_state
.update_node(node_id, |state| state.record_failure(now))
.await;
debug!(
target: LOG_TARGET,
%node_id,
err = %err.fmt_compact(),
"Ping failed after connection"
);
EndpointConnectResult::Failed(ConnectError::PingFailed { source: err })
}
}
}
Err(err) => {
let now = Timestamp::now();
self.p2p_state
.update_node(node_id, |state| state.record_failure(now))
.await;
if let Some(id) = rostra_id {
self.p2p_state
.update(id, |state| state.last_failure = Some(now))
.await;
}
debug!(
target: LOG_TARGET,
%node_id,
err = %err.fmt_compact(),
"Failed to connect to endpoint"
);
EndpointConnectResult::Failed(ConnectError::ConnectIroh { source: err })
}
}
}
pub async fn connect_uncached(&self, id: RostraId) -> ConnectResult<Connection> {
let now = Timestamp::now();
self.p2p_state
.update(id, |state| state.last_attempt = Some(now))
.await;
let node_ids = self.id_endpoint_lookup.get_node_ids(id).await;
trace!(
target: LOG_TARGET,
%id,
num_endpoints = node_ids.len(),
"Connecting to peer, trying known endpoints"
);
let mut connection_futures = FuturesUnordered::new();
for node_id in node_ids.iter().copied() {
let Ok(pub_key) = iroh::PublicKey::from_bytes(&node_id.to_bytes()) else {
debug!(target: LOG_TARGET, %id, "Invalid iroh id for rostra id found");
continue;
};
if self.p2p_state.is_node_in_backoff(node_id).await {
trace!(
target: LOG_TARGET,
%node_id,
%id,
"Node is in backoff, skipping"
);
continue;
}
let now = Timestamp::now();
self.p2p_state
.update_node(node_id, |state| {
state.last_attempt = Some(now);
state.source = NodeSource::NodeAnnouncement;
state.rostra_id = Some(id);
})
.await;
let endpoint = self.endpoint.clone();
let our_id = self.endpoint.id();
connection_futures.push(async move {
if pub_key == our_id {
return (
node_id,
None,
Err::<Connection, _>("skipped: own endpoint".to_string()),
);
}
let result = async {
let conn_result = endpoint.connect(pub_key, ROSTRA_P2P_V0_ALPN).await;
trace!(target: LOG_TARGET, %node_id, err = %conn_result.as_ref().err().fmt_option(), "Iroh connect result");
let conn = Connection::from(conn_result.context(ConnectionSnafu)?);
let ping_result = conn.ping(0).await;
trace!(target: LOG_TARGET, %node_id, err = %ping_result.as_ref().err().fmt_option(), "Ping result");
ping_result?;
Ok::<_, rostra_p2p::RpcError>(conn)
}
.await;
let err_str = result.as_ref().err().map(|e| e.fmt_compact().to_string());
(node_id, err_str, result.map_err(|e| e.to_string()))
});
}
while let Some((node_id, err_detail, result)) = connection_futures.next().await {
match result {
Ok(conn) => {
let now = Timestamp::now();
self.p2p_state
.update(id, |state| state.last_success = Some(now))
.await;
self.p2p_state
.update_node(node_id, |state| state.record_success(now))
.await;
debug!(
target: LOG_TARGET,
%id,
%node_id,
"Successfully connected to peer via known endpoint"
);
return Ok(conn);
}
Err(_) => {
let now = Timestamp::now();
self.p2p_state
.update_node(node_id, |state| state.record_failure(now))
.await;
trace!(
target: LOG_TARGET,
%id,
%node_id,
err = %err_detail.as_deref().unwrap_or("unknown"),
"Failed to connect to known endpoint"
);
}
}
}
if !node_ids.is_empty() {
trace!(
target: LOG_TARGET,
%id,
"All known endpoints failed, trying pkarr resolution"
);
}
self.connect_by_pkarr_resolution(id).await
}
pub async fn connect_by_pkarr_resolution(&self, id: RostraId) -> ConnectResult<Connection> {
let ticket = self.resolve_id_ticket(id).await.context(ResolveSnafu)?;
let endpoint_addr = EndpointAddr::from(ticket);
let node_id = IrohNodeId::from_bytes(*endpoint_addr.id.as_bytes());
match self
.connect_to_endpoint(endpoint_addr, NodeSource::Pkarr, Some(id))
.await
{
EndpointConnectResult::Success(conn) => Ok(conn),
EndpointConnectResult::Failed(err) => Err(err),
EndpointConnectResult::InBackoff => {
if let Some(remaining) = self.p2p_state.get_node_backoff_remaining(node_id).await {
trace!(
target: LOG_TARGET,
%id,
%node_id,
remaining_secs = remaining.as_secs(),
"Cannot connect to peer - node is in backoff"
);
}
Err(NodeInBackoffSnafu.build())
}
EndpointConnectResult::Skipped => Err(ConnectError::ResolvedToSelf),
}
}
pub async fn connect_ticket(&self, ticket: CompactTicket) -> ConnectResult<Connection> {
Ok(self
.endpoint
.connect(ticket, ROSTRA_P2P_V0_ALPN)
.await
.context(ConnectIrohSnafu)?
.into())
}
pub async fn resolve_id_data(&self, id: RostraId) -> IdResolveResult<IdResolvedData> {
let public_key = pkarr::PublicKey::try_from(id).context(InvalidIdSnafu)?;
let domain = public_key.to_string();
let packet = self
.pkarr_client
.resolve(&public_key)
.await
.context(PkarrResolveSnafu)?;
let timestamp = packet.timestamp();
let ticket = get_rrecord_typed(&packet, &domain, RRECORD_P2P_KEY).context(RRecordSnafu)?;
let head = get_rrecord_typed(&packet, &domain, RRECORD_HEAD_KEY).context(RRecordSnafu)?;
trace!(
target: LOG_TARGET,
%id,
ticket = %ticket.fmt_option(),
head = %head.fmt_option(),
"Resolved Id"
);
Ok(IdResolvedData {
published: IdPublishedData { ticket, head },
timestamp: timestamp.as_u64(),
})
}
pub async fn resolve_id_ticket(&self, id: RostraId) -> IdResolveResult<CompactTicket> {
self.resolve_id_data(id)
.await?
.published
.ticket
.context(MissingTicketSnafu)
}
}