use std::sync::Arc;
use futures::future::BoxFuture;
use ntk_common::Naddr;
use ntk_hooking::{
DeleteReservationRequest, EntryData, ExploreGNodeRequest, ExploreGNodeResponse, HookingStub,
HookingStubFactory, NetworkData, RequestPacket, ResponsePacket, SearchMigrationPathErrorPkt,
SearchMigrationPathRequest, SearchMigrationPathResponse,
};
use ntk_proto::domain::UnicastId;
use ntk_proto::v1::method_call::Call;
use ntk_proto::v1::response_payload::Value as RespValue;
use ntk_proto::v1::{CallerContext, MethodCall, TypedValue};
use ntk_qspn::{ArcId as QspnArcId, EtpMessage, MissingArcHandler, QspnStub, QspnStubFactory};
use ntk_rpc::{RpcClient, RpcError};
use crate::node::peers::PeerLinks;
use crate::node::registry::{LinkId, LinkRegistry};
fn empty_caller() -> CallerContext {
CallerContext {
source_id: None,
src_nic: None,
}
}
fn caller_for_id(id: ntk_neighborhood::NodeId) -> CallerContext {
CallerContext {
source_id: None,
src_nic: Some(crate::node::registry::encode_caller_id(id)),
}
}
fn main_identity_unicast_id() -> TypedValue {
UnicastId::MainIdentity.to_typed_value()
}
fn malformed(msg: impl Into<String>) -> RpcError {
RpcError::Malformed(msg.into())
}
#[derive(Debug)]
pub struct NeighborhoodStubFactoryAdapter {
pub broadcasters: std::collections::HashMap<String, Arc<ntk_rpc::UdpBroadcaster>>,
pub links: Arc<PeerLinks>,
pub registry: Arc<LinkRegistry>,
}
impl ntk_neighborhood::NeighborhoodStubFactory for NeighborhoodStubFactoryAdapter {
fn broadcast(&self, dev: &str) -> Arc<dyn RpcClient> {
let broadcaster = self
.broadcasters
.get(dev)
.unwrap_or_else(|| panic!("no UDP broadcaster bound for nic {dev}"))
.clone();
Arc::new(ntk_neighborhood::BroadcastRpcClient::new(broadcaster))
}
fn unicast(&self, arc: &ntk_neighborhood::Arc) -> Arc<dyn RpcClient> {
let link =
self.registry
.link_for_neighbour(arc.neighbour_id, &arc.neighbour_mac, &arc.my_dev);
Arc::new(LazyLinkClient {
link,
addr: arc.neighbour_nic_addr.clone(),
dev: arc.my_dev.clone(),
links: self.links.clone(),
})
}
}
struct LazyLinkClient {
link: LinkId,
addr: String,
dev: String,
links: Arc<PeerLinks>,
}
const LAZY_LINK_DIAL_RETRY_BUDGET: std::time::Duration = std::time::Duration::from_secs(5);
const LAZY_LINK_DIAL_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100);
const LAZY_LINK_DIAL_ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(300);
impl LazyLinkClient {
async fn resolve(&self) -> Result<Arc<dyn RpcClient>, RpcError> {
if let Some(client) = self.links.get(self.link) {
return Ok(client);
}
let port = self.links.port().ok_or_else(|| {
malformed("PeerLinks::set_port was never called before a neighbour needed dialing")
})?;
let dialer = crate::node::lifecycle::TcpDialer::default();
let deadline = tokio::time::Instant::now() + LAZY_LINK_DIAL_RETRY_BUDGET;
let mut attempt: u32 = 0;
loop {
attempt += 1;
let outcome = tokio::time::timeout(
LAZY_LINK_DIAL_ATTEMPT_TIMEOUT,
crate::node::lifecycle::Dialer::dial_via(
&dialer,
&self.addr,
port,
Some(&self.dev),
),
)
.await;
match &outcome {
Err(_elapsed) => tracing::debug!(
addr = %self.addr, dev = %self.dev, attempt,
"ntkd: dial attempt timed out (no response within the per-attempt budget)"
),
Ok(None) => tracing::debug!(
addr = %self.addr, dev = %self.dev, attempt,
"ntkd: dial attempt returned promptly but failed"
),
Ok(Some(_)) => {}
}
if let Some(client) = outcome.ok().flatten() {
self.links.insert(self.link, client.clone());
return Ok(client);
}
if tokio::time::Instant::now() >= deadline {
return Err(RpcError::ConnectionClosed);
}
tokio::time::sleep(LAZY_LINK_DIAL_RETRY_INTERVAL).await;
}
}
}
impl RpcClient for LazyLinkClient {
fn call<'a>(
&'a self,
caller: CallerContext,
unicast_id: TypedValue,
call: MethodCall,
) -> BoxFuture<'a, Result<ntk_proto::v1::ResponsePayload, RpcError>> {
Box::pin(async move { self.resolve().await?.call(caller, unicast_id, call).await })
}
fn notify<'a>(
&'a self,
caller: CallerContext,
unicast_id: TypedValue,
call: MethodCall,
) -> BoxFuture<'a, Result<(), RpcError>> {
Box::pin(async move { self.resolve().await?.notify(caller, unicast_id, call).await })
}
fn call_authenticated<'a>(
&'a self,
caller: CallerContext,
unicast_id: TypedValue,
call: MethodCall,
auth: Option<ntk_proto::v1::Auth>,
) -> BoxFuture<'a, Result<ntk_proto::v1::ResponsePayload, RpcError>> {
Box::pin(async move {
self.resolve()
.await?
.call_authenticated(caller, unicast_id, call, auth)
.await
})
}
fn notify_authenticated<'a>(
&'a self,
caller: CallerContext,
unicast_id: TypedValue,
call: MethodCall,
auth: Option<ntk_proto::v1::Auth>,
) -> BoxFuture<'a, Result<(), RpcError>> {
Box::pin(async move {
self.resolve()
.await?
.notify_authenticated(caller, unicast_id, call, auth)
.await
})
}
}
struct ClosedLinkClient;
impl RpcClient for ClosedLinkClient {
fn call<'a>(
&'a self,
_caller: CallerContext,
_unicast_id: TypedValue,
_call: MethodCall,
) -> BoxFuture<'a, Result<ntk_proto::v1::ResponsePayload, RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn notify<'a>(
&'a self,
_caller: CallerContext,
_unicast_id: TypedValue,
_call: MethodCall,
) -> BoxFuture<'a, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
}
#[derive(Debug)]
pub struct IdentityStubFactoryAdapter {
pub links: Arc<PeerLinks>,
pub registry: Arc<LinkRegistry>,
}
impl ntk_identities::IdentityStubFactory for IdentityStubFactoryAdapter {
fn stub(&self, arc: ntk_identities::ArcId) -> Arc<dyn RpcClient> {
self.links.get(LinkId(arc.0)).unwrap_or_else(|| {
tracing::debug!(
?arc,
"identities: no outbound connection for arc, calls will fail"
);
Arc::new(ClosedLinkClient)
})
}
fn arc_for_caller(&self, caller: &CallerContext) -> Option<ntk_identities::ArcId> {
self.registry
.link_for_caller(caller.src_nic.as_ref()?)
.map(LinkId::identities)
}
}
struct RpcQspnStub {
client: Arc<dyn RpcClient>,
my_id: ntk_neighborhood::NodeId,
}
impl RpcQspnStub {
fn unicast_id(&self) -> TypedValue {
main_identity_unicast_id()
}
}
impl QspnStub for RpcQspnStub {
fn get_full_etp(
&self,
requesting_address: Naddr,
) -> BoxFuture<'_, Result<EtpMessage, RpcError>> {
Box::pin(async move {
let arg = ntk_qspn::encode_naddr(&requesting_address);
let payload = self
.client
.call(
caller_for_id(self.my_id),
self.unicast_id(),
MethodCall {
call: Some(Call::QspnGetFullEtp(arg)),
},
)
.await?;
match payload.value {
Some(RespValue::Typed(tv)) => {
ntk_qspn::decode_etp_message(&tv).map_err(|e| malformed(e.to_string()))
}
_ => Err(malformed("qspn_get_full_etp: expected a typed ETP reply")),
}
})
}
fn send_etp(&self, etp: EtpMessage, is_full: bool) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
let args = ntk_proto::v1::QspnSendEtpArgs {
etp: Some(ntk_qspn::encode_etp_message(&etp)),
is_full,
};
self.client
.call(
caller_for_id(self.my_id),
self.unicast_id(),
MethodCall {
call: Some(Call::QspnSendEtp(args)),
},
)
.await?;
Ok(())
})
}
fn got_prepare_destroy(&self) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
self.client
.notify(
caller_for_id(self.my_id),
self.unicast_id(),
MethodCall {
call: Some(Call::QspnGotPrepareDestroy(ntk_proto::v1::Empty::VALUE)),
},
)
.await
})
}
fn got_destroy(&self) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
self.client
.notify(
caller_for_id(self.my_id),
self.unicast_id(),
MethodCall {
call: Some(Call::QspnGotDestroy(ntk_proto::v1::Empty::VALUE)),
},
)
.await
})
}
}
struct FanOutQspnStub {
targets: Vec<(QspnArcId, LazyQspnStub)>,
missing: Option<Arc<dyn MissingArcHandler>>,
}
impl QspnStub for FanOutQspnStub {
fn get_full_etp(
&self,
_requesting_address: Naddr,
) -> BoxFuture<'_, Result<EtpMessage, RpcError>> {
Box::pin(async move {
Err(malformed(
"get_full_etp is never sent over a broadcast stub",
))
})
}
fn send_etp(&self, etp: EtpMessage, is_full: bool) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
for (arc, target) in &self.targets {
if target.send_etp(etp.clone(), is_full).await.is_err()
&& let Some(missing) = &self.missing
{
missing.missing(*arc);
}
}
Ok(())
})
}
fn got_prepare_destroy(&self) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
for (_, target) in &self.targets {
let _ = target.got_prepare_destroy().await;
}
Ok(())
})
}
fn got_destroy(&self) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
for (_, target) in &self.targets {
let _ = target.got_destroy().await;
}
Ok(())
})
}
}
const LAZY_STUB_RETRY_BUDGET: std::time::Duration = std::time::Duration::from_millis(200);
const LAZY_STUB_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(2);
#[derive(Debug)]
pub struct QspnStubFactoryAdapter {
pub links: Arc<PeerLinks>,
pub registry: Arc<LinkRegistry>,
pub my_id: ntk_neighborhood::NodeId,
}
impl QspnStubFactoryAdapter {
fn lazy_stub(&self, arc: QspnArcId) -> LazyQspnStub {
LazyQspnStub {
registry: self.registry.clone(),
links: self.links.clone(),
arc,
my_id: self.my_id,
}
}
}
impl QspnStubFactory for QspnStubFactoryAdapter {
fn broadcast(
&self,
arcs: &[QspnArcId],
missing: Option<Arc<dyn MissingArcHandler>>,
) -> Arc<dyn QspnStub> {
let targets = arcs.iter().map(|a| (*a, self.lazy_stub(*a))).collect();
Arc::new(FanOutQspnStub { targets, missing })
}
fn tcp(&self, arc: QspnArcId) -> Arc<dyn QspnStub> {
Arc::new(self.lazy_stub(arc))
}
}
struct LazyQspnStub {
registry: Arc<LinkRegistry>,
links: Arc<PeerLinks>,
arc: QspnArcId,
my_id: ntk_neighborhood::NodeId,
}
impl LazyQspnStub {
async fn resolve(&self) -> Option<RpcQspnStub> {
let deadline = tokio::time::Instant::now() + LAZY_STUB_RETRY_BUDGET;
loop {
if let Some(link) = self.registry.link_of_qspn_arc(self.arc)
&& let Some(client) = self.links.get(link)
{
return Some(RpcQspnStub {
client,
my_id: self.my_id,
});
}
if tokio::time::Instant::now() >= deadline {
return None;
}
tokio::time::sleep(LAZY_STUB_RETRY_INTERVAL).await;
}
}
}
impl QspnStub for LazyQspnStub {
fn get_full_etp(
&self,
requesting_address: Naddr,
) -> BoxFuture<'_, Result<EtpMessage, RpcError>> {
Box::pin(async move {
match self.resolve().await {
Some(stub) => stub.get_full_etp(requesting_address).await,
None => Err(RpcError::ConnectionClosed),
}
})
}
fn send_etp(&self, etp: EtpMessage, is_full: bool) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
match self.resolve().await {
Some(stub) => stub.send_etp(etp, is_full).await,
None => Err(RpcError::ConnectionClosed),
}
})
}
fn got_prepare_destroy(&self) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
match self.resolve().await {
Some(stub) => stub.got_prepare_destroy().await,
None => Err(RpcError::ConnectionClosed),
}
})
}
fn got_destroy(&self) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move {
match self.resolve().await {
Some(stub) => stub.got_destroy().await,
None => Err(RpcError::ConnectionClosed),
}
})
}
}
struct RpcHookingStub {
client: Arc<dyn RpcClient>,
}
impl RpcHookingStub {
fn unicast_id(&self) -> TypedValue {
main_identity_unicast_id()
}
}
macro_rules! notify_call {
($self:ident, $variant:ident, $encoded:expr) => {
Box::pin(async move {
$self
.client
.notify(
empty_caller(),
$self.unicast_id(),
MethodCall {
call: Some(Call::$variant($encoded)),
},
)
.await
})
};
}
impl HookingStub for RpcHookingStub {
fn retrieve_network_data(
&self,
ask_coord: bool,
) -> BoxFuture<'_, Result<NetworkData, RpcError>> {
Box::pin(async move {
let payload = self
.client
.call(
empty_caller(),
self.unicast_id(),
MethodCall {
call: Some(Call::HookingRetrieveNetworkData(ask_coord)),
},
)
.await?;
match payload.value {
Some(RespValue::Typed(tv)) => {
ntk_hooking::decode_network_data(&tv).map_err(|e| malformed(e.to_string()))
}
_ => Err(malformed("retrieve_network_data: expected a typed reply")),
}
})
}
fn search_migration_path(&self, lvl: usize) -> BoxFuture<'_, Result<EntryData, RpcError>> {
Box::pin(async move {
let lvl_i32 = i32::try_from(lvl).unwrap_or(i32::MAX);
let payload = self
.client
.call(
empty_caller(),
self.unicast_id(),
MethodCall {
call: Some(Call::HookingSearchMigrationPath(lvl_i32)),
},
)
.await?;
let result = match payload.value {
Some(RespValue::Typed(tv)) => {
ntk_hooking::decode_entry_data(&tv).map_err(|e| malformed(e.to_string()))
}
_ => Err(malformed("search_migration_path: expected a typed reply")),
};
match &result {
Ok(entry) => tracing::info!(
ask_lvl = lvl,
target_network_id = entry.network_id,
target_pos = ?entry.pos,
"migration-instrumentation: search_migration_path resolved"
),
Err(err) => tracing::info!(
ask_lvl = lvl,
%err,
"migration-instrumentation: search_migration_path failed"
),
}
result
})
}
fn route_search_request(
&self,
req: SearchMigrationPathRequest,
) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteSearchRequest,
ntk_hooking::encode_search_request(&req)
)
}
fn route_search_error(
&self,
pkt: SearchMigrationPathErrorPkt,
) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteSearchError,
ntk_hooking::encode_search_error(&pkt)
)
}
fn route_search_response(
&self,
resp: SearchMigrationPathResponse,
) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteSearchResponse,
ntk_hooking::encode_search_response(&resp)
)
}
fn route_explore_request(
&self,
req: ExploreGNodeRequest,
) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteExploreRequest,
ntk_hooking::encode_explore_request(&req)
)
}
fn route_explore_response(
&self,
resp: ExploreGNodeResponse,
) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteExploreResponse,
ntk_hooking::encode_explore_response(&resp)
)
}
fn route_delete_reserve_request(
&self,
req: DeleteReservationRequest,
) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteDeleteReserveRequest,
ntk_hooking::encode_delete_reserve_request(&req)
)
}
fn route_mig_request(&self, req: RequestPacket) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteMigRequest,
ntk_hooking::encode_mig_request(&req)
)
}
fn route_mig_response(&self, resp: ResponsePacket) -> BoxFuture<'_, Result<(), RpcError>> {
notify_call!(
self,
HookingRouteMigResponse,
ntk_hooking::encode_mig_response(&resp)
)
}
}
#[derive(Debug)]
pub struct HookingStubFactoryAdapter {
pub qspn: crate::node::adapters::Live<ntk_qspn::QspnHandle>,
pub links: Arc<PeerLinks>,
pub registry: Arc<LinkRegistry>,
}
impl HookingStubFactory for HookingStubFactoryAdapter {
fn arc_stub(&self, arc: ntk_hooking::ArcId) -> Arc<dyn HookingStub> {
let link = LinkId(arc.0);
self.links.get(link).map_or_else(
|| Arc::new(UnreachableHookingStub) as Arc<dyn HookingStub>,
|client| Arc::new(RpcHookingStub { client }) as Arc<dyn HookingStub>,
)
}
fn gateway_stub(&self, hc: ntk_common::HCoord) -> Option<Arc<dyn HookingStub>> {
let snapshot = self.qspn.get().snapshot();
let entry = snapshot
.levels
.get(hc.level)?
.iter()
.find(|e| e.destination == hc)?;
let arc = entry.paths.first()?.arc;
let link = self.registry.link_of_qspn_arc(arc)?;
let client = self.links.get(link)?;
Some(Arc::new(RpcHookingStub { client }))
}
}
struct UnreachableHookingStub;
impl HookingStub for UnreachableHookingStub {
fn retrieve_network_data(
&self,
_ask_coord: bool,
) -> BoxFuture<'_, Result<NetworkData, RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn search_migration_path(&self, _lvl: usize) -> BoxFuture<'_, Result<EntryData, RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_search_request(
&self,
_req: SearchMigrationPathRequest,
) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_search_error(
&self,
_pkt: SearchMigrationPathErrorPkt,
) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_search_response(
&self,
_resp: SearchMigrationPathResponse,
) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_explore_request(
&self,
_req: ExploreGNodeRequest,
) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_explore_response(
&self,
_resp: ExploreGNodeResponse,
) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_delete_reserve_request(
&self,
_req: DeleteReservationRequest,
) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_mig_request(&self, _req: RequestPacket) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
fn route_mig_response(&self, _resp: ResponsePacket) -> BoxFuture<'_, Result<(), RpcError>> {
Box::pin(async move { Err(RpcError::ConnectionClosed) })
}
}
#[cfg(test)]
mod identity_stub_tests {
use std::sync::Arc;
use ntk_identities::IdentityStubFactory;
use super::IdentityStubFactoryAdapter;
use crate::node::peers::PeerLinks;
use crate::node::registry::LinkRegistry;
#[test]
fn stub_for_an_arc_without_a_connection_does_not_panic() {
let factory = IdentityStubFactoryAdapter {
links: Arc::new(PeerLinks::new()),
registry: Arc::new(LinkRegistry::new()),
};
let _client = factory.stub(ntk_identities::ArcId(42));
}
}