use crate::inbox::{BusMessage, Inbox, RequestContext};
use crate::reply::CorrelationId;
use crate::sequence::{SequenceReservations, Sequencer};
use crate::session::{self, IncomingSession, Session, SessionHandlers, Timing};
use crate::transport::{
AuthenticatedPeer, DeliveryProvenance, Inbound, ReplyRoute, SessionStream, Transport,
};
use crate::{BusError, Result, Topic};
use agent_mesh_discovery::{AnnounceConfig, Announcer, AnnouncerHandle};
use agent_mesh_protocol::{AgentKey, CertChain, Fingerprint, Recipient, SignedEnvelope, UserKey};
use agent_mesh_transport::{
do_handshake,
identity::agent_pubkey_to_iroh,
iroh_reexports::{Connection, Incoming, IncomingAddr, PublicKey, RecvStream, SendStream},
recv_envelope, send_envelope, Endpoint, LiveCheck, PeerResolver, ResolverHandle, Revocable,
RevocationGuard, TransportError,
};
use async_trait::async_trait;
use std::future::Future;
use std::net::{IpAddr, SocketAddr};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{broadcast, mpsc, watch, Mutex as AsyncMutex, Notify};
use tokio::task::{JoinHandle, JoinSet};
type IrohReverse = Option<(PublicKey, SocketAddr)>;
struct IrohReplyRoute {
conn: Connection,
reverse: IrohReverse,
}
#[derive(Clone, Copy)]
enum Posture {
Accepting { announce: bool },
OutboundOnly,
}
const RESOLVE_TIMEOUT: Duration = Duration::from_secs(5);
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Debug, Clone)]
pub struct BusOptions {
pub announce: bool,
}
impl Default for BusOptions {
fn default() -> Self {
Self { announce: true }
}
}
#[derive(Debug, Clone, Copy)]
pub struct PeerEndpoint {
pub agent_pubkey: [u8; 32],
pub addr: SocketAddr,
}
impl PeerEndpoint {
#[must_use]
pub fn new(agent_pubkey: [u8; 32], addr: SocketAddr) -> Self {
Self { agent_pubkey, addr }
}
#[must_use]
pub fn from_parts(agent_pubkey: [u8; 32], ip: IpAddr, port: u16) -> Self {
Self::new(agent_pubkey, SocketAddr::new(ip, port))
}
#[must_use]
pub fn fingerprint(&self) -> Fingerprint {
Fingerprint::of_bytes(&self.agent_pubkey)
}
}
pub struct Bus {
agent: Arc<AgentKey>,
user_fp: Fingerprint,
transport: Arc<dyn Transport>,
inbox: Arc<Inbox>,
sequence: Arc<Sequencer>,
sessions: Arc<SessionHandlers>,
shutdown: watch::Sender<()>,
stop: Arc<Notify>,
accept_task: JoinHandle<()>,
}
fn verified_local_user(agent: &AgentKey) -> Result<Fingerprint> {
agent.cert().verify()?;
Ok(agent.cert().user_fingerprint())
}
fn ensure_local_user(supplied_user_fp: Fingerprint, certified_user_fp: Fingerprint) -> Result<()> {
if supplied_user_fp != certified_user_fp {
return Err(BusError::LocalIdentityMismatch {
supplied_user_fp: supplied_user_fp.hex(),
certified_user_fp: certified_user_fp.hex(),
});
}
Ok(())
}
impl Bus {
pub async fn bind(
user: &UserKey,
agent: AgentKey,
port: u16,
revocation: RevocationGuard,
) -> Result<Self> {
Self::bind_with(user, agent, port, BusOptions::default(), revocation).await
}
pub async fn bind_with(
user: &UserKey,
agent: AgentKey,
port: u16,
opts: BusOptions,
revocation: RevocationGuard,
) -> Result<Self> {
Self::bind_listening(user, agent, port, opts, Sequencer::in_memory(), revocation).await
}
pub async fn bind_with_reserving(
user: &UserKey,
agent: AgentKey,
port: u16,
opts: BusOptions,
reservations: Arc<dyn SequenceReservations>,
revocation: RevocationGuard,
) -> Result<Self> {
Self::bind_listening(
user,
agent,
port,
opts,
Sequencer::reserving(reservations)?,
revocation,
)
.await
}
async fn bind_listening(
user: &UserKey,
agent: AgentKey,
port: u16,
opts: BusOptions,
sequence: Sequencer,
revocation: RevocationGuard,
) -> Result<Self> {
let user_fp = verified_local_user(&agent)?;
ensure_local_user(user.fingerprint(), user_fp)?;
let agent = Arc::new(agent);
let transport = IrohTransport::bind(user_fp, agent.clone(), port, opts, revocation).await?;
Self::assemble(agent, Arc::new(transport), sequence)
}
pub async fn bind_outbound_only(
user: &UserKey,
agent: AgentKey,
revocation: RevocationGuard,
) -> Result<Self> {
Self::bind_outbound(user, agent, Sequencer::in_memory(), revocation).await
}
pub async fn bind_outbound_only_reserving(
user: &UserKey,
agent: AgentKey,
reservations: Arc<dyn SequenceReservations>,
revocation: RevocationGuard,
) -> Result<Self> {
Self::bind_outbound(user, agent, Sequencer::reserving(reservations)?, revocation).await
}
async fn bind_outbound(
user: &UserKey,
agent: AgentKey,
sequence: Sequencer,
revocation: RevocationGuard,
) -> Result<Self> {
let user_fp = verified_local_user(&agent)?;
ensure_local_user(user.fingerprint(), user_fp)?;
let agent = Arc::new(agent);
let transport = IrohTransport::bind_posture(
user_fp,
agent.clone(),
0,
Posture::OutboundOnly,
revocation,
)
.await?;
Self::assemble(agent, Arc::new(transport), sequence)
}
pub fn bind_with_transport(
agent: Arc<AgentKey>,
transport: Arc<dyn Transport>,
) -> Result<Self> {
Self::assemble(agent, transport, Sequencer::in_memory())
}
pub fn bind_with_transport_reserving(
agent: Arc<AgentKey>,
transport: Arc<dyn Transport>,
reservations: Arc<dyn SequenceReservations>,
) -> Result<Self> {
Self::assemble(agent, transport, Sequencer::reserving(reservations)?)
}
fn assemble(
agent: Arc<AgentKey>,
transport: Arc<dyn Transport>,
sequence: Sequencer,
) -> Result<Self> {
let user_fp = verified_local_user(&agent)?;
let inbox = Arc::new(Inbox::new());
let sequence = Arc::new(sequence);
let sessions = Arc::new(SessionHandlers::default());
let (shutdown, _) = watch::channel(());
let stop = Arc::new(Notify::new());
let accept_task = spawn_accept_loop(
transport.clone(),
agent.clone(),
inbox.clone(),
sequence.clone(),
(sessions.clone(), shutdown.subscribe()),
stop.clone(),
);
Ok(Self {
agent,
user_fp,
transport,
inbox,
sequence,
sessions,
shutdown,
stop,
accept_task,
})
}
#[must_use]
pub fn user_fingerprint(&self) -> Fingerprint {
self.user_fp
}
#[must_use]
pub fn agent_fingerprint(&self) -> Fingerprint {
self.agent.fingerprint()
}
#[must_use]
pub fn local_port(&self) -> u16 {
self.transport.local_port()
}
pub async fn request(
&self,
peer_fp: Fingerprint,
topic: &Topic,
body: Vec<u8>,
timeout: Duration,
) -> Result<Vec<u8>> {
self.request_via(DialRoute::Resolve(peer_fp), topic, body, timeout)
.await
}
pub async fn request_direct(
&self,
peer: PeerEndpoint,
topic: &Topic,
body: Vec<u8>,
timeout: Duration,
) -> Result<Vec<u8>> {
self.request_via(DialRoute::Direct(peer), topic, body, timeout)
.await
}
async fn request_via(
&self,
route: DialRoute,
topic: &Topic,
body: Vec<u8>,
timeout: Duration,
) -> Result<Vec<u8>> {
let correlation = CorrelationId::new_random();
let expected_peer_fp = route.peer_fingerprint();
let waiter = self.inbox.register_reply(correlation, expected_peer_fp);
let msg = BusMessage::Request {
topic: topic.wire(),
correlation: correlation.0,
body,
};
self.send_via(route, msg, Some(timeout)).await?;
match tokio::time::timeout(timeout, waiter).await {
Ok(Ok(payload)) => Ok(payload),
Ok(Err(_)) => Err(BusError::LostReply),
Err(_) => Err(BusError::Timeout(timeout)),
}
}
pub fn handle_requests<F, Fut>(&self, topic: Topic, handler: F)
where
F: Fn(Vec<u8>) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<Vec<u8>>> + Send + 'static,
{
self.inbox.register_handler(topic, handler);
}
pub fn handle_requests_with_context<F, Fut>(&self, topic: Topic, handler: F)
where
F: Fn(RequestContext, Vec<u8>) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<Vec<u8>>> + Send + 'static,
{
self.inbox.register_handler_with_context(topic, handler);
}
pub async fn publish_to(
&self,
peer_fp: Fingerprint,
topic: &Topic,
body: Vec<u8>,
) -> Result<()> {
let msg = BusMessage::Publish {
topic: topic.wire(),
body,
};
self.send_via(DialRoute::Resolve(peer_fp), msg, None).await
}
pub async fn publish_to_direct(
&self,
peer: PeerEndpoint,
topic: &Topic,
body: Vec<u8>,
) -> Result<()> {
let msg = BusMessage::Publish {
topic: topic.wire(),
body,
};
self.send_via(DialRoute::Direct(peer), msg, None).await
}
pub async fn subscribe(&self, topic: &Topic) -> broadcast::Receiver<Vec<u8>> {
self.inbox.subscribe(topic).await
}
pub async fn open_session(&self, peer_fp: Fingerprint, topic: &Topic) -> Result<Session> {
let (peer, stream) = self.transport.open_stream_to(peer_fp).await?;
self.initiate(peer, stream, topic).await
}
pub async fn open_session_direct(&self, peer: PeerEndpoint, topic: &Topic) -> Result<Session> {
let (peer, stream) = self.transport.open_stream_to_endpoint(&peer).await?;
self.initiate(peer, stream, topic).await
}
async fn initiate(
&self,
peer: AuthenticatedPeer,
stream: SessionStream,
topic: &Topic,
) -> Result<Session> {
let (agent, shutdown) = (self.agent.clone(), self.shutdown.subscribe());
Session::initiate(agent, peer, stream, topic, Timing::default(), shutdown).await
}
pub fn handle_sessions<F, Fut>(&self, topic: Topic, handler: F)
where
F: Fn(IncomingSession) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<()>> + Send + 'static,
{
self.sessions.register(&topic, handler);
}
#[must_use]
pub fn with_session_shutdown_grace(self, grace: Duration) -> Self {
self.sessions.set_shutdown_grace(grace);
self
}
pub async fn close(self) -> Result<()> {
drop(self.shutdown);
self.stop.notify_one();
let _ = self.accept_task.await;
self.transport.close().await;
Ok(())
}
async fn send_via(
&self,
route: DialRoute,
msg: BusMessage,
reply_window: Option<Duration>,
) -> Result<()> {
let env = make_envelope(&self.agent, &self.sequence, route.peer_fingerprint(), msg)?;
match (route, reply_window) {
(DialRoute::Resolve(fp), None) => self.transport.send_to(fp, env).await,
(DialRoute::Resolve(fp), Some(w)) => self.transport.send_request_to(fp, env, w).await,
(DialRoute::Direct(peer), None) => self.transport.send_to_endpoint(&peer, env).await,
(DialRoute::Direct(peer), Some(w)) => {
self.transport.send_request_to_endpoint(&peer, env, w).await
}
}
}
}
fn make_envelope(
agent: &AgentKey,
sequence: &Sequencer,
peer_fp: Fingerprint,
msg: BusMessage,
) -> Result<SignedEnvelope> {
let seq = sequence.next()?;
let payload = serde_json::to_vec(&msg)?;
Ok(SignedEnvelope::new(
agent,
Recipient::Direct { agent_fp: peer_fp },
seq,
payload,
))
}
#[derive(Debug, Clone, Copy)]
enum DialRoute {
Resolve(Fingerprint),
Direct(PeerEndpoint),
}
impl DialRoute {
fn peer_fingerprint(self) -> Fingerprint {
match self {
Self::Resolve(peer_fp) => peer_fp,
Self::Direct(peer) => peer.fingerprint(),
}
}
}
async fn dial_endpoint(endpoint: &Endpoint, peer: PeerEndpoint) -> Result<Connection> {
let iroh_pk = agent_pubkey_to_iroh(&peer.agent_pubkey).ok_or_else(|| {
BusError::Unreachable(format!(
"direct-dial endpoint {} has an invalid ed25519 pubkey",
peer.fingerprint().short()
))
})?;
tracing::debug!(
peer = %peer.fingerprint().short(),
addr = %peer.addr,
"bus: direct-dialing peer endpoint (no mDNS)"
);
let conn = endpoint.dial(iroh_pk, [peer.addr]).await?;
Ok(conn)
}
fn is_scopeless_link_local(addr: &SocketAddr) -> bool {
match addr {
SocketAddr::V6(v6) => (v6.ip().segments()[0] & 0xffc0) == 0xfe80 && v6.scope_id() == 0,
SocketAddr::V4(_) => false,
}
}
fn reply_dial_candidates(source: SocketAddr) -> Vec<SocketAddr> {
let port = source.port();
let mut candidates: Vec<SocketAddr> = Vec::with_capacity(3);
for addr in [
source,
SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.into(), port),
SocketAddr::new(std::net::Ipv6Addr::LOCALHOST.into(), port),
] {
if is_scopeless_link_local(&addr) {
continue;
}
if !candidates.contains(&addr) {
candidates.push(addr);
}
}
candidates
}
async fn dial_peer(
endpoint: &Endpoint,
resolver: &PeerResolver,
peer_fp: Fingerprint,
) -> Result<Connection> {
let peer = resolver
.resolve(&peer_fp, RESOLVE_TIMEOUT)
.await
.ok_or_else(|| {
BusError::Unreachable(format!(
"peer {} not announced within {:?}",
peer_fp.short(),
RESOLVE_TIMEOUT
))
})?;
let pubkey = peer.agent_pubkey.ok_or_else(|| {
BusError::Unreachable(format!(
"peer {} announced without ed25519 pubkey",
peer_fp.short()
))
})?;
let advertised_fp = Fingerprint::of_bytes(&pubkey);
if advertised_fp != peer_fp {
return Err(BusError::Unreachable(format!(
"peer {} advertised pubkey for different agent {}",
peer_fp.short(),
advertised_fp.short()
)));
}
let iroh_pk = agent_pubkey_to_iroh(&pubkey).ok_or_else(|| {
BusError::Unreachable(format!(
"peer {} advertised invalid ed25519 pubkey",
peer_fp.short()
))
})?;
let mut socket_addrs: Vec<SocketAddr> = peer
.addrs
.iter()
.copied()
.map(|ip| SocketAddr::new(ip, peer.port))
.collect();
let lo_v4: std::net::IpAddr = std::net::Ipv4Addr::LOCALHOST.into();
let lo_v6: std::net::IpAddr = std::net::Ipv6Addr::LOCALHOST.into();
let lo_addrs = [
SocketAddr::new(lo_v4, peer.port),
SocketAddr::new(lo_v6, peer.port),
];
for addr in lo_addrs {
if !socket_addrs.contains(&addr) {
socket_addrs.push(addr);
}
}
socket_addrs.retain(|addr| !is_scopeless_link_local(addr));
tracing::debug!(
peer = %peer_fp.short(),
addrs = ?socket_addrs,
"bus: dialing peer"
);
if socket_addrs.is_empty() {
return Err(BusError::Unreachable(format!(
"peer {} announced without socket addresses",
peer_fp.short()
)));
}
let conn = endpoint.dial(iroh_pk, socket_addrs).await?;
Ok(conn)
}
fn authenticated_iroh_peer(
session_id: &PublicKey,
peer_cert: &CertChain,
revocation: &RevocationGuard,
) -> Result<AuthenticatedPeer> {
revocation.check(peer_cert).map_err(TransportError::from)?;
if &peer_cert.agent_pubkey != session_id.as_bytes() {
return Err(BusError::Transport(TransportError::Handshake(format!(
"peer Hello certificate agent {} is not bound to QUIC session {}",
peer_cert.agent_fingerprint().short(),
Fingerprint::of_bytes(session_id.as_bytes()).short(),
))));
}
Ok(AuthenticatedPeer::new(
peer_cert.user_fingerprint(),
peer_cert.agent_fingerprint(),
))
}
fn ensure_intended_iroh_peer(peer: AuthenticatedPeer, expected_peer_fp: Fingerprint) -> Result<()> {
if peer.agent_fp != expected_peer_fp {
return Err(BusError::Transport(TransportError::Handshake(format!(
"authenticated peer {} does not match intended target {}",
peer.agent_fp.short(),
expected_peer_fp.short(),
))));
}
Ok(())
}
async fn open_bound_stream(
conn: &Connection,
our_cert: &CertChain,
revocation: &RevocationGuard,
expected_peer_fp: Fingerprint,
) -> Result<(AuthenticatedPeer, CertChain, SendStream, RecvStream)> {
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| BusError::Transport(TransportError::Iroh(format!("open_bi: {e}"))))?;
let peer_cert = tokio::time::timeout(
HANDSHAKE_TIMEOUT,
do_handshake(our_cert, &mut send, &mut recv, true, revocation),
)
.await
.map_err(|_| BusError::Timeout(HANDSHAKE_TIMEOUT))??;
let peer = authenticated_iroh_peer(&conn.remote_id(), &peer_cert, revocation)?;
ensure_intended_iroh_peer(peer, expected_peer_fp)?;
Ok((peer, peer_cert, send, recv))
}
async fn send_env_on_conn(
conn: &Connection,
our_cert: &CertChain,
revocation: &RevocationGuard,
expected_peer_fp: Fingerprint,
env: &SignedEnvelope,
) -> Result<()> {
let (_, _, mut send, _recv) =
open_bound_stream(conn, our_cert, revocation, expected_peer_fp).await?;
send_envelope(&mut send, env).await?;
send.finish()
.map_err(|e| BusError::Transport(TransportError::Iroh(format!("finish: {e}"))))?;
let _ = send.stopped().await;
Ok(())
}
async fn open_session_stream(
conn: &Connection,
our_cert: &CertChain,
revocation: &RevocationGuard,
expected_peer_fp: Fingerprint,
) -> Result<(AuthenticatedPeer, SessionStream)> {
let (peer, peer_cert, send, recv) =
open_bound_stream(conn, our_cert, revocation, expected_peer_fp).await?;
let live = LiveCheck::new(revocation.clone(), peer_cert)?;
Ok((peer, session_stream(send, recv, live)))
}
fn session_stream(send: SendStream, recv: RecvStream, live: LiveCheck) -> SessionStream {
SessionStream {
send: Box::new(Revocable::new(send, live.clone())),
recv: Box::new(Revocable::new(recv, live.clone())),
live,
}
}
struct Queued {
inbound: Inbound,
live: LiveCheck,
}
fn spawn_accept_loop(
transport: Arc<dyn Transport>,
agent: Arc<AgentKey>,
inbox: Arc<Inbox>,
sequence: Arc<Sequencer>,
(sessions, shutdown): (Arc<SessionHandlers>, watch::Receiver<()>),
stop: Arc<Notify>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut tasks: JoinSet<()> = JoinSet::new();
let mut session_tasks: JoinSet<()> = JoinSet::new();
loop {
let inbound = tokio::select! {
biased;
() = stop.notified() => break,
Some(_) = tasks.join_next(), if !tasks.is_empty() => continue,
Some(_) = session_tasks.join_next(), if !session_tasks.is_empty() => continue,
inbound = transport.recv() => match inbound {
Some(inbound) => inbound,
None => break,
},
};
let transport = transport.clone();
let agent = agent.clone();
let inbox = inbox.clone();
let sequence = sequence.clone();
let (sessions, shutdown) = (sessions.clone(), shutdown.clone());
let opens_session = inbound.stream.is_some()
&& crate::session::open_request(&inbound.envelope).is_some();
let set = if opens_session {
&mut session_tasks
} else {
&mut tasks
};
set.spawn(async move {
let Inbound {
envelope,
provenance,
reply_route,
stream,
} = inbound;
if let Some(stream) = stream.filter(|_| session::open_request(&envelope).is_some())
{
let timing = Timing::default();
match IncomingSession::admit(
agent,
&envelope,
provenance,
stream,
timing,
shutdown.clone(),
) {
Ok(incoming) => sessions.dispatch(incoming, shutdown).await,
Err(e) => tracing::warn!(error = %e, "bus: session open rejected"),
}
return;
}
let local_user_fp = agent.cert().user_fingerprint();
let local_agent_fp = agent.fingerprint();
match inbox
.on_envelope(envelope, provenance, local_user_fp, local_agent_fp)
.await
{
Ok(Some(reply)) => {
let msg = BusMessage::Reply {
correlation: reply.correlation.0,
body: reply.body,
};
match make_envelope(&agent, &sequence, reply.peer_fp, msg) {
Ok(env) => {
if let Err(e) =
transport.reply(reply.peer_fp, &reply_route, env).await
{
tracing::warn!(error = %e, "bus: reply ship failed");
}
}
Err(e) => tracing::warn!(error = %e, "bus: reply encode failed"),
}
}
Ok(None) => {}
Err(e) => tracing::warn!(error = %e, "bus: inbox rejected envelope"),
}
});
}
abort_and_drain(&mut tasks).await;
drain(&mut session_tasks).await;
tracing::debug!("bus accept loop: stopped");
})
}
async fn drain(set: &mut JoinSet<()>) {
while set.join_next().await.is_some() {}
}
async fn abort_and_drain(set: &mut JoinSet<()>) {
set.abort_all();
drain(set).await;
}
async fn dial_reply_peer(
endpoint: &Endpoint,
resolver: &PeerResolver,
peer_fp: Fingerprint,
reverse: IrohReverse,
) -> Result<Connection> {
if let Some((pubkey, addr)) = reverse {
if Fingerprint::of_bytes(pubkey.as_bytes()) == peer_fp {
let candidates = reply_dial_candidates(addr);
if !candidates.is_empty() {
match endpoint.dial(pubkey, candidates.iter().copied()).await {
Ok(conn) => return Ok(conn),
Err(e) => tracing::debug!(
peer = %peer_fp.short(),
?candidates,
error = %e,
"bus: reply dial-back failed, falling back to mDNS"
),
}
}
} else {
tracing::warn!(
peer = %peer_fp.short(),
"bus: connection key does not match envelope sender; ignoring dial-back route"
);
}
}
dial_peer(endpoint, resolver, peer_fp).await
}
pub struct IrohTransport {
endpoint: Arc<Endpoint>,
resolver: Arc<PeerResolver>,
agent: Arc<AgentKey>,
revocation: RevocationGuard,
_resolver_handle: ResolverHandle,
_announcer: Option<AnnouncerHandle>,
inbound: AsyncMutex<mpsc::UnboundedReceiver<Queued>>,
inbound_tx: mpsc::UnboundedSender<Queued>,
stop: Arc<Notify>,
accept_task: std::sync::Mutex<Option<JoinHandle<()>>>,
reply_tasks: std::sync::Mutex<JoinSet<()>>,
}
impl IrohTransport {
pub async fn bind(
user_fp: Fingerprint,
agent: Arc<AgentKey>,
port: u16,
opts: BusOptions,
revocation: RevocationGuard,
) -> Result<Self> {
let posture = Posture::Accepting {
announce: opts.announce,
};
Self::bind_posture(user_fp, agent, port, posture, revocation).await
}
async fn bind_posture(
user_fp: Fingerprint,
agent: Arc<AgentKey>,
port: u16,
posture: Posture,
revocation: RevocationGuard,
) -> Result<Self> {
let certified_user_fp = verified_local_user(&agent)?;
ensure_local_user(user_fp, certified_user_fp)?;
let log_user = revocation.state().map_err(TransportError::from)?.user;
ensure_local_user(log_user, certified_user_fp)?;
let endpoint = Endpoint::bind(&agent, port).await?;
let local_port = endpoint.port();
let endpoint = Arc::new(endpoint);
let (resolver, resolver_handle) = PeerResolver::start()?;
let resolver = Arc::new(resolver);
let announcer = if matches!(posture, Posture::Accepting { announce: true }) {
Some(
Announcer::start(AnnounceConfig {
agent_fp: agent.fingerprint(),
agent_pubkey: Some(agent.public_bytes()),
user_fp,
capabilities: agent.cert().metadata.capabilities.clone(),
role: agent.cert().metadata.role.clone(),
host: agent.cert().metadata.host.clone(),
port: local_port,
})
.map_err(|e| {
BusError::Transport(TransportError::Iroh(format!("announce start: {e}")))
})?,
)
} else {
None
};
let (inbound_tx, inbound_rx) = mpsc::unbounded_channel();
let stop = Arc::new(Notify::new());
let accept_task = spawn_iroh_accept_loop(
endpoint.clone(),
agent.clone(),
revocation.clone(),
inbound_tx.clone(),
posture,
stop.clone(),
);
Ok(Self {
endpoint,
resolver,
agent,
revocation,
_resolver_handle: resolver_handle,
_announcer: announcer,
inbound: AsyncMutex::new(inbound_rx),
inbound_tx,
stop,
accept_task: std::sync::Mutex::new(Some(accept_task)),
reply_tasks: std::sync::Mutex::new(JoinSet::new()),
})
}
}
#[async_trait]
impl Transport for IrohTransport {
async fn send_to(&self, fp: Fingerprint, env: SignedEnvelope) -> Result<()> {
let conn = dial_peer(&self.endpoint, &self.resolver, fp).await?;
send_env_on_conn(&conn, self.agent.cert(), &self.revocation, fp, &env).await
}
async fn send_to_endpoint(&self, peer: &PeerEndpoint, env: SignedEnvelope) -> Result<()> {
let conn = dial_endpoint(&self.endpoint, *peer).await?;
send_env_on_conn(
&conn,
self.agent.cert(),
&self.revocation,
peer.fingerprint(),
&env,
)
.await
}
async fn send_request_to(
&self,
fp: Fingerprint,
env: SignedEnvelope,
window: Duration,
) -> Result<()> {
let conn = dial_peer(&self.endpoint, &self.resolver, fp).await?;
send_env_on_conn(&conn, self.agent.cert(), &self.revocation, fp, &env).await?;
self.await_reply_on(conn, window);
Ok(())
}
async fn send_request_to_endpoint(
&self,
peer: &PeerEndpoint,
env: SignedEnvelope,
window: Duration,
) -> Result<()> {
let conn = dial_endpoint(&self.endpoint, *peer).await?;
send_env_on_conn(
&conn,
self.agent.cert(),
&self.revocation,
peer.fingerprint(),
&env,
)
.await?;
self.await_reply_on(conn, window);
Ok(())
}
async fn open_stream_to(&self, fp: Fingerprint) -> Result<(AuthenticatedPeer, SessionStream)> {
let conn = dial_peer(&self.endpoint, &self.resolver, fp).await?;
open_session_stream(&conn, self.agent.cert(), &self.revocation, fp).await
}
async fn open_stream_to_endpoint(
&self,
peer: &PeerEndpoint,
) -> Result<(AuthenticatedPeer, SessionStream)> {
let conn = dial_endpoint(&self.endpoint, *peer).await?;
open_session_stream(
&conn,
self.agent.cert(),
&self.revocation,
peer.fingerprint(),
)
.await
}
async fn reply(&self, fp: Fingerprint, route: &ReplyRoute, env: SignedEnvelope) -> Result<()> {
let route = route.downcast_ref::<IrohReplyRoute>();
if let Some(route) = route {
if Fingerprint::of_bytes(route.conn.remote_id().as_bytes()) == fp {
match send_env_on_conn(&route.conn, self.agent.cert(), &self.revocation, fp, &env)
.await
{
Ok(()) => return Ok(()),
Err(e) => tracing::debug!(
peer = %fp.short(),
error = %e,
"bus: reply on the request's connection failed; dialing back"
),
}
}
}
let reverse = route.and_then(|r| r.reverse);
let conn = dial_reply_peer(&self.endpoint, &self.resolver, fp, reverse).await?;
send_env_on_conn(&conn, self.agent.cert(), &self.revocation, fp, &env).await
}
async fn recv(&self) -> Option<Inbound> {
let mut queue = self.inbound.lock().await;
loop {
let Queued { inbound, live } = queue.recv().await?;
match live.admit() {
Ok(()) => return Some(inbound),
Err(e) => tracing::warn!(error = %e, "iroh transport: dropped a queued envelope"),
}
}
}
fn local_port(&self) -> u16 {
self.endpoint.port()
}
async fn close(&self) {
self.stop.notify_one();
let accept = self
.accept_task
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
if let Some(accept) = accept {
let _ = accept.await;
}
let mut replies = std::mem::take(
&mut *self
.reply_tasks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
);
abort_and_drain(&mut replies).await;
self.endpoint.close().await;
}
}
impl IrohTransport {
fn await_reply_on(&self, conn: Connection, window: Duration) {
let (agent, revocation, inbound_tx) = (
self.agent.clone(),
self.revocation.clone(),
self.inbound_tx.clone(),
);
let mut tasks = self
.reply_tasks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
while tasks.try_join_next().is_some() {}
tasks.spawn(async move {
let route: ReplyRoute = Arc::new(IrohReplyRoute {
conn: conn.clone(),
reverse: None,
});
let first =
tokio::time::timeout(window, next_envelope(&conn, &agent, &revocation)).await;
if let Ok(Some((provenance, envelope, stream))) = first {
let _ = inbound_tx.send(Queued {
inbound: Inbound {
envelope,
provenance,
reply_route: route,
stream: None,
},
live: stream.live,
});
}
});
}
}
fn spawn_iroh_accept_loop(
endpoint: Arc<Endpoint>,
agent: Arc<AgentKey>,
revocation: RevocationGuard,
inbound_tx: mpsc::UnboundedSender<Queued>,
posture: Posture,
stop: Arc<Notify>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut conns: JoinSet<()> = JoinSet::new();
loop {
let incoming = tokio::select! {
biased;
() = stop.notified() => break,
Some(_) = conns.join_next(), if !conns.is_empty() => continue,
incoming = endpoint.accept() => incoming,
};
let Some(incoming) = incoming else {
tracing::debug!("iroh transport accept loop: endpoint closed");
break;
};
if matches!(posture, Posture::OutboundOnly) {
tracing::debug!("iroh transport: outbound-only; refusing inbound connection");
incoming.refuse();
continue;
}
let agent = agent.clone();
let revocation = revocation.clone();
let inbound_tx = inbound_tx.clone();
conns.spawn(async move {
if let Err(e) = accept_conn(incoming, agent, revocation, inbound_tx).await {
tracing::warn!(error = %e, "iroh transport: incoming connection error");
}
});
}
abort_and_drain(&mut conns).await;
})
}
async fn accept_conn(
incoming: Incoming,
agent: Arc<AgentKey>,
revocation: RevocationGuard,
inbound_tx: mpsc::UnboundedSender<Queued>,
) -> Result<()> {
let reverse_addr = match incoming.remote_addr() {
IncomingAddr::Ip(addr) => Some(addr),
_ => None,
};
let conn = incoming
.await
.map_err(|e| BusError::Transport(TransportError::Iroh(format!("incoming: {e}"))))?;
let reply_route: ReplyRoute = Arc::new(IrohReplyRoute {
conn: conn.clone(),
reverse: reverse_addr.map(|addr| (conn.remote_id(), addr)),
});
while let Some((provenance, envelope, stream)) = next_envelope(&conn, &agent, &revocation).await
{
let live = stream.live.clone();
let inbound = Inbound {
envelope,
provenance,
reply_route: reply_route.clone(),
stream: Some(stream),
};
if inbound_tx.send(Queued { inbound, live }).is_err() {
tracing::debug!("iroh transport: bus receiver dropped; stopping");
break;
}
}
Ok(())
}
async fn next_envelope(
conn: &Connection,
agent: &AgentKey,
revocation: &RevocationGuard,
) -> Option<(DeliveryProvenance, SignedEnvelope, SessionStream)> {
loop {
let (mut send, mut recv) = match conn.accept_bi().await {
Ok(streams) => streams,
Err(e) => {
tracing::debug!(error = %e, "iroh transport: accept_bi ended (peer closed)");
return None;
}
};
let cert = agent.cert().clone();
let peer_cert = match tokio::time::timeout(
HANDSHAKE_TIMEOUT,
do_handshake(&cert, &mut send, &mut recv, false, revocation),
)
.await
{
Ok(Ok(peer_cert)) => peer_cert,
Ok(Err(e)) => {
tracing::warn!(error = %e, "iroh transport: handshake rejected");
continue;
}
Err(_) => {
tracing::warn!("iroh transport: handshake timed out");
continue;
}
};
let carrier = match authenticated_iroh_peer(&conn.remote_id(), &peer_cert, revocation) {
Ok(peer) => peer,
Err(e) => {
tracing::warn!(error = %e, "iroh transport: Hello/session identity mismatch");
continue;
}
};
let live = match LiveCheck::new(revocation.clone(), peer_cert) {
Ok(live) => live,
Err(e) => {
tracing::warn!(error = %e, "iroh transport: peer revoked");
continue;
}
};
match recv_envelope(&mut recv, &live).await {
Ok(env) => {
let stream = session_stream(send, recv, live);
return Some((DeliveryProvenance::Direct { carrier }, env, stream));
}
Err(e) => tracing::warn!(error = %e, "iroh transport: envelope read failed"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::reply::ReplyDelivery;
use crate::transport::MeshNet;
use agent_mesh_protocol::{AgentMetadata, Caveats, UserKey};
fn agent(user: &UserKey, role: &str) -> AgentKey {
AgentKey::issue(
user,
AgentMetadata {
role: role.into(),
host: "test".into(),
capabilities: vec!["test".into()],
issued_at: "2026-05-28T00:00:00Z".into(),
expires_at: None,
caveats: Caveats::top(),
},
)
}
fn direct_provenance(agent: &AgentKey) -> DeliveryProvenance {
DeliveryProvenance::Direct {
carrier: AuthenticatedPeer::new(agent.cert().user_fingerprint(), agent.fingerprint()),
}
}
enum CapturedSend {
Resolve(Fingerprint, SignedEnvelope),
Direct(PeerEndpoint, SignedEnvelope),
}
struct CaptureTransport {
sent_tx: mpsc::UnboundedSender<CapturedSend>,
sent_rx: AsyncMutex<mpsc::UnboundedReceiver<CapturedSend>>,
stall: bool,
}
impl CaptureTransport {
fn new() -> Arc<Self> {
Self::build(false)
}
fn stalling() -> Arc<Self> {
Self::build(true)
}
fn build(stall: bool) -> Arc<Self> {
let (sent_tx, sent_rx) = mpsc::unbounded_channel();
Arc::new(Self {
sent_tx,
sent_rx: AsyncMutex::new(sent_rx),
stall,
})
}
async fn next_sent(&self) -> CapturedSend {
self.sent_rx
.lock()
.await
.recv()
.await
.expect("bus keeps capture transport alive")
}
}
struct Gone(Arc<std::sync::atomic::AtomicBool>);
impl Drop for Gone {
fn drop(&mut self) {
self.0.store(true, std::sync::atomic::Ordering::SeqCst);
}
}
#[tokio::test]
async fn abort_and_drain_returns_only_after_the_futures_are_dropped() {
let gone = Arc::new(std::sync::atomic::AtomicBool::new(false));
let mut set = JoinSet::new();
let (g, started) = (Gone(gone.clone()), Arc::new(Notify::new()));
let s = started.clone();
set.spawn(async move {
let _g = g;
s.notify_one();
std::future::pending::<()>().await;
});
started.notified().await;
abort_and_drain(&mut set).await;
assert!(gone.load(std::sync::atomic::Ordering::SeqCst));
assert!(set.is_empty());
}
#[tokio::test]
async fn drain_waits_for_tasks_to_finish() {
let done = Arc::new(std::sync::atomic::AtomicBool::new(false));
let mut set = JoinSet::new();
let (d, go) = (done.clone(), Arc::new(Notify::new()));
let g = go.clone();
set.spawn(async move {
g.notified().await;
d.store(true, std::sync::atomic::Ordering::SeqCst);
});
go.notify_one();
drain(&mut set).await;
assert!(done.load(std::sync::atomic::Ordering::SeqCst));
}
#[async_trait]
impl Transport for CaptureTransport {
async fn send_to(&self, fp: Fingerprint, env: SignedEnvelope) -> Result<()> {
self.sent_tx
.send(CapturedSend::Resolve(fp, env))
.expect("capture receiver is alive");
if self.stall {
std::future::pending::<()>().await;
}
Ok(())
}
async fn send_to_endpoint(&self, peer: &PeerEndpoint, env: SignedEnvelope) -> Result<()> {
self.sent_tx
.send(CapturedSend::Direct(*peer, env))
.expect("capture receiver is alive");
Ok(())
}
async fn reply(
&self,
_fp: Fingerprint,
_route: &ReplyRoute,
_env: SignedEnvelope,
) -> Result<()> {
unreachable!("capture transport does not receive requests")
}
async fn recv(&self) -> Option<Inbound> {
std::future::pending().await
}
fn local_port(&self) -> u16 {
0
}
async fn close(&self) {}
}
fn reply_envelope(
sender: &AgentKey,
recipient_fp: Fingerprint,
correlation: CorrelationId,
body: &[u8],
) -> SignedEnvelope {
let payload = serde_json::to_vec(&BusMessage::Reply {
correlation: correlation.0,
body: body.to_vec(),
})
.expect("encode reply");
let env = SignedEnvelope::new(
sender,
Recipient::Direct {
agent_fp: recipient_fp,
},
1,
payload,
);
env.verify().expect("test reply envelope verifies");
env
}
async fn wrong_signer_cannot_win_for_request_route(direct: bool) {
let user = UserKey::generate();
let asker = Arc::new(agent(&user, "asker"));
let expected = agent(&user, "expected");
let attacker = agent(&user, "attacker");
let asker_fp = asker.fingerprint();
let expected_fp = expected.fingerprint();
let transport = CaptureTransport::new();
let bus = Bus::bind_with_transport(asker, Arc::clone(&transport) as Arc<dyn Transport>)
.expect("bind asker to capture transport");
let route = if direct {
DialRoute::Direct(PeerEndpoint::new(
expected.public_bytes(),
"127.0.0.1:7".parse().unwrap(),
))
} else {
DialRoute::Resolve(expected_fp)
};
let topic = Topic::new(user.fingerprint(), "bound-reply");
{
let request =
bus.request_via(route, &topic, b"request".to_vec(), Duration::from_secs(1));
tokio::pin!(request);
let captured = tokio::select! {
biased;
sent = transport.next_sent() => sent,
result = &mut request => panic!("request completed before a reply: {result:?}"),
};
let (was_direct, target_fp, outbound) = match captured {
CapturedSend::Resolve(fp, env) => (false, fp, env),
CapturedSend::Direct(peer, env) => (true, peer.fingerprint(), env),
};
assert_eq!(was_direct, direct, "request must use the selected route");
assert_eq!(target_fp, expected_fp);
outbound.verify().expect("outbound request verifies");
let correlation = match serde_json::from_slice::<BusMessage>(outbound.payload.as_ref())
.expect("decode captured request")
{
BusMessage::Request { correlation, .. } => CorrelationId(correlation),
other => panic!("expected Request, got {other:?}"),
};
assert_eq!(
bus.inbox.pending_replies(),
1,
"request route must register its waiter before sending"
);
bus.inbox
.on_envelope(
reply_envelope(&attacker, asker_fp, correlation, b"forged"),
direct_provenance(&attacker),
user.fingerprint(),
asker_fp,
)
.await
.unwrap();
assert_eq!(
bus.inbox.pending_replies(),
1,
"wrong signer must not consume the route-bound waiter"
);
bus.inbox
.on_envelope(
reply_envelope(&expected, asker_fp, correlation, b"legitimate"),
direct_provenance(&expected),
user.fingerprint(),
asker_fp,
)
.await
.unwrap();
assert_eq!(request.await.unwrap(), b"legitimate");
assert_eq!(bus.inbox.pending_replies(), 0);
}
bus.close().await.unwrap();
}
#[test]
fn local_identity_requires_a_valid_cert_and_matching_user() {
let certified_user = UserKey::generate();
let different_user = UserKey::generate();
let local_agent = agent(&certified_user, "local");
let certified_fp = verified_local_user(&local_agent).expect("valid local certificate");
let mismatch = ensure_local_user(different_user.fingerprint(), certified_fp).unwrap_err();
assert!(matches!(mismatch, BusError::LocalIdentityMismatch { .. }));
let mut tampered_cert = local_agent.cert().clone();
tampered_cert.metadata.role = "tampered".into();
let invalid_agent =
AgentKey::from_seed_and_cert(&local_agent.signing_key_bytes(), tampered_cert)
.expect("seed still matches the unchanged leaf key");
assert!(
verified_local_user(&invalid_agent).is_err(),
"a matching private key must not make an invalid certificate trusted"
);
}
#[test]
fn hello_certificate_is_bound_to_its_quic_session() {
let user = UserKey::generate();
let a = agent(&user, "a");
let b = agent(&user, "b");
let a_session = agent_pubkey_to_iroh(&a.public_bytes()).expect("valid ed25519 key");
let b_session = agent_pubkey_to_iroh(&b.public_bytes()).expect("valid ed25519 key");
let peer =
authenticated_iroh_peer(&a_session, a.cert(), &RevocationGuard::in_memory(&user))
.expect("A owns A's session");
assert_eq!(peer.agent_fp, a.fingerprint());
assert_eq!(peer.user_fp, user.fingerprint());
ensure_intended_iroh_peer(peer, a.fingerprint()).expect("A is the intended target");
let err = ensure_intended_iroh_peer(peer, b.fingerprint()).unwrap_err();
assert!(
matches!(err, BusError::Transport(TransportError::Handshake(_))),
"an authenticated A session must not satisfy an intended B target, got {err:?}"
);
let err = authenticated_iroh_peer(&b_session, a.cert(), &RevocationGuard::in_memory(&user))
.unwrap_err();
assert!(
matches!(err, BusError::Transport(TransportError::Handshake(_))),
"a copied A Hello cert on B's TLS session must be rejected, got {err:?}"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_queued_envelope_is_not_delivered_once_its_carrier_is_revoked() {
use agent_mesh_protocol::{RevocationEntry, RevocationLog};
let user = UserKey::generate();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("revocation.json");
let mut log = RevocationLog::genesis(&user).unwrap();
std::fs::write(&path, serde_json::to_vec(&log).unwrap()).unwrap();
let sender = agent(&user, "sender");
let recipient = Arc::new(agent(&user, "recipient"));
let recipient_fp = recipient.fingerprint();
let recipient_id = agent_pubkey_to_iroh(&recipient.public_bytes()).unwrap();
let transport = IrohTransport::bind(
user.fingerprint(),
recipient,
0,
BusOptions { announce: false },
RevocationGuard::open(&path, user.fingerprint()).unwrap(),
)
.await
.unwrap();
let addr = SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.into(), transport.local_port());
let ep = Endpoint::bind(&sender, 0).await.unwrap();
let conn = ep.dial(recipient_id, [addr]).await.unwrap();
let msg = BusMessage::Publish {
topic: Topic::new(user.fingerprint(), "q").wire(),
body: b"queued".to_vec(),
};
let env = SignedEnvelope::new(
&sender,
Recipient::Direct {
agent_fp: recipient_fp,
},
1,
serde_json::to_vec(&msg).unwrap(),
);
send_env_on_conn(
&conn,
sender.cert(),
&RevocationGuard::in_memory(&user),
recipient_fp,
&env,
)
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(300)).await;
log.append(
&user,
RevocationEntry::Revoke {
agent: sender.fingerprint(),
},
)
.unwrap();
std::fs::write(&path, serde_json::to_vec(&log).unwrap()).unwrap();
let got = tokio::time::timeout(Duration::from_secs(1), transport.recv()).await;
assert!(
got.is_err(),
"a revoked sender's queued envelope was delivered"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn relayed_envelope_on_sibling_quic_session_is_rejected_before_replay() {
let user = UserKey::generate();
let original_signer = agent(&user, "original-signer");
let sibling_carrier = agent(&user, "sibling-carrier");
let recipient = Arc::new(agent(&user, "recipient"));
let recipient_pubkey = recipient.public_bytes();
let recipient_fp = recipient.fingerprint();
let recipient_transport = IrohTransport::bind(
user.fingerprint(),
recipient.clone(),
0,
BusOptions { announce: false },
RevocationGuard::in_memory(&user),
)
.await
.expect("bind recipient transport");
let inbox = Inbox::new();
let topic = Topic::new(user.fingerprint(), "carrier-binding");
let msg = BusMessage::Publish {
topic: topic.wire(),
body: b"authentic payload".to_vec(),
};
let env = SignedEnvelope::new(
&original_signer,
Recipient::Direct {
agent_fp: recipient_fp,
},
1,
serde_json::to_vec(&msg).expect("encode bus message"),
);
let recipient_id =
agent_pubkey_to_iroh(&recipient_pubkey).expect("recipient has a valid iroh key");
let recipient_addr = SocketAddr::new(
std::net::Ipv4Addr::LOCALHOST.into(),
recipient_transport.local_port(),
);
let sibling_endpoint = Endpoint::bind(&sibling_carrier, 0)
.await
.expect("bind sibling endpoint");
let sibling_conn = sibling_endpoint
.dial(recipient_id, [recipient_addr])
.await
.expect("sibling dials recipient");
send_env_on_conn(
&sibling_conn,
sibling_carrier.cert(),
&RevocationGuard::in_memory(&user),
recipient_fp,
&env,
)
.await
.expect("relay bytes reach the recipient boundary");
drop(sibling_conn);
let relayed = tokio::time::timeout(Duration::from_secs(3), recipient_transport.recv())
.await
.expect("relay reaches transport boundary before timeout")
.expect("recipient transport remains open");
assert_eq!(
relayed.provenance,
DeliveryProvenance::Direct {
carrier: AuthenticatedPeer::new(user.fingerprint(), sibling_carrier.fingerprint(),),
}
);
let error = inbox
.on_envelope(
relayed.envelope,
relayed.provenance,
user.fingerprint(),
recipient_fp,
)
.await
.expect_err("a sibling carrier must not speak as the original signer");
assert!(matches!(error, BusError::CarrierAgentMismatch { .. }));
assert!(inbox.nonce_cache().is_empty());
assert_eq!(
inbox
.sequence_tracker()
.last_seen(&original_signer.fingerprint()),
None
);
let signer_endpoint = Endpoint::bind(&original_signer, 0)
.await
.expect("bind signer endpoint");
let signer_conn = signer_endpoint
.dial(recipient_id, [recipient_addr])
.await
.expect("signer dials recipient");
send_env_on_conn(
&signer_conn,
original_signer.cert(),
&RevocationGuard::in_memory(&user),
recipient_fp,
&env,
)
.await
.expect("honest envelope reaches recipient");
let honest = tokio::time::timeout(Duration::from_secs(3), recipient_transport.recv())
.await
.expect("honest delivery reaches transport boundary before timeout")
.expect("recipient transport remains open");
inbox
.on_envelope(
honest.envelope,
honest.provenance,
user.fingerprint(),
recipient_fp,
)
.await
.expect("the signer's own carrier session is admitted");
assert_eq!(inbox.nonce_cache().len(), 1);
assert_eq!(
inbox
.sequence_tracker()
.last_seen(&original_signer.fingerprint()),
Some(1)
);
recipient_transport.close().await;
}
#[tokio::test]
async fn request_reply_roundtrip_over_in_memory_transport() {
let user = UserKey::generate();
let alice = Arc::new(agent(&user, "alice"));
let bob = Arc::new(agent(&user, "bob"));
let bob_fp = bob.fingerprint();
let net = MeshNet::new();
let alice_bus =
Bus::bind_with_transport(alice.clone(), Arc::new(net.transport_for(&alice)))
.expect("bind Alice to in-memory transport");
let bob_bus = Bus::bind_with_transport(bob.clone(), Arc::new(net.transport_for(&bob)))
.expect("bind Bob to in-memory transport");
let topic = Topic::new(user.fingerprint(), "echo");
bob_bus.handle_requests(topic.clone(), |body| async move {
Ok(format!("echo: {}", String::from_utf8_lossy(&body)).into_bytes())
});
let reply = alice_bus
.request(bob_fp, &topic, b"hi".to_vec(), Duration::from_secs(5))
.await
.expect("round-trip reply");
assert_eq!(reply, b"echo: hi");
alice_bus.close().await.unwrap();
bob_bus.close().await.unwrap();
}
#[tokio::test]
async fn a_rebound_sender_is_admitted_by_a_receiver_that_kept_running() {
let user = UserKey::generate();
let alice = Arc::new(agent(&user, "alice"));
let bob = Arc::new(agent(&user, "bob"));
let bob_fp = bob.fingerprint();
let net = MeshNet::new();
let bob_bus = Bus::bind_with_transport(bob.clone(), Arc::new(net.transport_for(&bob)))
.expect("bind Bob to in-memory transport");
let topic = Topic::new(user.fingerprint(), "echo");
bob_bus.handle_requests(topic.clone(), |body| async move { Ok(body) });
let kept = Arc::new(crate::sequence::tests::Kept::default());
let bind_alice = || {
Bus::bind_with_transport_reserving(
alice.clone(),
Arc::new(net.transport_for(&alice)),
kept.clone(),
)
.expect("bind Alice to in-memory transport")
};
let first = bind_alice();
for _ in 0..3 {
first
.request(bob_fp, &topic, b"hi".to_vec(), Duration::from_secs(5))
.await
.expect("the first incarnation is admitted");
}
first.close().await.unwrap();
let second = bind_alice();
let reply = second
.request(bob_fp, &topic, b"again".to_vec(), Duration::from_secs(2))
.await
.expect("the re-bound sender is admitted");
assert_eq!(reply, b"again");
second.close().await.unwrap();
bob_bus.close().await.unwrap();
}
#[tokio::test]
async fn a_predecessors_envelope_is_refused_after_its_successor_is_admitted() {
let user = UserKey::generate();
let alice = agent(&user, "alice");
let bob_fp = agent(&user, "bob").fingerprint();
let kept = Arc::new(crate::sequence::tests::Kept::default());
let note = |sequence: &Sequencer| {
let msg = BusMessage::Publish {
topic: Topic::new(user.fingerprint(), "note").wire(),
body: b"note".to_vec(),
};
make_envelope(&alice, sequence, bob_fp, msg).expect("sign")
};
let captured = note(&Sequencer::reserving(kept.clone()).unwrap());
let successor = note(&Sequencer::reserving(kept).unwrap());
let inbox = Inbox::new();
let carrier = || DeliveryProvenance::Direct {
carrier: AuthenticatedPeer::new(user.fingerprint(), alice.fingerprint()),
};
let admit = |env| inbox.on_envelope(env, carrier(), user.fingerprint(), bob_fp);
admit(captured.clone())
.await
.expect("the predecessor is admitted");
admit(successor).await.expect("its successor is admitted");
assert!(
admit(captured).await.is_err(),
"the predecessor's envelope is refused on replay"
);
}
#[tokio::test]
async fn resolver_request_binds_reply_to_resolved_peer() {
wrong_signer_cannot_win_for_request_route(false).await;
}
#[tokio::test]
async fn direct_request_binds_reply_to_endpoint_peer() {
wrong_signer_cannot_win_for_request_route(true).await;
}
#[tokio::test]
async fn timed_out_request_removes_waiter_and_drops_late_reply() {
let user = UserKey::generate();
let asker = Arc::new(agent(&user, "asker"));
let expected = agent(&user, "expected");
let asker_fp = asker.fingerprint();
let expected_fp = expected.fingerprint();
let transport = CaptureTransport::new();
let bus = Bus::bind_with_transport(asker, Arc::clone(&transport) as Arc<dyn Transport>)
.expect("bind asker to capture transport");
let topic = Topic::new(user.fingerprint(), "timeout");
let result = bus
.request(
expected_fp,
&topic,
b"request".to_vec(),
Duration::from_millis(1),
)
.await;
assert!(matches!(result, Err(BusError::Timeout(_))));
assert_eq!(bus.inbox.pending_replies(), 0);
let outbound = match transport.next_sent().await {
CapturedSend::Resolve(fp, env) => {
assert_eq!(fp, expected_fp);
env
}
CapturedSend::Direct(_, _) => panic!("request used the wrong route"),
};
let correlation = match serde_json::from_slice::<BusMessage>(outbound.payload.as_ref())
.expect("decode captured request")
{
BusMessage::Request { correlation, .. } => CorrelationId(correlation),
other => panic!("expected Request, got {other:?}"),
};
bus.inbox
.on_envelope(
reply_envelope(&expected, asker_fp, correlation, b"late"),
direct_provenance(&expected),
user.fingerprint(),
asker_fp,
)
.await
.unwrap();
assert_eq!(
bus.inbox.pending_replies(),
0,
"late reply must not recreate a timed-out waiter"
);
bus.close().await.unwrap();
}
async fn cancelled_request_removes_its_waiter(transport: Arc<CaptureTransport>) {
let user = UserKey::generate();
let asker = Arc::new(agent(&user, "asker"));
let expected = agent(&user, "expected");
let bus = Bus::bind_with_transport(asker, Arc::clone(&transport) as Arc<dyn Transport>)
.expect("bind asker to capture transport");
let topic = Topic::new(user.fingerprint(), "cancelled");
let mut request = Box::pin(bus.request(
expected.fingerprint(),
&topic,
b"request".to_vec(),
Duration::from_secs(60),
));
let outbound = tokio::select! {
biased;
result = &mut request => panic!("request completed: {result:?}"),
CapturedSend::Resolve(_, env) = transport.next_sent() => env,
};
assert_eq!(bus.inbox.pending_replies(), 1, "waiter registered");
drop(request);
assert_eq!(
bus.inbox.pending_replies(),
0,
"cancellation removes the waiter"
);
let BusMessage::Request { correlation, .. } =
serde_json::from_slice(outbound.payload.as_ref()).expect("decode captured request")
else {
panic!("expected Request");
};
assert_eq!(
bus.inbox
.waiters
.deliver(CorrelationId(correlation), expected.fingerprint(), vec![]),
ReplyDelivery::Unknown,
"a late reply finds no waiter"
);
assert_eq!(bus.inbox.pending_replies(), 0);
bus.close().await.unwrap();
}
#[tokio::test]
async fn request_cancelled_while_awaiting_its_reply_removes_its_waiter() {
cancelled_request_removes_its_waiter(CaptureTransport::new()).await;
}
#[tokio::test]
async fn request_cancelled_during_its_send_removes_its_waiter() {
cancelled_request_removes_its_waiter(CaptureTransport::stalling()).await;
}
#[tokio::test]
async fn context_handler_sees_the_calling_agent_over_the_transport() {
let user = UserKey::generate();
let alice = Arc::new(agent(&user, "alice"));
let bob = Arc::new(agent(&user, "bob"));
let alice_fp = alice.fingerprint();
let bob_fp = bob.fingerprint();
let net = MeshNet::new();
let alice_bus =
Bus::bind_with_transport(alice.clone(), Arc::new(net.transport_for(&alice)))
.expect("bind Alice to in-memory transport");
let bob_bus = Bus::bind_with_transport(bob.clone(), Arc::new(net.transport_for(&bob)))
.expect("bind Bob to in-memory transport");
let topic = Topic::new(user.fingerprint(), "whoami");
bob_bus.handle_requests_with_context(
topic.clone(),
|ctx: RequestContext, _body| async move {
Ok(ctx.caller_agent_fp.hex().into_bytes())
},
);
let reply = alice_bus
.request(bob_fp, &topic, b"".to_vec(), Duration::from_secs(5))
.await
.expect("round-trip reply");
assert_eq!(
String::from_utf8(reply).unwrap(),
alice_fp.hex(),
"the responder must see ALICE as the caller"
);
alice_bus.close().await.unwrap();
bob_bus.close().await.unwrap();
}
#[tokio::test]
async fn handle_requests_registers_synchronously_no_spawn_race() {
let user = UserKey::generate();
let bob = Arc::new(agent(&user, "bob"));
let net = MeshNet::new();
let bob_bus = Bus::bind_with_transport(bob.clone(), Arc::new(net.transport_for(&bob)))
.expect("bind Bob to in-memory transport");
let topic = Topic::new(user.fingerprint(), "echo");
assert_eq!(
bob_bus.inbox.handler_count(),
0,
"no handler registered before handle_requests"
);
bob_bus.handle_requests(topic, |body| async move { Ok(body) });
assert_eq!(
bob_bus.inbox.handler_count(),
1,
"handle_requests must register the handler before returning"
);
bob_bus.close().await.unwrap();
}
#[tokio::test]
async fn in_memory_send_to_unknown_peer_is_unreachable() {
let user = UserKey::generate();
let alice = Arc::new(agent(&user, "alice"));
let net = MeshNet::new();
let alice_bus =
Bus::bind_with_transport(alice.clone(), Arc::new(net.transport_for(&alice)))
.expect("bind Alice to in-memory transport");
let topic = Topic::new(user.fingerprint(), "echo");
let phantom = Fingerprint([0xfeu8; 32]);
match alice_bus
.request(phantom, &topic, b"x".to_vec(), Duration::from_millis(200))
.await
{
Err(BusError::Unreachable(_)) => {}
other => panic!("expected Unreachable, got {other:?}"),
}
alice_bus.close().await.unwrap();
}
#[tokio::test(flavor = "multi_thread")]
async fn bind_exposes_local_port_and_fingerprints() {
let user = UserKey::generate();
let a = agent(&user, "worker");
let a_fp = a.fingerprint();
let bus = Bus::bind(&user, a, 0, RevocationGuard::in_memory(&user))
.await
.expect("bind");
assert!(bus.local_port() > 0);
assert_eq!(bus.user_fingerprint(), user.fingerprint());
assert_eq!(bus.agent_fingerprint(), a_fp);
bus.close().await.expect("close");
}
#[tokio::test(flavor = "multi_thread")]
async fn request_to_unknown_peer_errors_unreachable() {
let user = UserKey::generate();
let a = agent(&user, "alice");
let bus = Bus::bind(&user, a, 0, RevocationGuard::in_memory(&user))
.await
.expect("bind");
let topic = Topic::new(user.fingerprint(), "echo");
let phantom_fp = Fingerprint([0xfeu8; 32]);
let res = bus
.request(
phantom_fp,
&topic,
b"x".to_vec(),
Duration::from_millis(200),
)
.await;
match res {
Err(BusError::Unreachable(_)) => {}
other => panic!("expected Unreachable, got {other:?}"),
}
bus.close().await.unwrap();
}
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV6};
fn lo_v4(port: u16) -> SocketAddr {
SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port)
}
fn lo_v6(port: u16) -> SocketAddr {
SocketAddr::new(Ipv6Addr::LOCALHOST.into(), port)
}
#[test]
fn scopeless_link_local_source_dropped_loopback_survives() {
let src: SocketAddr = "[fe80::1]:9000".parse().unwrap(); let cands = reply_dial_candidates(src);
assert!(!cands.is_empty(), "identity must retain a usable path");
assert!(
!cands.contains(&src),
"scopeless link-local is undialable; must be filtered out"
);
assert!(
cands.contains(&lo_v4(9000)),
"same-host v4 loopback must be a candidate"
);
assert!(
cands.contains(&lo_v6(9000)),
"same-host v6 loopback must be a candidate"
);
}
#[test]
fn scoped_link_local_source_is_kept() {
let ip: Ipv6Addr = "fe80::1".parse().unwrap();
let src = SocketAddr::V6(SocketAddrV6::new(ip, 9000, 0, 3)); let cands = reply_dial_candidates(src);
assert!(
cands.contains(&src),
"a scoped link-local is dialable; keep it"
);
}
#[test]
fn routable_source_yields_source_plus_loopback() {
let src: SocketAddr = "192.0.2.50:9000".parse().unwrap(); let cands = reply_dial_candidates(src);
assert!(cands.contains(&src));
assert!(cands.contains(&lo_v4(9000)));
assert!(cands.contains(&lo_v6(9000)));
assert!(cands.len() >= 2, "never a single load-bearing address");
}
#[test]
fn loopback_source_not_duplicated() {
let src = lo_v4(9000);
let cands = reply_dial_candidates(src);
assert_eq!(
cands.iter().filter(|a| **a == src).count(),
1,
"no duplicate loopback candidate"
);
}
#[test]
fn scopeless_link_local_predicate_boundaries() {
assert!(is_scopeless_link_local(&"[fe80::abcd]:1".parse().unwrap()));
assert!(is_scopeless_link_local(&"[febf::1]:1".parse().unwrap())); let scoped = SocketAddr::V6(SocketAddrV6::new("fe80::1".parse().unwrap(), 1, 0, 2));
assert!(!is_scopeless_link_local(&scoped));
assert!(!is_scopeless_link_local(&"[::1]:1".parse().unwrap()));
assert!(!is_scopeless_link_local(
&"[2001:db8::1]:1".parse().unwrap()
));
assert!(!is_scopeless_link_local(&"127.0.0.1:1".parse().unwrap()));
assert!(!is_scopeless_link_local(&"[fec0::1]:1".parse().unwrap())); }
async fn observe_and_dial_back(target_ip: IpAddr) -> (SocketAddr, bool) {
let user = UserKey::generate();
let asker = agent(&user, "probe-asker");
let responder = agent(&user, "probe-responder");
let a_ep = Arc::new(Endpoint::bind(&asker, 0).await.expect("bind asker"));
let b_ep = Arc::new(Endpoint::bind(&responder, 0).await.expect("bind responder"));
let a_pubkey = a_ep.public_key();
let b_pubkey = b_ep.public_key();
let target = SocketAddr::new(target_ip, b_ep.port());
let a_accept = a_ep.clone();
let a_task = tokio::spawn(async move {
while let Some(incoming) = a_accept.accept().await {
tokio::spawn(async move {
let _ = incoming.await;
});
}
});
let (tx, rx) = tokio::sync::oneshot::channel::<Option<SocketAddr>>();
let b_accept = b_ep.clone();
let b_task = tokio::spawn(async move {
if let Some(incoming) = b_accept.accept().await {
let src = match incoming.remote_addr() {
IncomingAddr::Ip(addr) => Some(addr),
_ => None,
};
let _ = tx.send(src);
let _ = incoming.await;
}
});
a_ep.dial(b_pubkey, [target])
.await
.expect("asker dials responder over the chosen local path");
let observed = tokio::time::timeout(Duration::from_secs(5), rx)
.await
.expect("responder observed a source within 5s")
.expect("observation channel not dropped")
.expect("iroh reported an IP source (not a relay path)");
let candidates = reply_dial_candidates(observed);
assert!(
!candidates.is_empty(),
"candidate set must never be empty for a same-host peer"
);
let dial_back_ok = b_ep
.dial(a_pubkey, candidates.iter().copied())
.await
.is_ok();
b_task.abort();
a_task.abort();
(observed, dial_back_ok)
}
#[tokio::test(flavor = "multi_thread")]
async fn probe_loopback_v4_source_is_dialable_and_reconnects() {
let _ = tracing_subscriber::fmt().with_test_writer().try_init();
let (observed, dial_back_ok) = observe_and_dial_back(Ipv4Addr::LOCALHOST.into()).await;
eprintln!("[#61 probe] loopback-v4 dial → iroh observed source = {observed}");
assert!(
!is_scopeless_link_local(&observed),
"a same-host loopback source must not be an undialable scopeless link-local"
);
assert!(
dial_back_ok,
"responder must reconnect to the asker over reply_dial_candidates({observed})"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn probe_loopback_v6_source_is_dialable_and_reconnects() {
let _ = tracing_subscriber::fmt().with_test_writer().try_init();
let (observed, dial_back_ok) = observe_and_dial_back(Ipv6Addr::LOCALHOST.into()).await;
eprintln!("[#61 probe] loopback-v6 dial → iroh observed source = {observed}");
assert!(
!is_scopeless_link_local(&observed),
"a same-host loopback source must not be an undialable scopeless link-local"
);
assert!(
dial_back_ok,
"responder must reconnect to the asker over reply_dial_candidates({observed})"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn probe_scopeless_link_local_source_reconnects_via_loopback_fallback() {
let _ = tracing_subscriber::fmt().with_test_writer().try_init();
let user = UserKey::generate();
let asker = agent(&user, "probe-asker");
let responder = agent(&user, "probe-responder");
let a_ep = Arc::new(Endpoint::bind(&asker, 0).await.expect("bind asker"));
let b_ep = Arc::new(Endpoint::bind(&responder, 0).await.expect("bind responder"));
let a_accept = a_ep.clone();
let a_task = tokio::spawn(async move {
while let Some(incoming) = a_accept.accept().await {
tokio::spawn(async move {
let _ = incoming.await;
});
}
});
let scopeless_source = SocketAddr::V6(SocketAddrV6::new(
"fe80::dead:beef".parse().unwrap(),
a_ep.port(),
0,
0, ));
assert!(
is_scopeless_link_local(&scopeless_source),
"precondition: the source is the undialable #61 shape"
);
let candidates = reply_dial_candidates(scopeless_source);
assert!(
!candidates.contains(&scopeless_source),
"the undialable scopeless link-local must be filtered out of the set"
);
assert!(
!candidates.is_empty(),
"the identity must retain a reachable candidate (loopback)"
);
let dial_back_ok = b_ep
.dial(a_ep.public_key(), candidates.iter().copied())
.await
.is_ok();
a_task.abort();
assert!(
dial_back_ok,
"responder must reach the same-host asker via loopback despite an \
undialable scopeless link-local source ({scopeless_source})"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_dialed_connection_admits_one_envelope_back_per_request() {
let user = UserKey::generate();
let host_agent = Arc::new(agent(&user, "host"));
let host_fp = host_agent.fingerprint();
let host = IrohTransport::bind_posture(
user.fingerprint(),
host_agent.clone(),
0,
Posture::OutboundOnly,
RevocationGuard::in_memory(&user),
)
.await
.expect("bind host");
let hub_agent = Arc::new(agent(&user, "hub"));
let hub = IrohTransport::bind(
user.fingerprint(),
hub_agent.clone(),
0,
BusOptions { announce: false },
RevocationGuard::in_memory(&user),
)
.await
.expect("bind hub");
let hub_ep = PeerEndpoint::new(
hub_agent.public_bytes(),
SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.into(), hub.local_port()),
);
let note = |from: &AgentKey, to: Fingerprint, seq: u64, body: &[u8]| {
let msg = BusMessage::Publish {
topic: Topic::new(user.fingerprint(), "note").wire(),
body: body.to_vec(),
};
make_envelope(from, &Sequencer::after(seq - 1), to, msg).expect("sign")
};
let bounded = Duration::from_secs(5);
host.send_request_to_endpoint(
&hub_ep,
note(&host_agent, hub_ep.fingerprint(), 1, b"poll"),
Duration::from_secs(60),
)
.await
.expect("host sends its request");
let polled = tokio::time::timeout(bounded, hub.recv())
.await
.expect("hub receives within 5s")
.expect("hub transport open");
let conn = polled
.reply_route
.downcast_ref::<IrohReplyRoute>()
.expect("an iroh reply route")
.conn
.clone();
send_env_on_conn(
&conn,
hub_agent.cert(),
&RevocationGuard::in_memory(&user),
host_fp,
¬e(&hub_agent, host_fp, 1, b"first"),
)
.await
.expect("the first envelope back is read");
let first = tokio::time::timeout(bounded, host.recv())
.await
.expect("host admits the first envelope within 5s")
.expect("host transport open");
assert_eq!(first.envelope.sender_agent_fp(), hub_agent.fingerprint());
assert!(conn.close_reason().is_none(), "connection still open");
let second = tokio::time::timeout(
HANDSHAKE_TIMEOUT + bounded,
send_env_on_conn(
&conn,
hub_agent.cert(),
&RevocationGuard::in_memory(&user),
host_fp,
¬e(&hub_agent, host_fp, 2, b"second"),
),
)
.await
.expect("the send resolves within its own handshake deadline");
assert!(
matches!(second, Err(BusError::Timeout(d)) if d == HANDSHAKE_TIMEOUT),
"the host must not read a second envelope; got {second:?}"
);
assert!(
conn.close_reason().is_none(),
"still open: the refusal is the cap"
);
assert!(
tokio::time::timeout(Duration::from_millis(500), host.recv())
.await
.is_err(),
"nothing further reaches the host's inbound queue"
);
drop(first);
}
}