Skip to main content

arachne_node/
lib.rs

1//! Direct-peer live pub/sub assembled over the portable routing module.
2//!
3//! Experimental: callers supply VERIFIED policies and peer address hints. No
4//! invitation verification, group encryption, retained delivery or identity
5//! storage yet. This module must not be presented as secure group management.
6//! Traffic is direct QUIC/TLS, bound to the authenticated peer's identity; no
7//! forwarding through other endpoints. Only locally installed policies authorize.
8use std::{
9    collections::{BTreeMap, VecDeque},
10    net::SocketAddr,
11    sync::{
12        Arc, Mutex as StdMutex,
13        atomic::{AtomicBool, Ordering},
14    },
15    time::Duration,
16};
17
18mod budget;
19mod connections;
20mod control;
21mod overlay;
22pub mod resources;
23mod wire;
24pub use budget::{CapacityCounts, ConnectionBudget};
25pub use control::{ControlClient, ControlRequest, ControlTiming, InquiryResponder, MAX_CONTROL_REPLY, Timing};
26
27use connections::Connections;
28use arachne_routing::RoutingTable;
29pub use arachne_routing::{PeerId, Permissions, Topic, WorkspaceId};
30use futures_util::{StreamExt, stream};
31use serde::{Deserialize, Serialize};
32use tokio::{
33    sync::{Mutex, mpsc, watch},
34    task::JoinHandle,
35};
36
37const ALPN: &[u8] = b"data-fabric/pubsub-experiment/1";
38const MAX_PAYLOAD: usize = 16 * 1024;
39const MAX_RECIPIENTS: usize = 64;
40const MAX_FRAME: usize = 128 * 1024;
41const TIMEOUT: Duration = Duration::from_secs(5);
42const CONNECTION_IDLE: Duration = Duration::from_secs(60);
43const CRITICAL_QUEUE: usize = 256;
44const CURRENT_QUEUE: usize = 64;
45const BULK_QUEUE: usize = 64;
46
47/// Address discovery and path policy; never workspace authority.
48#[derive(Clone, Copy, Debug)]
49pub enum NetworkProfile {
50    Direct,
51    Lan,
52    Nearby,
53    Wan,
54    RelayOnly,
55    WanOnly,
56}
57
58impl NetworkProfile {
59    fn settings(self) -> (Option<&'static str>, bool, bool, bool) {
60        match self {
61            Self::Direct => (None, false, false, true),
62            Self::Lan => (Some("data-fabric"), false, false, true),
63            Self::Nearby => (Some("arachne-nearby"), false, false, true),
64            Self::Wan => (Some("data-fabric"), true, false, false),
65            Self::RelayOnly => (None, true, true, false),
66            Self::WanOnly => (None, true, false, false),
67        }
68    }
69}
70
71/// Caller-supplied relay transport settings for a controlled qualification or
72/// an operator-managed relay deployment.
73#[derive(Clone, Debug)]
74pub struct RelayOptions {
75    pub(crate) map: iroh::RelayMap,
76    pub(crate) tls: iroh::tls::CaTlsConfig,
77}
78
79impl RelayOptions {
80    pub fn new(map: iroh::RelayMap, tls: iroh::tls::CaTlsConfig) -> Self {
81        Self { map, tls }
82    }
83}
84
85#[derive(Debug, thiserror::Error)]
86pub enum Error {
87    #[error("routing rejected: {0:?}")]
88    Routing(#[from] arachne_routing::Error),
89    #[error("transport: {0}")]
90    Transport(String),
91    #[error("unknown peer address")]
92    MissingPeer,
93    #[error("control request not sent: {0}")]
94    ControlNotSent(&'static str),
95    #[error("message exceeds limit")]
96    TooLarge,
97    #[error("peer rejected operation")]
98    Rejected,
99    #[error("operation deadline exceeded during {0}; admission outcome may be unknown")]
100    Timeout(&'static str),
101    #[error("operation cancelled because the local session closed")]
102    Cancelled,
103    #[error("invalid frame")]
104    InvalidFrame,
105    #[error("consumer queue full or closed")]
106    Backpressure,
107    #[error("peer has no announced subscription to this topic")]
108    NotSubscribed,
109}
110type Result<T> = std::result::Result<T, Error>;
111
112#[derive(Debug, Clone, PartialEq, Eq)]
113pub struct Message {
114    pub workspace: WorkspaceId,
115    pub revision: u64,
116    pub sender: PeerId,
117    /// Authenticated endpoint that handed this frame to us. Equal to `sender`
118    /// for direct traffic; forwarding code must never treat it as the author.
119    pub received_from: PeerId,
120    pub topic: Topic,
121    pub payload: Vec<u8>,
122    /// Nonempty only for a direct publication. These are workspace member IDs;
123    /// endpoint selection is authenticated transport metadata, not application identity.
124    pub recipients: Vec<[u8; 32]>,
125    pub delivery: DeliveryClass,
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
129#[serde(rename_all = "snake_case")]
130/// Payload-neutral queue behavior declared by the publishing adapter.
131pub enum DeliveryClass {
132    #[default]
133    Critical,
134    Current {
135        replacement_key: [u8; 32],
136    },
137    Bulk,
138}
139
140type CurrentKey = (
141    WorkspaceId,
142    u64,
143    String,
144    PeerId,
145    PeerId,
146    [u8; 32],
147    Vec<[u8; 32]>,
148);
149
150#[derive(Default)]
151struct DeliveryState {
152    critical: VecDeque<Message>,
153    current: BTreeMap<CurrentKey, Message>,
154    bulk: VecDeque<Message>,
155    critical_run: u8,
156    bulk_turn: bool,
157}
158
159#[derive(Clone, Default)]
160struct DeliveryQueue {
161    state: Arc<StdMutex<DeliveryState>>,
162    changed: Arc<tokio::sync::Notify>,
163    closed: Arc<AtomicBool>,
164}
165
166impl DeliveryQueue {
167    fn push(&self, message: Message) -> Result<()> {
168        if self.closed.load(Ordering::Acquire) {
169            return Err(Error::Backpressure);
170        }
171        let mut state = self.state.lock().unwrap();
172        match message.delivery {
173            DeliveryClass::Critical if state.critical.len() < CRITICAL_QUEUE => {
174                state.critical.push_back(message)
175            }
176            DeliveryClass::Current { replacement_key } => {
177                let key = (
178                    message.workspace,
179                    message.revision,
180                    message.topic.as_str().into(),
181                    message.sender,
182                    message.received_from,
183                    replacement_key,
184                    message.recipients.clone(),
185                );
186                if state.current.len() == CURRENT_QUEUE && !state.current.contains_key(&key) {
187                    return Err(Error::Backpressure);
188                }
189                state.current.insert(key, message);
190            }
191            DeliveryClass::Bulk if state.bulk.len() < BULK_QUEUE => state.bulk.push_back(message),
192            _ => return Err(Error::Backpressure),
193        }
194        drop(state);
195        self.changed.notify_one();
196        Ok(())
197    }
198
199    fn pop(&self) -> Option<Message> {
200        let mut state = self.state.lock().unwrap();
201        if state.critical_run < 8
202            && let Some(message) = state.critical.pop_front()
203        {
204            state.critical_run += 1;
205            return Some(message);
206        }
207        let lower = if state.bulk_turn {
208            state
209                .bulk
210                .pop_front()
211                .or_else(|| state.current.pop_first().map(|(_, message)| message))
212        } else {
213            state
214                .current
215                .pop_first()
216                .map(|(_, message)| message)
217                .or_else(|| state.bulk.pop_front())
218        };
219        if lower.is_some() {
220            state.critical_run = 0;
221            state.bulk_turn = !state.bulk_turn;
222            return lower;
223        }
224        state.critical.pop_front()
225    }
226
227    fn close(&self) {
228        self.closed.store(true, Ordering::Release);
229        self.changed.notify_waiters();
230    }
231}
232
233pub struct MessageReceiver {
234    queue: DeliveryQueue,
235}
236
237impl MessageReceiver {
238    /// Snapshot of all delivery classes, without consuming the next message.
239    pub fn is_empty(&self) -> bool {
240        let state = self.queue.state.lock().unwrap();
241        state.critical.is_empty() && state.current.is_empty() && state.bulk.is_empty()
242    }
243
244    pub fn try_recv(&mut self) -> std::result::Result<Message, mpsc::error::TryRecvError> {
245        self.queue.pop().ok_or_else(|| {
246            if self.queue.closed.load(Ordering::Acquire) {
247                mpsc::error::TryRecvError::Disconnected
248            } else {
249                mpsc::error::TryRecvError::Empty
250            }
251        })
252    }
253
254    pub async fn recv(&mut self) -> Option<Message> {
255        loop {
256            let notify = self.queue.changed.clone();
257            let changed = notify.notified();
258            match self.try_recv() {
259                Ok(message) => return Some(message),
260                Err(mpsc::error::TryRecvError::Disconnected) => return None,
261                Err(mpsc::error::TryRecvError::Empty) => changed.await,
262            }
263        }
264    }
265}
266
267/// Acceptance of an operation. Publications enter consumer queues; this is not
268/// a delivery/read receipt. Subscription operations update interest. A timeout
269/// may have occurred after admission; retry semantics are not implemented yet.
270#[derive(Debug, Default)]
271pub struct AdmissionReport {
272    pub admitted: Vec<PeerId>,
273    pub failed: Vec<(PeerId, Error)>,
274    /// A bounded live overlay accepted this publication locally. This is not a
275    /// remote admission, delivery, retention or read receipt.
276    pub queued: bool,
277}
278
279#[derive(Serialize, Deserialize)]
280#[serde(deny_unknown_fields)]
281struct Frame {
282    workspace: WorkspaceId,
283    revision: u64,
284    #[serde(deserialize_with = "wire::topic")]
285    topic: String,
286    delivery: DeliveryClass,
287    operation: Operation,
288}
289
290#[derive(Serialize, Deserialize)]
291enum Operation {
292    Subscribe,
293    Unsubscribe,
294    Publish(#[serde(with = "wire::payload")] Vec<u8>),
295    DirectPublish {
296        #[serde(with = "wire::payload")]
297        payload: Vec<u8>,
298        #[serde(deserialize_with = "wire::recipients")]
299        recipients: Vec<[u8; 32]>,
300    },
301}
302
303/// Local endpoint counters. These include transport overhead and retransmission,
304/// not application delivery receipts. They reset when the endpoint is opened.
305#[derive(Debug, serde::Serialize)]
306pub struct TransportMetrics {
307    pub received_bytes: u64,
308    pub sent_bytes: u64,
309    pub receive_queue: usize,
310    pub paths: Vec<PeerPath>,
311    pub paths_limited: bool,
312}
313
314#[derive(Debug, serde::Serialize)]
315pub struct PeerPath {
316    pub endpoint: PeerId,
317    pub route: &'static str,
318    pub rtt_ms: u64,
319}
320
321pub struct Node {
322    controls: mpsc::Receiver<ControlRequest>,
323    control_inbox: control::ControlInbox,
324    control_signal: Arc<tokio::sync::Notify>,
325    control_cancel: watch::Sender<bool>,
326    deferred_controls: std::collections::VecDeque<ControlRequest>,
327    connections: Connections,
328    routing: Arc<Mutex<RoutingTable>>,
329    resources: resources::ResourceTransfers,
330    events: DeliveryQueue,
331    overlays: Arc<Mutex<BTreeMap<WorkspaceId, Arc<overlay::Overlay>>>>,
332    membership: overlay::MembershipInbox,
333    listener: JoinHandle<()>,
334}
335
336impl Node {
337    pub fn transport_metrics(&self) -> TransportMetrics {
338        let mut metrics = self.connections.metrics();
339        let queue = self.events.state.lock().unwrap();
340        metrics.receive_queue = queue.critical.len() + queue.current.len() + queue.bulk.len();
341        metrics
342    }
343
344    /// Bind a directly reachable local address. Public discovery, relays and
345    /// port mapping are disabled. Fresh identity on every bind in this version.
346    pub async fn bind(address: SocketAddr) -> Result<(Self, MessageReceiver)> {
347        Self::bind_with_profile(
348            address,
349            None,
350            NetworkProfile::Direct,
351            ConnectionBudget::default(),
352        )
353        .await
354    }
355
356    /// Bind using a caller-owned, securely generated and stored 32-byte endpoint
357    /// credential. This restores transport identity only, never workspace authority.
358    /// The caller must prevent concurrent endpoint instances using this credential.
359    pub async fn bind_with_identity(
360        address: SocketAddr,
361        secret: &[u8; 32],
362    ) -> Result<(Self, MessageReceiver)> {
363        Self::bind_with_profile(
364            address,
365            Some(secret),
366            NetworkProfile::Direct,
367            ConnectionBudget::default(),
368        )
369        .await
370    }
371
372    /// Opt in to local-network address advertisement/lookup. This publishes the
373    /// endpoint key and addresses, never workspace names or membership authority.
374    /// Public lookup services and relay transports remain disabled.
375    pub async fn bind_lan(address: SocketAddr) -> Result<(Self, MessageReceiver)> {
376        Self::bind_with_profile(
377            address,
378            None,
379            NetworkProfile::Lan,
380            ConnectionBudget::default(),
381        )
382        .await
383    }
384
385    /// Opt in to LAN lookup while retaining a workspace-scoped endpoint identity.
386    /// Advertises this endpoint key and addresses, never membership authority.
387    pub async fn bind_lan_with_identity(
388        address: SocketAddr,
389        secret: &[u8; 32],
390    ) -> Result<(Self, MessageReceiver)> {
391        Self::bind_with_profile(
392            address,
393            Some(secret),
394            NetworkProfile::Lan,
395            ConnectionBudget::default(),
396        )
397        .await
398    }
399
400    /// Local device-level invitation discovery, isolated from workspace endpoint discovery.
401    pub async fn bind_nearby_with_identity(
402        address: SocketAddr,
403        secret: &[u8; 32],
404    ) -> Result<(Self, MessageReceiver)> {
405        Self::bind_with_profile(
406            address,
407            Some(secret),
408            NetworkProfile::Nearby,
409            ConnectionBudget::default(),
410        )
411        .await
412    }
413
414    /// Opt in to Iroh's public Pkarr address lookup and relay network while
415    /// retaining LAN lookup. These services provide routes, never membership.
416    pub async fn bind_wan_with_identity(
417        address: SocketAddr,
418        secret: &[u8; 32],
419    ) -> Result<(Self, MessageReceiver)> {
420        Self::bind_with_profile(
421            address,
422            Some(secret),
423            NetworkProfile::Wan,
424            ConnectionBudget::default(),
425        )
426        .await
427    }
428
429    /// Diagnostic WAN profile that forces authenticated traffic through Iroh
430    /// relays while leaving the device's Wi-Fi connection intact.
431    pub async fn bind_relay_with_identity(
432        address: SocketAddr,
433        secret: &[u8; 32],
434    ) -> Result<(Self, MessageReceiver)> {
435        Self::bind_with_profile(
436            address,
437            Some(secret),
438            NetworkProfile::RelayOnly,
439            ConnectionBudget::default(),
440        )
441        .await
442    }
443
444    /// Diagnostic WAN profile that uses only public endpoint lookup and ignores
445    /// LAN discovery and saved IP hints while retaining direct Iroh paths.
446    pub async fn bind_wan_only_with_identity(
447        address: SocketAddr,
448        secret: &[u8; 32],
449    ) -> Result<(Self, MessageReceiver)> {
450        Self::bind_with_profile(
451            address,
452            Some(secret),
453            NetworkProfile::WanOnly,
454            ConnectionBudget::default(),
455        )
456        .await
457    }
458
459    /// Bind an independently identified endpoint using host-owned shared device
460    /// capacity. Clone the same budget for each workspace and discovery endpoint.
461    pub async fn bind_with_profile(
462        address: SocketAddr,
463        secret: Option<&[u8; 32]>,
464        profile: NetworkProfile,
465        budget: ConnectionBudget,
466    ) -> Result<(Self, MessageReceiver)> {
467        Self::bind_with_profile_and_relays(address, secret, profile, budget, None).await
468    }
469
470    /// Bind with a caller-supplied relay map and TLS trust configuration.
471    ///
472    /// This is intended for controlled relay qualification and operator-managed
473    /// relay deployments. The caller must select `NetworkProfile::RelayOnly`
474    /// when direct paths must be excluded.
475    pub async fn bind_with_profile_and_relay(
476        address: SocketAddr,
477        secret: Option<&[u8; 32]>,
478        profile: NetworkProfile,
479        budget: ConnectionBudget,
480        relay: RelayOptions,
481    ) -> Result<(Self, MessageReceiver)> {
482        Self::bind_with_profile_and_relays(address, secret, profile, budget, Some(relay)).await
483    }
484
485    async fn bind_with_profile_and_relays(
486        address: SocketAddr,
487        secret: Option<&[u8; 32]>,
488        profile: NetworkProfile,
489        budget: ConnectionBudget,
490        relay: Option<RelayOptions>,
491    ) -> Result<(Self, MessageReceiver)> {
492        let connections = Connections::bind(
493            address,
494            profile,
495            secret.map(iroh::SecretKey::from_bytes),
496            budget.clone(),
497            vec![ALPN.to_vec(), control::ALPN.to_vec()],
498            relay,
499        )
500        .await?;
501        let routing = Arc::new(Mutex::new(RoutingTable::default()));
502        let resources = resources::ResourceTransfers::new(connections.clone(), routing.clone());
503        let events = DeliveryQueue::default();
504        let receiver = MessageReceiver {
505            queue: events.clone(),
506        };
507        let (control_inbox, controls, control_signal) = control::ControlInbox::new(512);
508        let (control_cancel, _) = watch::channel(false);
509        let node_control_inbox = control_inbox.clone();
510        let overlays: Arc<Mutex<BTreeMap<WorkspaceId, Arc<overlay::Overlay>>>> =
511            Arc::new(Mutex::new(BTreeMap::new()));
512        let accepted_connections = connections.clone();
513        let shared = routing.clone();
514        let output = events.clone();
515        let accepted_overlays = overlays.clone();
516        let accepted_resources = resources.clone();
517        let listener = tokio::spawn(async move {
518            // Handshakes release their slot before application work. Slow data
519            // peers cannot occupy the slots needed to reach the control budget.
520            let handshake_capacity = budget.handshakes;
521            let [data_capacity, control_capacity] = budget.exchanges;
522            let mut workers = tokio::task::JoinSet::new();
523            while let Some(incoming) = accepted_connections.accept().await {
524                while workers.try_join_next().is_some() {}
525                let Ok(permit) = handshake_capacity.clone().try_acquire_owned() else {
526                    incoming.refuse();
527                    continue;
528                };
529                let control_inbox = control_inbox.clone();
530                let control_capacity = control_capacity.clone();
531                let data_capacity = data_capacity.clone();
532                let routing = shared.clone();
533                let overlays = accepted_overlays.clone();
534                let connections = accepted_connections.clone();
535                let output = output.clone();
536                let resources = accepted_resources.clone();
537                workers.spawn(async move {
538                    let mut stage = "accept connection";
539                    let mut remote = None;
540                    let mut connection_id = None;
541                    let mut observed_connection = None;
542                    let result = async {
543                        let observed_address = incoming.remote_addr();
544                        let connection = tokio::time::timeout(TIMEOUT, incoming).await
545                            .map_err(|_| Error::Timeout("accept connection"))?.map_err(transport)?;
546                        drop(permit);
547                        observed_connection = Some(connection.clone());
548                        let sender = *connection.remote_id().as_bytes();
549                        remote = Some(connection.remote_id());
550                        connection_id = Some(connection.stable_id());
551                        let is_control = connection.alpn() == control::ALPN;
552                        let overlay = {
553                            let overlays = overlays.lock().await;
554                            overlays
555                                .values()
556                                .find(|overlay| overlay.alpn.as_slice() == connection.alpn())
557                                .map(|overlay| {
558                                    (
559                                        overlay.workspace,
560                                        overlay.revision(),
561                                        overlay.gossip.clone(),
562                                    )
563                                })
564                        };
565                        if let Some((workspace, revision, gossip)) = overlay {
566                            routing
567                                .lock()
568                                .await
569                                .authorizes_endpoint(workspace, revision, sender)?;
570                            stage = "accept gossip";
571                            tokio::time::timeout(control::CONTROL_TIMEOUT, gossip.handle_connection(connection))
572                                .await.map_err(|_| Error::Timeout("accept gossip"))?.map_err(transport)?;
573                            return Ok(());
574                        }
575                        // A removed overlay's already-negotiated ALPN must never
576                        // be interpreted as the direct-frame schema.
577                        if !is_control && connection.alpn() != ALPN {
578                            return Err(Error::InvalidFrame);
579                        }
580                        // An idle exchange link may give way when the device is full.
581                        let connection_key = connection.stable_id();
582                        connections.mark_evictable(connection_key);
583                        let (capacity, budget) = if is_control {
584                            (control_capacity, control::CONTROL_TIMEOUT)
585                        } else {
586                            (data_capacity, TIMEOUT)
587                        };
588                        let address = match observed_address {
589                            iroh::endpoint::IncomingAddr::Ip(address) => Some(address),
590                            _ => None,
591                        };
592                        stage = "receive exchanges";
593                        tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id, paths = ?connection.paths(), "TRANSPORT_RECEIVE_CONNECTED");
594                        let connection = &connection;
595                        let connections = &connections;
596                        let routing = &routing;
597                        let output = &output;
598                        let control_inbox = &control_inbox;
599                        let resources = &resources;
600                        let mut exchanges = stream::FuturesUnordered::new();
601                        let idle = tokio::time::sleep(CONNECTION_IDLE);
602                        tokio::pin!(idle);
603                        loop {
604                            tokio::select! {
605                                streams = connection.accept_bi() => {
606                                    let Ok((mut send, mut recv)) = streams else { return Ok(()); };
607                                    let Ok(permit) = capacity.clone().try_acquire_owned() else {
608                                        let _ = send.reset(1u8.into());
609                                        let _ = recv.stop(1u8.into());
610                                        continue;
611                                    };
612                                    idle.as_mut().reset(tokio::time::Instant::now() + CONNECTION_IDLE);
613                                    let guard = connections.exchange(connection_key);
614                                    exchanges.push(async move {
615                                        let _permit = permit;
616                                        let _guard = guard;
617                                        let result = async {
618                                            if !is_control {
619                                                let mut kind = [0; 1];
620                                                tokio::time::timeout(TIMEOUT, recv.read_exact(&mut kind)).await
621                                                    .map_err(|_| Error::Timeout("stream kind"))?.map_err(transport)?;
622                                                match kind[0] {
623                                                    resources::STREAM_KIND => return resources.serve(connection, &mut send, &mut recv).await,
624                                                    0 => (),
625                                                    _ => return Err(Error::InvalidFrame),
626                                                }
627                                            }
628                                            tokio::time::timeout(budget, async {
629                                              if is_control {
630                                                control::receive(connection, control_inbox, address, (&mut send, &mut recv)).await
631                                              } else {
632                                                receive_frame(connection, connections, routing, output, address, (&mut send, &mut recv)).await
633                                              }
634                                            }).await.map_err(|_| Error::Timeout("receive exchange"))?
635                                        }.await;
636                                        // Dropping a QUIC SendStream finishes it. An
637                                        // abandoned/expired exchange must reset instead
638                                        // of looking like a successful empty reply.
639                                        if result.is_err() {
640                                            let _ = send.reset(1u8.into());
641                                            let _ = recv.stop(1u8.into());
642                                        }
643                                        tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id, is_control, ?result, "TRANSPORT_EXCHANGE_END");
644                                    });
645                                }
646                                _ = exchanges.next(), if !exchanges.is_empty() => {
647                                    idle.as_mut().reset(tokio::time::Instant::now() + CONNECTION_IDLE);
648                                }
649                                _ = &mut idle, if exchanges.is_empty() => {
650                                    connection.close(0u8.into(), b"idle");
651                                    return Ok(());
652                                }
653                            }
654                        }
655                    }
656                    .await;
657                    tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id, stage, ?result, "TRANSPORT_RECEIVE_END");
658                    if result.is_err() {
659                        tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id,
660                            stats = ?observed_connection.as_ref().map(|connection| connection.stats()),
661                            "TRANSPORT_RECEIVE_STATS");
662                    }
663                });
664            }
665        });
666        let control_signal_for_membership = control_signal.clone();
667        Ok((
668            Self {
669                controls,
670                control_inbox: node_control_inbox,
671                control_signal,
672                control_cancel,
673                deferred_controls: std::collections::VecDeque::new(),
674                connections,
675                routing,
676                resources,
677                events,
678                overlays,
679                membership: overlay::MembershipInbox::new(control_signal_for_membership),
680                listener,
681            },
682            receiver,
683        ))
684    }
685
686    pub fn id(&self) -> PeerId {
687        self.connections.id()
688    }
689
690    /// A session-local cancellation latch for outbound control exchanges.
691    pub fn control_cancellation(&self) -> watch::Sender<bool> {
692        self.control_cancel.clone()
693    }
694
695    /// Next membership step received by gossip, as (workspace, opaque bytes).
696    /// Delivered without a policy revision check (ADR 0008): the caller must
697    /// verify the step itself against its own saved state before using it.
698    pub fn poll_membership_gossip(&self) -> Option<(WorkspaceId, Vec<u8>)> {
699        self.membership.pop()
700    }
701
702    /// Wait for one queued membership envelope for `workspace`.
703    /// The caller owns the deadline; no polling interval is introduced here.
704    pub async fn wait_for_membership_gossip(&self, workspace: WorkspaceId) -> Option<Vec<u8>> {
705        loop {
706            let notified = self.control_signal.notified();
707            if let Some(payload) = self.membership.pop_for(workspace) {
708                return Some(payload);
709            }
710            notified.await;
711        }
712    }
713
714    /// Broadcast a committed membership step on the workspace overlay's
715    /// reserved membership topic. Owned so the host can spawn it. Ok(false)
716    /// means no overlay or no remote member to send to.
717    pub fn broadcast_membership(
718        &self,
719        workspace: WorkspaceId,
720        payload: Vec<u8>,
721    ) -> impl std::future::Future<Output = Result<bool>> + Send + 'static {
722        let overlays = self.overlays.clone();
723        let sender = self.id();
724        async move {
725            let Some(overlay) = overlays.lock().await.get(&workspace).cloned() else {
726                return Ok(false);
727            };
728            overlay.broadcast_membership(sender, payload).await
729        }
730    }
731    pub fn address(&self) -> SocketAddr {
732        self.connections.address()
733    }
734
735    /// Wait until the endpoint has an active relay connection.
736    ///
737    /// This is only needed by callers that explicitly selected a relay-only
738    /// profile. It is event-driven and does not add a polling delay or fallback.
739    pub async fn wait_online(&self) {
740        self.connections.wait_online().await;
741    }
742
743    pub fn resources(&self) -> resources::ResourceTransfers {
744        self.resources.clone()
745    }
746
747    /// Android cannot detect host network changes from native code. The Java
748    /// connectivity callback must forward them so Iroh can rebind its sockets.
749    pub async fn network_change(&self) {
750        self.connections.network_change().await;
751    }
752
753    pub async fn add_address_hint(&self, peer: PeerId, address: SocketAddr) -> Result<()> {
754        self.connections.add_address_hint(peer, address).await?;
755        let overlays = self
756            .overlays
757            .lock()
758            .await
759            .values()
760            .cloned()
761            .collect::<Vec<_>>();
762        for overlay in overlays {
763            let authorized = self
764                .routing
765                .lock()
766                .await
767                .authorizes_endpoint(overlay.workspace, overlay.revision(), peer)
768                .is_ok();
769            if authorized && peer != self.id() {
770                if overlay.is_joined() {
771                    overlay.join_peer(peer).await?;
772                } else {
773                    tracing::info!(target: "data_fabric_transport", peer = %iroh::EndpointId::from_bytes(&peer).map(|p| p.fmt_short().to_string()).unwrap_or_default(), "GOSSIP_OVERLAY_REBUILD_ON_HINT");
774                    let peers = self
775                        .routing
776                        .lock()
777                        .await
778                        .authorized_endpoints(overlay.workspace, overlay.revision())?;
779                    let replacement = overlay::Overlay::prepare(
780                        &self.connections,
781                        overlay.workspace,
782                        overlay.revision(),
783                        peers,
784                        self.routing.clone(),
785                        self.events.clone(),
786                        self.membership.clone(),
787                    )
788                    .await?;
789                    self.overlays
790                        .lock()
791                        .await
792                        .insert(overlay.workspace, Arc::new(replacement));
793                }
794            }
795        }
796        Ok(())
797    }
798
799    /// Remember an authenticated return path without synchronously changing
800    /// any workspace gossip overlay. Control-plane observations use this
801    /// narrow path so a presence reply cannot wait on peer discovery.
802    pub async fn remember_observed(&self, peer: PeerId, address: SocketAddr) {
803        self.connections.remember_observed(peer, address).await;
804    }
805
806    /// A cached route is only a hint. Callers must separately authorize any
807    /// workspace metadata they share with it.
808    pub async fn address_hint(&self, peer: PeerId) -> Option<SocketAddr> {
809        self.connections.address_hint(peer).await
810    }
811
812    /// Endpoints passively observed on the application-named local mDNS service.
813    /// These are route hints only and carry no workspace or human identity claim.
814    pub async fn nearby_peers(&self) -> Vec<PeerId> {
815        self.connections.nearby_peers(true).await
816    }
817
818    /// Return a bounded batch for workspace advertisement queries.
819    pub async fn nearby_workspace_peers(&self) -> Vec<PeerId> {
820        self.connections.nearby_peers(false).await
821    }
822
823    /// Trusted application seam. Never invoke with an unverified network policy.
824    /// The caller resolves member/device bindings into endpoint permissions; this
825    /// module neither establishes those bindings nor verifies admin authority.
826    pub async fn install_verified_policy(
827        &self,
828        workspace: WorkspaceId,
829        revision: u64,
830        endpoint_permissions: BTreeMap<PeerId, Permissions>,
831    ) -> Result<()> {
832        let replace_overlay = self.overlays.lock().await.contains_key(&workspace);
833        let peers = endpoint_permissions.keys().copied().collect::<Vec<_>>();
834        self.routing.lock().await.install_verified_policy(
835            workspace,
836            revision,
837            endpoint_permissions,
838        )?;
839        self.resources.policy_changed();
840        if replace_overlay {
841            if !peers.contains(&self.id()) {
842                self.connections
843                    .authorize_gossip(overlay::alpn(workspace), Vec::new())
844                    .await;
845                self.overlays.lock().await.remove(&workspace);
846                return Ok(());
847            }
848            self.connections
849                .authorize_gossip(overlay::alpn(workspace), peers.clone())
850                .await;
851            // An epoch that only adds members keeps the swarm: rebuilding it on
852            // every admission dropped all neighbors mid-broadcast (ADR 0008).
853            let existing = self.overlays.lock().await.get(&workspace).cloned();
854            if existing.is_some_and(|overlay| overlay.advance(revision, &peers)) {
855                return Ok(());
856            }
857            let candidate = overlay::Overlay::prepare(
858                &self.connections,
859                workspace,
860                revision,
861                peers,
862                self.routing.clone(),
863                self.events.clone(),
864                self.membership.clone(),
865            )
866            .await?;
867            self.overlays
868                .lock()
869                .await
870                .insert(workspace, Arc::new(candidate));
871        }
872        Ok(())
873    }
874
875    /// Enable one bounded workspace-wide live overlay after installing a verified
876    /// policy. Direct-recipient publications keep their acknowledged path.
877    pub async fn enable_gossip(&self, workspace: WorkspaceId, revision: u64) -> Result<()> {
878        if self
879            .overlays
880            .lock()
881            .await
882            .get(&workspace)
883            .is_some_and(|overlay| overlay.revision() == revision)
884        {
885            return Ok(());
886        }
887        let peers = self
888            .routing
889            .lock()
890            .await
891            .authorized_endpoints(workspace, revision)?;
892        self.connections
893            .authorize_gossip(overlay::alpn(workspace), peers.clone())
894            .await;
895        let candidate = overlay::Overlay::prepare(
896            &self.connections,
897            workspace,
898            revision,
899            peers,
900            self.routing.clone(),
901            self.events.clone(),
902            self.membership.clone(),
903        )
904        .await?;
905        self.overlays
906            .lock()
907            .await
908            .insert(workspace, Arc::new(candidate));
909        Ok(())
910    }
911
912    /// Read the current routing policy under the same lock used for data routing.
913    /// The callback must be bounded, synchronous and must not reenter this node.
914    /// Owned results may leave the lock; policy references cannot escape it.
915    pub async fn with_routing_policy<T>(&self, inspect: impl FnOnce(&RoutingTable) -> T) -> T {
916        let routing = self.routing.lock().await;
917        inspect(&routing)
918    }
919
920    /// Current maintained live neighbors for a workspace overlay. Zero means
921    /// no overlay or no connected neighbor; it is not a workspace member count.
922    pub async fn live_neighbor_count(&self, workspace: WorkspaceId) -> usize {
923        self.overlays
924            .lock()
925            .await
926            .get(&workspace)
927            .map_or(0, |overlay| overlay.neighbor_count())
928    }
929
930    /// Authenticated direct neighbors in the bounded workspace Gossip view.
931    /// These are advisory retrieval candidates, not proof that history exists.
932    pub async fn live_neighbors(&self, workspace: WorkspaceId) -> Vec<PeerId> {
933        self.overlays
934            .lock()
935            .await
936            .get(&workspace)
937            .map_or_else(Vec::new, |overlay| overlay.neighbors())
938    }
939
940    /// Wait for the native Gossip overlay to observe at least one neighbor.
941    /// The caller owns the deadline; no polling interval is introduced here.
942    pub async fn wait_for_gossip_neighbor(&self, workspace: WorkspaceId) -> bool {
943        let Some(overlay) = self.overlays.lock().await.get(&workspace).cloned() else {
944            return false;
945        };
946        overlay.wait_for_neighbor().await
947    }
948
949    pub async fn subscribe(
950        &self,
951        workspace: WorkspaceId,
952        revision: u64,
953        topic: Topic,
954    ) -> Result<AdmissionReport> {
955        self.interest(workspace, revision, topic, true).await
956    }
957
958    pub async fn unsubscribe(
959        &self,
960        workspace: WorkspaceId,
961        revision: u64,
962        topic: Topic,
963    ) -> Result<AdmissionReport> {
964        self.interest(workspace, revision, topic, false).await
965    }
966
967    async fn interest(
968        &self,
969        workspace: WorkspaceId,
970        revision: u64,
971        topic: Topic,
972        subscribe: bool,
973    ) -> Result<AdmissionReport> {
974        self.prepare_interest(workspace, revision, topic, subscribe)
975            .await?
976            .await
977    }
978
979    /// Apply local interest before returning an owned remote announcement.
980    /// Hosts may schedule this future without holding their session lock. They
981    /// must serialize announcements; dropping one does not undo local interest.
982    pub async fn prepare_interest(
983        &self,
984        workspace: WorkspaceId,
985        revision: u64,
986        topic: Topic,
987        subscribe: bool,
988    ) -> Result<impl std::future::Future<Output = Result<AdmissionReport>> + Send + 'static> {
989        let mut peers = {
990            let mut routing = self.routing.lock().await;
991            let peers = routing.publishers(workspace, revision, self.id(), &topic)?;
992            if subscribe {
993                routing.subscribe(workspace, revision, self.id(), topic.clone())?;
994            } else {
995                routing.unsubscribe(workspace, revision, self.id(), &topic)?;
996            }
997            peers
998        };
999        let local = self.id();
1000        let admitted_local = peers.contains(&local);
1001        peers.retain(|peer| *peer != local);
1002        // Local interest is already installed. A delayed self-announcement must
1003        // never overwrite a more recent local unsubscribe.
1004        let remote = self.fanout(
1005            peers,
1006            Frame {
1007                workspace,
1008                revision,
1009                topic: topic.as_str().into(),
1010                delivery: DeliveryClass::Critical,
1011                operation: if subscribe {
1012                    Operation::Subscribe
1013                } else {
1014                    Operation::Unsubscribe
1015                },
1016            },
1017        );
1018        Ok(async move {
1019            let mut report = remote.await?;
1020            if admitted_local {
1021                report.admitted.push(local);
1022                report.admitted.sort_unstable();
1023            }
1024            Ok(report)
1025        })
1026    }
1027
1028    pub async fn publish(
1029        &self,
1030        workspace: WorkspaceId,
1031        revision: u64,
1032        topic: Topic,
1033        payload: Vec<u8>,
1034    ) -> Result<AdmissionReport> {
1035        self.publish_with_class(workspace, revision, topic, DeliveryClass::Critical, payload)
1036            .await
1037    }
1038
1039    pub async fn publish_with_class(
1040        &self,
1041        workspace: WorkspaceId,
1042        revision: u64,
1043        topic: Topic,
1044        delivery: DeliveryClass,
1045        payload: Vec<u8>,
1046    ) -> Result<AdmissionReport> {
1047        if payload.len() > MAX_PAYLOAD {
1048            return Err(Error::TooLarge);
1049        }
1050        let overlay = self.overlays.lock().await.get(&workspace).cloned();
1051        if let Some(overlay) = overlay {
1052            if overlay.revision() != revision {
1053                return Err(Error::Routing(arachne_routing::Error::WrongPolicyRevision));
1054            }
1055            let recipients =
1056                self.routing
1057                    .lock()
1058                    .await
1059                    .recipients(workspace, revision, self.id(), &topic)?;
1060            let mut report = AdmissionReport::default();
1061            if recipients.contains(&self.id()) {
1062                apply(
1063                    &self.routing,
1064                    &self.events,
1065                    self.id(),
1066                    self.id(),
1067                    self.id(),
1068                    Frame {
1069                        workspace,
1070                        revision,
1071                        topic: topic.as_str().into(),
1072                        delivery,
1073                        operation: Operation::Publish(payload.clone()),
1074                    },
1075                )
1076                .await?;
1077                report.admitted.push(self.id());
1078            }
1079            report.queued = overlay
1080                .broadcast(self.id(), &topic, delivery, payload)
1081                .await?;
1082            return Ok(report);
1083        }
1084        let peers = self
1085            .routing
1086            .lock()
1087            .await
1088            .recipients(workspace, revision, self.id(), &topic)?;
1089        self.fanout(
1090            peers,
1091            Frame {
1092                workspace,
1093                revision,
1094                topic: topic.as_str().into(),
1095                delivery,
1096                operation: Operation::Publish(payload),
1097            },
1098        )
1099        .await
1100    }
1101
1102    /// Send to authorized subscribers within the resolved audience. Each selected
1103    /// endpoint without announced interest gets a non-admission result.
1104    pub async fn publish_to(
1105        &self,
1106        workspace: WorkspaceId,
1107        revision: u64,
1108        topic: Topic,
1109        endpoints: Vec<PeerId>,
1110        recipients: Vec<[u8; 32]>,
1111        payload: Vec<u8>,
1112    ) -> Result<AdmissionReport> {
1113        self.publish_to_with_class(
1114            workspace,
1115            revision,
1116            topic,
1117            endpoints,
1118            recipients,
1119            DeliveryClass::Critical,
1120            payload,
1121        )
1122        .await
1123    }
1124
1125    #[allow(clippy::too_many_arguments)]
1126    pub async fn publish_to_with_class(
1127        &self,
1128        workspace: WorkspaceId,
1129        revision: u64,
1130        topic: Topic,
1131        endpoints: Vec<PeerId>,
1132        recipients: Vec<[u8; 32]>,
1133        delivery: DeliveryClass,
1134        payload: Vec<u8>,
1135    ) -> Result<AdmissionReport> {
1136        if payload.len() > MAX_PAYLOAD
1137            || recipients.is_empty()
1138            || recipients.len() > MAX_RECIPIENTS
1139            || recipients.windows(2).any(|pair| pair[0] >= pair[1])
1140        {
1141            return Err(Error::Rejected);
1142        }
1143        let peers = self.routing.lock().await.direct_recipients(
1144            workspace,
1145            revision,
1146            self.id(),
1147            &topic,
1148            &endpoints,
1149        )?;
1150        let skipped: Vec<_> = endpoints
1151            .into_iter()
1152            .filter(|peer| peers.binary_search(peer).is_err())
1153            .collect();
1154        let mut report = self
1155            .fanout(
1156                peers,
1157                Frame {
1158                    workspace,
1159                    revision,
1160                    topic: topic.as_str().into(),
1161                    delivery,
1162                    operation: Operation::DirectPublish {
1163                        payload,
1164                        recipients,
1165                    },
1166                },
1167            )
1168            .await?;
1169        report
1170            .failed
1171            .extend(skipped.into_iter().map(|peer| (peer, Error::NotSubscribed)));
1172        report.failed.sort_unstable_by_key(|(peer, _)| *peer);
1173        Ok(report)
1174    }
1175
1176    fn fanout(
1177        &self,
1178        peers: Vec<PeerId>,
1179        frame: Frame,
1180    ) -> impl std::future::Future<Output = Result<AdmissionReport>> + Send + 'static {
1181        let connections = self.connections.clone();
1182        let routing = self.routing.clone();
1183        let events = self.events.clone();
1184        let local = self.id();
1185        async move {
1186            let bytes = wire::encode(&frame)?;
1187            let mut report = AdmissionReport::default();
1188            // ponytail: 16 concurrent operations per fanout; connection reuse and a
1189            // sparse overlay still need measurement before large-workspace claims.
1190            let mut pending = stream::iter(peers.into_iter().map(|peer| {
1191                let frame = &frame;
1192                let bytes = &bytes;
1193                let connections = &connections;
1194                let routing = &routing;
1195                let events = &events;
1196                async move {
1197                    let result = if peer == local {
1198                        match wire::decode(bytes) {
1199                            Ok(copy) => apply(routing, events, local, local, local, copy).await,
1200                            Err(_) => Err(Error::InvalidFrame),
1201                        }
1202                    } else {
1203                        send_frame(connections, routing, peer, frame, bytes).await
1204                    };
1205                    (peer, result)
1206                }
1207            }))
1208            .buffer_unordered(16);
1209            while let Some((peer, result)) = pending.next().await {
1210                match result {
1211                    Ok(()) => report.admitted.push(peer),
1212                    Err(error) => report.failed.push((peer, error)),
1213                }
1214            }
1215            // Preserve the public report's deterministic peer order.
1216            report.admitted.sort_unstable();
1217            report.failed.sort_unstable_by_key(|(peer, _)| *peer);
1218            Ok(report)
1219        }
1220    }
1221
1222    pub async fn close(mut self) {
1223        self.resources.close().await;
1224        self.overlays.lock().await.clear();
1225        self.connections.close().await;
1226        self.listener.abort();
1227        let _ = (&mut self.listener).await;
1228
1229    }
1230}
1231
1232async fn receive_frame(
1233    connection: &iroh::endpoint::Connection,
1234    connections: &Connections,
1235    routing: &Mutex<RoutingTable>,
1236    output: &DeliveryQueue,
1237    address: Option<SocketAddr>,
1238    (send, recv): (
1239        &mut iroh::endpoint::SendStream,
1240        &mut iroh::endpoint::RecvStream,
1241    ),
1242) -> Result<()> {
1243    let sender = *connection.remote_id().as_bytes();
1244    let bytes = recv.read_to_end(MAX_FRAME).await.map_err(transport)?;
1245    tracing::info!(target: "data_fabric_transport", remote = %connection.remote_id(), connection_id = connection.stable_id(), bytes = bytes.len(), "TRANSPORT_FRAME_RECEIVED");
1246    let outcome = match wire::decode::<Frame>(&bytes) {
1247        Ok(frame) => apply(routing, output, connections.id(), sender, sender, frame).await,
1248        Err(_) => Err(Error::InvalidFrame),
1249    };
1250    // Reusing transport identity never caches workspace authority. Remember an
1251    // observed direct path only after this particular frame is authorized.
1252    if outcome.is_ok()
1253        && let Some(address) = address
1254    {
1255        connections.remember_observed(sender, address).await;
1256    }
1257    tracing::info!(target: "data_fabric_transport", remote = %connection.remote_id(), connection_id = connection.stable_id(), ?outcome, "TRANSPORT_ADMISSION");
1258    send.write_all(&[u8::from(outcome.is_ok())])
1259        .await
1260        .map_err(transport)?;
1261    send.finish().map_err(transport)?;
1262    Ok(())
1263}
1264
1265async fn send_frame(
1266    connections: &Connections,
1267    routing: &Mutex<RoutingTable>,
1268    peer: PeerId,
1269    frame: &Frame,
1270    bytes: &[u8],
1271) -> Result<()> {
1272    let key = iroh::PublicKey::from_bytes(&peer).map_err(transport)?;
1273    let mut stage = "connect";
1274    let mut observed_connection = None;
1275    tracing::info!(target: "data_fabric_transport", remote = %key, bytes = bytes.len(), "TRANSPORT_SEND_BEGIN");
1276    let result = tokio::time::timeout(TIMEOUT, async {
1277        let connection = connections.connect(peer, ALPN).await?;
1278        observed_connection = Some(connection.clone());
1279        tracing::info!(target: "data_fabric_transport", remote = %key, connection_id = connection.stable_id(), paths = ?connection.paths(), "TRANSPORT_SEND_CONNECTED");
1280        stage = "open stream";
1281        let (mut send, mut recv) = connection.open_bi().await.map_err(transport)?;
1282        // Connection establishment can outlive a policy change. Recheck at
1283        // outbound admission; already-admitted bytes may remain in flight.
1284        stage = "recheck authorization";
1285        let topic = Topic::new(frame.topic.clone())?;
1286        let routing = routing.lock().await;
1287        let allowed = match &frame.operation {
1288            Operation::Publish(_) => {
1289                routing.recipients(frame.workspace, frame.revision, connections.id(), &topic)?
1290            }
1291            Operation::DirectPublish { .. } => routing.direct_recipients(
1292                frame.workspace,
1293                frame.revision,
1294                connections.id(),
1295                &topic,
1296                &[peer],
1297            )?,
1298            _ => {
1299                routing.publishers(frame.workspace, frame.revision, connections.id(), &topic)?
1300            }
1301        };
1302        if !allowed.contains(&peer) {
1303            return Err(Error::Routing(arachne_routing::Error::Denied));
1304        }
1305        drop(routing);
1306        stage = "write frame";
1307        send.write_all(&[0]).await.map_err(transport)?;
1308        send.write_all(bytes).await.map_err(transport)?;
1309        send.finish().map_err(transport)?;
1310        stage = "read acknowledgment";
1311        let ack = recv.read_to_end(1).await.map_err(transport)?;
1312        if ack == [1] {
1313            Ok(())
1314        } else {
1315            Err(Error::Rejected)
1316        }
1317    })
1318    .await
1319    .map_err(|_| Error::Timeout(stage))
1320    .and_then(|result| result);
1321    tracing::info!(target: "data_fabric_transport", remote = %key, stage, ?result, "TRANSPORT_SEND_END");
1322    if result.is_err() {
1323        tracing::info!(target: "data_fabric_transport", remote = %key,
1324            stats = ?observed_connection.as_ref().map(|connection| connection.stats()),
1325            "TRANSPORT_SEND_STATS");
1326    }
1327    result
1328}
1329
1330impl Drop for Node {
1331    fn drop(&mut self) {
1332        self.resources.stop();
1333        self.events.close();
1334        self.listener.abort();
1335    }
1336}
1337
1338fn transport(error: impl std::fmt::Display) -> Error {
1339    Error::Transport(error.to_string())
1340}
1341
1342async fn apply(
1343    routing: &Mutex<RoutingTable>,
1344    events: &DeliveryQueue,
1345    local: PeerId,
1346    sender: PeerId,
1347    received_from: PeerId,
1348    frame: Frame,
1349) -> Result<()> {
1350    let topic = Topic::new(frame.topic)?;
1351    let delivery = frame.delivery;
1352    let mut routing = routing.lock().await;
1353    match frame.operation {
1354        Operation::Subscribe => {
1355            routing.subscribe(frame.workspace, frame.revision, sender, topic)?
1356        }
1357        Operation::Unsubscribe => {
1358            routing.unsubscribe(frame.workspace, frame.revision, sender, &topic)?
1359        }
1360        Operation::Publish(payload) => {
1361            if payload.len() > MAX_PAYLOAD {
1362                return Err(Error::TooLarge);
1363            }
1364            let recipients = routing.recipients(frame.workspace, frame.revision, sender, &topic)?;
1365            if !recipients.contains(&local) {
1366                return Err(Error::Rejected);
1367            }
1368            events.push(Message {
1369                workspace: frame.workspace,
1370                revision: frame.revision,
1371                sender,
1372                received_from,
1373                topic,
1374                payload,
1375                recipients: Vec::new(),
1376                delivery,
1377            })?;
1378        }
1379        Operation::DirectPublish {
1380            payload,
1381            recipients,
1382        } => {
1383            if payload.len() > MAX_PAYLOAD
1384                || recipients.is_empty()
1385                || recipients.len() > MAX_RECIPIENTS
1386                || recipients.windows(2).any(|pair| pair[0] >= pair[1])
1387            {
1388                return Err(Error::Rejected);
1389            }
1390            let recipient_endpoints = routing.direct_recipients(
1391                frame.workspace,
1392                frame.revision,
1393                sender,
1394                &topic,
1395                &[local],
1396            )?;
1397            if recipient_endpoints != [local] {
1398                return Err(Error::Rejected);
1399            }
1400            events.push(Message {
1401                workspace: frame.workspace,
1402                revision: frame.revision,
1403                sender,
1404                received_from,
1405                topic,
1406                payload,
1407                recipients,
1408                delivery,
1409            })?;
1410        }
1411    }
1412    Ok(())
1413}
1414
1415#[tokio::test]
1416async fn receiver_checks_local_interest_even_if_sender_routes_a_direct_frame() {
1417    let routing = Mutex::new(RoutingTable::default());
1418    let events = DeliveryQueue::default();
1419    let mut received = MessageReceiver {
1420        queue: events.clone(),
1421    };
1422    let topic = Topic::new("streams/opaque").unwrap();
1423    routing
1424        .lock()
1425        .await
1426        .install_verified_policy(
1427            [9; 32],
1428            1,
1429            BTreeMap::from([
1430                ([1; 32], Permissions::AllTopics),
1431                ([2; 32], Permissions::AllTopics),
1432            ]),
1433        )
1434        .unwrap();
1435    let frame = || Frame {
1436        workspace: [9; 32],
1437        revision: 1,
1438        topic: topic.as_str().into(),
1439        delivery: DeliveryClass::Critical,
1440        operation: Operation::DirectPublish {
1441            payload: vec![0, 255],
1442            recipients: vec![[3; 32]],
1443        },
1444    };
1445    assert!(
1446        apply(&routing, &events, [2; 32], [1; 32], [1; 32], frame())
1447            .await
1448            .is_err(),
1449        "receiver admitted a direct frame without local interest"
1450    );
1451    routing
1452        .lock()
1453        .await
1454        .subscribe([9; 32], 1, [2; 32], topic.clone())
1455        .unwrap();
1456    apply(&routing, &events, [2; 32], [1; 32], [1; 32], frame())
1457        .await
1458        .unwrap();
1459    assert_eq!(received.recv().await.unwrap().payload, vec![0, 255]);
1460    routing
1461        .lock()
1462        .await
1463        .unsubscribe([9; 32], 1, [2; 32], &topic)
1464        .unwrap();
1465    assert!(
1466        apply(&routing, &events, [2; 32], [1; 32], [1; 32], frame())
1467            .await
1468            .is_err(),
1469        "stale sender interest defeated local unsubscribe"
1470    );
1471    assert!(received.try_recv().is_err());
1472}