1use std::{
9 collections::{BTreeMap, VecDeque},
10 net::SocketAddr,
11 sync::{
12 Arc, Mutex as StdMutex,
13 atomic::{AtomicBool, Ordering},
14 },
15 time::Duration,
16};
17
18mod budget;
19mod connections;
20mod control;
21mod overlay;
22pub mod resources;
23mod wire;
24pub use budget::{CapacityCounts, ConnectionBudget};
25pub use control::{ControlClient, ControlRequest, ControlTiming, InquiryResponder, MAX_CONTROL_REPLY, Timing};
26
27use connections::Connections;
28use arachne_routing::RoutingTable;
29pub use arachne_routing::{PeerId, Permissions, Topic, WorkspaceId};
30use futures_util::{StreamExt, stream};
31use serde::{Deserialize, Serialize};
32use tokio::{
33 sync::{Mutex, mpsc, watch},
34 task::JoinHandle,
35};
36
37const ALPN: &[u8] = b"data-fabric/pubsub-experiment/1";
38const MAX_PAYLOAD: usize = 16 * 1024;
39const MAX_RECIPIENTS: usize = 64;
40const MAX_FRAME: usize = 128 * 1024;
41const TIMEOUT: Duration = Duration::from_secs(5);
42const CONNECTION_IDLE: Duration = Duration::from_secs(60);
43const CRITICAL_QUEUE: usize = 256;
44const CURRENT_QUEUE: usize = 64;
45const BULK_QUEUE: usize = 64;
46
47#[derive(Clone, Copy, Debug)]
49pub enum NetworkProfile {
50 Direct,
51 Lan,
52 Nearby,
53 Wan,
54 RelayOnly,
55 WanOnly,
56}
57
58impl NetworkProfile {
59 fn settings(self) -> (Option<&'static str>, bool, bool, bool) {
60 match self {
61 Self::Direct => (None, false, false, true),
62 Self::Lan => (Some("data-fabric"), false, false, true),
63 Self::Nearby => (Some("arachne-nearby"), false, false, true),
64 Self::Wan => (Some("data-fabric"), true, false, false),
65 Self::RelayOnly => (None, true, true, false),
66 Self::WanOnly => (None, true, false, false),
67 }
68 }
69}
70
71#[derive(Clone, Debug)]
74pub struct RelayOptions {
75 pub(crate) map: iroh::RelayMap,
76 pub(crate) tls: iroh::tls::CaTlsConfig,
77}
78
79impl RelayOptions {
80 pub fn new(map: iroh::RelayMap, tls: iroh::tls::CaTlsConfig) -> Self {
81 Self { map, tls }
82 }
83}
84
85#[derive(Debug, thiserror::Error)]
86pub enum Error {
87 #[error("routing rejected: {0:?}")]
88 Routing(#[from] arachne_routing::Error),
89 #[error("transport: {0}")]
90 Transport(String),
91 #[error("unknown peer address")]
92 MissingPeer,
93 #[error("control request not sent: {0}")]
94 ControlNotSent(&'static str),
95 #[error("message exceeds limit")]
96 TooLarge,
97 #[error("peer rejected operation")]
98 Rejected,
99 #[error("operation deadline exceeded during {0}; admission outcome may be unknown")]
100 Timeout(&'static str),
101 #[error("operation cancelled because the local session closed")]
102 Cancelled,
103 #[error("invalid frame")]
104 InvalidFrame,
105 #[error("consumer queue full or closed")]
106 Backpressure,
107 #[error("peer has no announced subscription to this topic")]
108 NotSubscribed,
109}
110type Result<T> = std::result::Result<T, Error>;
111
112#[derive(Debug, Clone, PartialEq, Eq)]
113pub struct Message {
114 pub workspace: WorkspaceId,
115 pub revision: u64,
116 pub sender: PeerId,
117 pub received_from: PeerId,
120 pub topic: Topic,
121 pub payload: Vec<u8>,
122 pub recipients: Vec<[u8; 32]>,
125 pub delivery: DeliveryClass,
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
129#[serde(rename_all = "snake_case")]
130pub enum DeliveryClass {
132 #[default]
133 Critical,
134 Current {
135 replacement_key: [u8; 32],
136 },
137 Bulk,
138}
139
140type CurrentKey = (
141 WorkspaceId,
142 u64,
143 String,
144 PeerId,
145 PeerId,
146 [u8; 32],
147 Vec<[u8; 32]>,
148);
149
150#[derive(Default)]
151struct DeliveryState {
152 critical: VecDeque<Message>,
153 current: BTreeMap<CurrentKey, Message>,
154 bulk: VecDeque<Message>,
155 critical_run: u8,
156 bulk_turn: bool,
157}
158
159#[derive(Clone, Default)]
160struct DeliveryQueue {
161 state: Arc<StdMutex<DeliveryState>>,
162 changed: Arc<tokio::sync::Notify>,
163 closed: Arc<AtomicBool>,
164}
165
166impl DeliveryQueue {
167 fn push(&self, message: Message) -> Result<()> {
168 if self.closed.load(Ordering::Acquire) {
169 return Err(Error::Backpressure);
170 }
171 let mut state = self.state.lock().unwrap();
172 match message.delivery {
173 DeliveryClass::Critical if state.critical.len() < CRITICAL_QUEUE => {
174 state.critical.push_back(message)
175 }
176 DeliveryClass::Current { replacement_key } => {
177 let key = (
178 message.workspace,
179 message.revision,
180 message.topic.as_str().into(),
181 message.sender,
182 message.received_from,
183 replacement_key,
184 message.recipients.clone(),
185 );
186 if state.current.len() == CURRENT_QUEUE && !state.current.contains_key(&key) {
187 return Err(Error::Backpressure);
188 }
189 state.current.insert(key, message);
190 }
191 DeliveryClass::Bulk if state.bulk.len() < BULK_QUEUE => state.bulk.push_back(message),
192 _ => return Err(Error::Backpressure),
193 }
194 drop(state);
195 self.changed.notify_one();
196 Ok(())
197 }
198
199 fn pop(&self) -> Option<Message> {
200 let mut state = self.state.lock().unwrap();
201 if state.critical_run < 8
202 && let Some(message) = state.critical.pop_front()
203 {
204 state.critical_run += 1;
205 return Some(message);
206 }
207 let lower = if state.bulk_turn {
208 state
209 .bulk
210 .pop_front()
211 .or_else(|| state.current.pop_first().map(|(_, message)| message))
212 } else {
213 state
214 .current
215 .pop_first()
216 .map(|(_, message)| message)
217 .or_else(|| state.bulk.pop_front())
218 };
219 if lower.is_some() {
220 state.critical_run = 0;
221 state.bulk_turn = !state.bulk_turn;
222 return lower;
223 }
224 state.critical.pop_front()
225 }
226
227 fn close(&self) {
228 self.closed.store(true, Ordering::Release);
229 self.changed.notify_waiters();
230 }
231}
232
233pub struct MessageReceiver {
234 queue: DeliveryQueue,
235}
236
237impl MessageReceiver {
238 pub fn is_empty(&self) -> bool {
240 let state = self.queue.state.lock().unwrap();
241 state.critical.is_empty() && state.current.is_empty() && state.bulk.is_empty()
242 }
243
244 pub fn try_recv(&mut self) -> std::result::Result<Message, mpsc::error::TryRecvError> {
245 self.queue.pop().ok_or_else(|| {
246 if self.queue.closed.load(Ordering::Acquire) {
247 mpsc::error::TryRecvError::Disconnected
248 } else {
249 mpsc::error::TryRecvError::Empty
250 }
251 })
252 }
253
254 pub async fn recv(&mut self) -> Option<Message> {
255 loop {
256 let notify = self.queue.changed.clone();
257 let changed = notify.notified();
258 match self.try_recv() {
259 Ok(message) => return Some(message),
260 Err(mpsc::error::TryRecvError::Disconnected) => return None,
261 Err(mpsc::error::TryRecvError::Empty) => changed.await,
262 }
263 }
264 }
265}
266
267#[derive(Debug, Default)]
271pub struct AdmissionReport {
272 pub admitted: Vec<PeerId>,
273 pub failed: Vec<(PeerId, Error)>,
274 pub queued: bool,
277}
278
279#[derive(Serialize, Deserialize)]
280#[serde(deny_unknown_fields)]
281struct Frame {
282 workspace: WorkspaceId,
283 revision: u64,
284 #[serde(deserialize_with = "wire::topic")]
285 topic: String,
286 delivery: DeliveryClass,
287 operation: Operation,
288}
289
290#[derive(Serialize, Deserialize)]
291enum Operation {
292 Subscribe,
293 Unsubscribe,
294 Publish(#[serde(with = "wire::payload")] Vec<u8>),
295 DirectPublish {
296 #[serde(with = "wire::payload")]
297 payload: Vec<u8>,
298 #[serde(deserialize_with = "wire::recipients")]
299 recipients: Vec<[u8; 32]>,
300 },
301}
302
303#[derive(Debug, serde::Serialize)]
306pub struct TransportMetrics {
307 pub received_bytes: u64,
308 pub sent_bytes: u64,
309 pub receive_queue: usize,
310 pub paths: Vec<PeerPath>,
311 pub paths_limited: bool,
312}
313
314#[derive(Debug, serde::Serialize)]
315pub struct PeerPath {
316 pub endpoint: PeerId,
317 pub route: &'static str,
318 pub rtt_ms: u64,
319}
320
321pub struct Node {
322 controls: mpsc::Receiver<ControlRequest>,
323 control_inbox: control::ControlInbox,
324 control_signal: Arc<tokio::sync::Notify>,
325 control_cancel: watch::Sender<bool>,
326 deferred_controls: std::collections::VecDeque<ControlRequest>,
327 connections: Connections,
328 routing: Arc<Mutex<RoutingTable>>,
329 resources: resources::ResourceTransfers,
330 events: DeliveryQueue,
331 overlays: Arc<Mutex<BTreeMap<WorkspaceId, Arc<overlay::Overlay>>>>,
332 membership: overlay::MembershipInbox,
333 listener: JoinHandle<()>,
334}
335
336impl Node {
337 pub fn transport_metrics(&self) -> TransportMetrics {
338 let mut metrics = self.connections.metrics();
339 let queue = self.events.state.lock().unwrap();
340 metrics.receive_queue = queue.critical.len() + queue.current.len() + queue.bulk.len();
341 metrics
342 }
343
344 pub async fn bind(address: SocketAddr) -> Result<(Self, MessageReceiver)> {
347 Self::bind_with_profile(
348 address,
349 None,
350 NetworkProfile::Direct,
351 ConnectionBudget::default(),
352 )
353 .await
354 }
355
356 pub async fn bind_with_identity(
360 address: SocketAddr,
361 secret: &[u8; 32],
362 ) -> Result<(Self, MessageReceiver)> {
363 Self::bind_with_profile(
364 address,
365 Some(secret),
366 NetworkProfile::Direct,
367 ConnectionBudget::default(),
368 )
369 .await
370 }
371
372 pub async fn bind_lan(address: SocketAddr) -> Result<(Self, MessageReceiver)> {
376 Self::bind_with_profile(
377 address,
378 None,
379 NetworkProfile::Lan,
380 ConnectionBudget::default(),
381 )
382 .await
383 }
384
385 pub async fn bind_lan_with_identity(
388 address: SocketAddr,
389 secret: &[u8; 32],
390 ) -> Result<(Self, MessageReceiver)> {
391 Self::bind_with_profile(
392 address,
393 Some(secret),
394 NetworkProfile::Lan,
395 ConnectionBudget::default(),
396 )
397 .await
398 }
399
400 pub async fn bind_nearby_with_identity(
402 address: SocketAddr,
403 secret: &[u8; 32],
404 ) -> Result<(Self, MessageReceiver)> {
405 Self::bind_with_profile(
406 address,
407 Some(secret),
408 NetworkProfile::Nearby,
409 ConnectionBudget::default(),
410 )
411 .await
412 }
413
414 pub async fn bind_wan_with_identity(
417 address: SocketAddr,
418 secret: &[u8; 32],
419 ) -> Result<(Self, MessageReceiver)> {
420 Self::bind_with_profile(
421 address,
422 Some(secret),
423 NetworkProfile::Wan,
424 ConnectionBudget::default(),
425 )
426 .await
427 }
428
429 pub async fn bind_relay_with_identity(
432 address: SocketAddr,
433 secret: &[u8; 32],
434 ) -> Result<(Self, MessageReceiver)> {
435 Self::bind_with_profile(
436 address,
437 Some(secret),
438 NetworkProfile::RelayOnly,
439 ConnectionBudget::default(),
440 )
441 .await
442 }
443
444 pub async fn bind_wan_only_with_identity(
447 address: SocketAddr,
448 secret: &[u8; 32],
449 ) -> Result<(Self, MessageReceiver)> {
450 Self::bind_with_profile(
451 address,
452 Some(secret),
453 NetworkProfile::WanOnly,
454 ConnectionBudget::default(),
455 )
456 .await
457 }
458
459 pub async fn bind_with_profile(
462 address: SocketAddr,
463 secret: Option<&[u8; 32]>,
464 profile: NetworkProfile,
465 budget: ConnectionBudget,
466 ) -> Result<(Self, MessageReceiver)> {
467 Self::bind_with_profile_and_relays(address, secret, profile, budget, None).await
468 }
469
470 pub async fn bind_with_profile_and_relay(
476 address: SocketAddr,
477 secret: Option<&[u8; 32]>,
478 profile: NetworkProfile,
479 budget: ConnectionBudget,
480 relay: RelayOptions,
481 ) -> Result<(Self, MessageReceiver)> {
482 Self::bind_with_profile_and_relays(address, secret, profile, budget, Some(relay)).await
483 }
484
485 async fn bind_with_profile_and_relays(
486 address: SocketAddr,
487 secret: Option<&[u8; 32]>,
488 profile: NetworkProfile,
489 budget: ConnectionBudget,
490 relay: Option<RelayOptions>,
491 ) -> Result<(Self, MessageReceiver)> {
492 let connections = Connections::bind(
493 address,
494 profile,
495 secret.map(iroh::SecretKey::from_bytes),
496 budget.clone(),
497 vec![ALPN.to_vec(), control::ALPN.to_vec()],
498 relay,
499 )
500 .await?;
501 let routing = Arc::new(Mutex::new(RoutingTable::default()));
502 let resources = resources::ResourceTransfers::new(connections.clone(), routing.clone());
503 let events = DeliveryQueue::default();
504 let receiver = MessageReceiver {
505 queue: events.clone(),
506 };
507 let (control_inbox, controls, control_signal) = control::ControlInbox::new(512);
508 let (control_cancel, _) = watch::channel(false);
509 let node_control_inbox = control_inbox.clone();
510 let overlays: Arc<Mutex<BTreeMap<WorkspaceId, Arc<overlay::Overlay>>>> =
511 Arc::new(Mutex::new(BTreeMap::new()));
512 let accepted_connections = connections.clone();
513 let shared = routing.clone();
514 let output = events.clone();
515 let accepted_overlays = overlays.clone();
516 let accepted_resources = resources.clone();
517 let listener = tokio::spawn(async move {
518 let handshake_capacity = budget.handshakes;
521 let [data_capacity, control_capacity] = budget.exchanges;
522 let mut workers = tokio::task::JoinSet::new();
523 while let Some(incoming) = accepted_connections.accept().await {
524 while workers.try_join_next().is_some() {}
525 let Ok(permit) = handshake_capacity.clone().try_acquire_owned() else {
526 incoming.refuse();
527 continue;
528 };
529 let control_inbox = control_inbox.clone();
530 let control_capacity = control_capacity.clone();
531 let data_capacity = data_capacity.clone();
532 let routing = shared.clone();
533 let overlays = accepted_overlays.clone();
534 let connections = accepted_connections.clone();
535 let output = output.clone();
536 let resources = accepted_resources.clone();
537 workers.spawn(async move {
538 let mut stage = "accept connection";
539 let mut remote = None;
540 let mut connection_id = None;
541 let mut observed_connection = None;
542 let result = async {
543 let observed_address = incoming.remote_addr();
544 let connection = tokio::time::timeout(TIMEOUT, incoming).await
545 .map_err(|_| Error::Timeout("accept connection"))?.map_err(transport)?;
546 drop(permit);
547 observed_connection = Some(connection.clone());
548 let sender = *connection.remote_id().as_bytes();
549 remote = Some(connection.remote_id());
550 connection_id = Some(connection.stable_id());
551 let is_control = connection.alpn() == control::ALPN;
552 let overlay = {
553 let overlays = overlays.lock().await;
554 overlays
555 .values()
556 .find(|overlay| overlay.alpn.as_slice() == connection.alpn())
557 .map(|overlay| {
558 (
559 overlay.workspace,
560 overlay.revision(),
561 overlay.gossip.clone(),
562 )
563 })
564 };
565 if let Some((workspace, revision, gossip)) = overlay {
566 routing
567 .lock()
568 .await
569 .authorizes_endpoint(workspace, revision, sender)?;
570 stage = "accept gossip";
571 tokio::time::timeout(control::CONTROL_TIMEOUT, gossip.handle_connection(connection))
572 .await.map_err(|_| Error::Timeout("accept gossip"))?.map_err(transport)?;
573 return Ok(());
574 }
575 if !is_control && connection.alpn() != ALPN {
578 return Err(Error::InvalidFrame);
579 }
580 let connection_key = connection.stable_id();
582 connections.mark_evictable(connection_key);
583 let (capacity, budget) = if is_control {
584 (control_capacity, control::CONTROL_TIMEOUT)
585 } else {
586 (data_capacity, TIMEOUT)
587 };
588 let address = match observed_address {
589 iroh::endpoint::IncomingAddr::Ip(address) => Some(address),
590 _ => None,
591 };
592 stage = "receive exchanges";
593 tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id, paths = ?connection.paths(), "TRANSPORT_RECEIVE_CONNECTED");
594 let connection = &connection;
595 let connections = &connections;
596 let routing = &routing;
597 let output = &output;
598 let control_inbox = &control_inbox;
599 let resources = &resources;
600 let mut exchanges = stream::FuturesUnordered::new();
601 let idle = tokio::time::sleep(CONNECTION_IDLE);
602 tokio::pin!(idle);
603 loop {
604 tokio::select! {
605 streams = connection.accept_bi() => {
606 let Ok((mut send, mut recv)) = streams else { return Ok(()); };
607 let Ok(permit) = capacity.clone().try_acquire_owned() else {
608 let _ = send.reset(1u8.into());
609 let _ = recv.stop(1u8.into());
610 continue;
611 };
612 idle.as_mut().reset(tokio::time::Instant::now() + CONNECTION_IDLE);
613 let guard = connections.exchange(connection_key);
614 exchanges.push(async move {
615 let _permit = permit;
616 let _guard = guard;
617 let result = async {
618 if !is_control {
619 let mut kind = [0; 1];
620 tokio::time::timeout(TIMEOUT, recv.read_exact(&mut kind)).await
621 .map_err(|_| Error::Timeout("stream kind"))?.map_err(transport)?;
622 match kind[0] {
623 resources::STREAM_KIND => return resources.serve(connection, &mut send, &mut recv).await,
624 0 => (),
625 _ => return Err(Error::InvalidFrame),
626 }
627 }
628 tokio::time::timeout(budget, async {
629 if is_control {
630 control::receive(connection, control_inbox, address, (&mut send, &mut recv)).await
631 } else {
632 receive_frame(connection, connections, routing, output, address, (&mut send, &mut recv)).await
633 }
634 }).await.map_err(|_| Error::Timeout("receive exchange"))?
635 }.await;
636 if result.is_err() {
640 let _ = send.reset(1u8.into());
641 let _ = recv.stop(1u8.into());
642 }
643 tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id, is_control, ?result, "TRANSPORT_EXCHANGE_END");
644 });
645 }
646 _ = exchanges.next(), if !exchanges.is_empty() => {
647 idle.as_mut().reset(tokio::time::Instant::now() + CONNECTION_IDLE);
648 }
649 _ = &mut idle, if exchanges.is_empty() => {
650 connection.close(0u8.into(), b"idle");
651 return Ok(());
652 }
653 }
654 }
655 }
656 .await;
657 tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id, stage, ?result, "TRANSPORT_RECEIVE_END");
658 if result.is_err() {
659 tracing::info!(target: "data_fabric_transport", ?remote, ?connection_id,
660 stats = ?observed_connection.as_ref().map(|connection| connection.stats()),
661 "TRANSPORT_RECEIVE_STATS");
662 }
663 });
664 }
665 });
666 let control_signal_for_membership = control_signal.clone();
667 Ok((
668 Self {
669 controls,
670 control_inbox: node_control_inbox,
671 control_signal,
672 control_cancel,
673 deferred_controls: std::collections::VecDeque::new(),
674 connections,
675 routing,
676 resources,
677 events,
678 overlays,
679 membership: overlay::MembershipInbox::new(control_signal_for_membership),
680 listener,
681 },
682 receiver,
683 ))
684 }
685
686 pub fn id(&self) -> PeerId {
687 self.connections.id()
688 }
689
690 pub fn control_cancellation(&self) -> watch::Sender<bool> {
692 self.control_cancel.clone()
693 }
694
695 pub fn poll_membership_gossip(&self) -> Option<(WorkspaceId, Vec<u8>)> {
699 self.membership.pop()
700 }
701
702 pub async fn wait_for_membership_gossip(&self, workspace: WorkspaceId) -> Option<Vec<u8>> {
705 loop {
706 let notified = self.control_signal.notified();
707 if let Some(payload) = self.membership.pop_for(workspace) {
708 return Some(payload);
709 }
710 notified.await;
711 }
712 }
713
714 pub fn broadcast_membership(
718 &self,
719 workspace: WorkspaceId,
720 payload: Vec<u8>,
721 ) -> impl std::future::Future<Output = Result<bool>> + Send + 'static {
722 let overlays = self.overlays.clone();
723 let sender = self.id();
724 async move {
725 let Some(overlay) = overlays.lock().await.get(&workspace).cloned() else {
726 return Ok(false);
727 };
728 overlay.broadcast_membership(sender, payload).await
729 }
730 }
731 pub fn address(&self) -> SocketAddr {
732 self.connections.address()
733 }
734
735 pub async fn wait_online(&self) {
740 self.connections.wait_online().await;
741 }
742
743 pub fn resources(&self) -> resources::ResourceTransfers {
744 self.resources.clone()
745 }
746
747 pub async fn network_change(&self) {
750 self.connections.network_change().await;
751 }
752
753 pub async fn add_address_hint(&self, peer: PeerId, address: SocketAddr) -> Result<()> {
754 self.connections.add_address_hint(peer, address).await?;
755 let overlays = self
756 .overlays
757 .lock()
758 .await
759 .values()
760 .cloned()
761 .collect::<Vec<_>>();
762 for overlay in overlays {
763 let authorized = self
764 .routing
765 .lock()
766 .await
767 .authorizes_endpoint(overlay.workspace, overlay.revision(), peer)
768 .is_ok();
769 if authorized && peer != self.id() {
770 if overlay.is_joined() {
771 overlay.join_peer(peer).await?;
772 } else {
773 tracing::info!(target: "data_fabric_transport", peer = %iroh::EndpointId::from_bytes(&peer).map(|p| p.fmt_short().to_string()).unwrap_or_default(), "GOSSIP_OVERLAY_REBUILD_ON_HINT");
774 let peers = self
775 .routing
776 .lock()
777 .await
778 .authorized_endpoints(overlay.workspace, overlay.revision())?;
779 let replacement = overlay::Overlay::prepare(
780 &self.connections,
781 overlay.workspace,
782 overlay.revision(),
783 peers,
784 self.routing.clone(),
785 self.events.clone(),
786 self.membership.clone(),
787 )
788 .await?;
789 self.overlays
790 .lock()
791 .await
792 .insert(overlay.workspace, Arc::new(replacement));
793 }
794 }
795 }
796 Ok(())
797 }
798
799 pub async fn remember_observed(&self, peer: PeerId, address: SocketAddr) {
803 self.connections.remember_observed(peer, address).await;
804 }
805
806 pub async fn address_hint(&self, peer: PeerId) -> Option<SocketAddr> {
809 self.connections.address_hint(peer).await
810 }
811
812 pub async fn nearby_peers(&self) -> Vec<PeerId> {
815 self.connections.nearby_peers(true).await
816 }
817
818 pub async fn nearby_workspace_peers(&self) -> Vec<PeerId> {
820 self.connections.nearby_peers(false).await
821 }
822
823 pub async fn install_verified_policy(
827 &self,
828 workspace: WorkspaceId,
829 revision: u64,
830 endpoint_permissions: BTreeMap<PeerId, Permissions>,
831 ) -> Result<()> {
832 let replace_overlay = self.overlays.lock().await.contains_key(&workspace);
833 let peers = endpoint_permissions.keys().copied().collect::<Vec<_>>();
834 self.routing.lock().await.install_verified_policy(
835 workspace,
836 revision,
837 endpoint_permissions,
838 )?;
839 self.resources.policy_changed();
840 if replace_overlay {
841 if !peers.contains(&self.id()) {
842 self.connections
843 .authorize_gossip(overlay::alpn(workspace), Vec::new())
844 .await;
845 self.overlays.lock().await.remove(&workspace);
846 return Ok(());
847 }
848 self.connections
849 .authorize_gossip(overlay::alpn(workspace), peers.clone())
850 .await;
851 let existing = self.overlays.lock().await.get(&workspace).cloned();
854 if existing.is_some_and(|overlay| overlay.advance(revision, &peers)) {
855 return Ok(());
856 }
857 let candidate = overlay::Overlay::prepare(
858 &self.connections,
859 workspace,
860 revision,
861 peers,
862 self.routing.clone(),
863 self.events.clone(),
864 self.membership.clone(),
865 )
866 .await?;
867 self.overlays
868 .lock()
869 .await
870 .insert(workspace, Arc::new(candidate));
871 }
872 Ok(())
873 }
874
875 pub async fn enable_gossip(&self, workspace: WorkspaceId, revision: u64) -> Result<()> {
878 if self
879 .overlays
880 .lock()
881 .await
882 .get(&workspace)
883 .is_some_and(|overlay| overlay.revision() == revision)
884 {
885 return Ok(());
886 }
887 let peers = self
888 .routing
889 .lock()
890 .await
891 .authorized_endpoints(workspace, revision)?;
892 self.connections
893 .authorize_gossip(overlay::alpn(workspace), peers.clone())
894 .await;
895 let candidate = overlay::Overlay::prepare(
896 &self.connections,
897 workspace,
898 revision,
899 peers,
900 self.routing.clone(),
901 self.events.clone(),
902 self.membership.clone(),
903 )
904 .await?;
905 self.overlays
906 .lock()
907 .await
908 .insert(workspace, Arc::new(candidate));
909 Ok(())
910 }
911
912 pub async fn with_routing_policy<T>(&self, inspect: impl FnOnce(&RoutingTable) -> T) -> T {
916 let routing = self.routing.lock().await;
917 inspect(&routing)
918 }
919
920 pub async fn live_neighbor_count(&self, workspace: WorkspaceId) -> usize {
923 self.overlays
924 .lock()
925 .await
926 .get(&workspace)
927 .map_or(0, |overlay| overlay.neighbor_count())
928 }
929
930 pub async fn live_neighbors(&self, workspace: WorkspaceId) -> Vec<PeerId> {
933 self.overlays
934 .lock()
935 .await
936 .get(&workspace)
937 .map_or_else(Vec::new, |overlay| overlay.neighbors())
938 }
939
940 pub async fn wait_for_gossip_neighbor(&self, workspace: WorkspaceId) -> bool {
943 let Some(overlay) = self.overlays.lock().await.get(&workspace).cloned() else {
944 return false;
945 };
946 overlay.wait_for_neighbor().await
947 }
948
949 pub async fn subscribe(
950 &self,
951 workspace: WorkspaceId,
952 revision: u64,
953 topic: Topic,
954 ) -> Result<AdmissionReport> {
955 self.interest(workspace, revision, topic, true).await
956 }
957
958 pub async fn unsubscribe(
959 &self,
960 workspace: WorkspaceId,
961 revision: u64,
962 topic: Topic,
963 ) -> Result<AdmissionReport> {
964 self.interest(workspace, revision, topic, false).await
965 }
966
967 async fn interest(
968 &self,
969 workspace: WorkspaceId,
970 revision: u64,
971 topic: Topic,
972 subscribe: bool,
973 ) -> Result<AdmissionReport> {
974 self.prepare_interest(workspace, revision, topic, subscribe)
975 .await?
976 .await
977 }
978
979 pub async fn prepare_interest(
983 &self,
984 workspace: WorkspaceId,
985 revision: u64,
986 topic: Topic,
987 subscribe: bool,
988 ) -> Result<impl std::future::Future<Output = Result<AdmissionReport>> + Send + 'static> {
989 let mut peers = {
990 let mut routing = self.routing.lock().await;
991 let peers = routing.publishers(workspace, revision, self.id(), &topic)?;
992 if subscribe {
993 routing.subscribe(workspace, revision, self.id(), topic.clone())?;
994 } else {
995 routing.unsubscribe(workspace, revision, self.id(), &topic)?;
996 }
997 peers
998 };
999 let local = self.id();
1000 let admitted_local = peers.contains(&local);
1001 peers.retain(|peer| *peer != local);
1002 let remote = self.fanout(
1005 peers,
1006 Frame {
1007 workspace,
1008 revision,
1009 topic: topic.as_str().into(),
1010 delivery: DeliveryClass::Critical,
1011 operation: if subscribe {
1012 Operation::Subscribe
1013 } else {
1014 Operation::Unsubscribe
1015 },
1016 },
1017 );
1018 Ok(async move {
1019 let mut report = remote.await?;
1020 if admitted_local {
1021 report.admitted.push(local);
1022 report.admitted.sort_unstable();
1023 }
1024 Ok(report)
1025 })
1026 }
1027
1028 pub async fn publish(
1029 &self,
1030 workspace: WorkspaceId,
1031 revision: u64,
1032 topic: Topic,
1033 payload: Vec<u8>,
1034 ) -> Result<AdmissionReport> {
1035 self.publish_with_class(workspace, revision, topic, DeliveryClass::Critical, payload)
1036 .await
1037 }
1038
1039 pub async fn publish_with_class(
1040 &self,
1041 workspace: WorkspaceId,
1042 revision: u64,
1043 topic: Topic,
1044 delivery: DeliveryClass,
1045 payload: Vec<u8>,
1046 ) -> Result<AdmissionReport> {
1047 if payload.len() > MAX_PAYLOAD {
1048 return Err(Error::TooLarge);
1049 }
1050 let overlay = self.overlays.lock().await.get(&workspace).cloned();
1051 if let Some(overlay) = overlay {
1052 if overlay.revision() != revision {
1053 return Err(Error::Routing(arachne_routing::Error::WrongPolicyRevision));
1054 }
1055 let recipients =
1056 self.routing
1057 .lock()
1058 .await
1059 .recipients(workspace, revision, self.id(), &topic)?;
1060 let mut report = AdmissionReport::default();
1061 if recipients.contains(&self.id()) {
1062 apply(
1063 &self.routing,
1064 &self.events,
1065 self.id(),
1066 self.id(),
1067 self.id(),
1068 Frame {
1069 workspace,
1070 revision,
1071 topic: topic.as_str().into(),
1072 delivery,
1073 operation: Operation::Publish(payload.clone()),
1074 },
1075 )
1076 .await?;
1077 report.admitted.push(self.id());
1078 }
1079 report.queued = overlay
1080 .broadcast(self.id(), &topic, delivery, payload)
1081 .await?;
1082 return Ok(report);
1083 }
1084 let peers = self
1085 .routing
1086 .lock()
1087 .await
1088 .recipients(workspace, revision, self.id(), &topic)?;
1089 self.fanout(
1090 peers,
1091 Frame {
1092 workspace,
1093 revision,
1094 topic: topic.as_str().into(),
1095 delivery,
1096 operation: Operation::Publish(payload),
1097 },
1098 )
1099 .await
1100 }
1101
1102 pub async fn publish_to(
1105 &self,
1106 workspace: WorkspaceId,
1107 revision: u64,
1108 topic: Topic,
1109 endpoints: Vec<PeerId>,
1110 recipients: Vec<[u8; 32]>,
1111 payload: Vec<u8>,
1112 ) -> Result<AdmissionReport> {
1113 self.publish_to_with_class(
1114 workspace,
1115 revision,
1116 topic,
1117 endpoints,
1118 recipients,
1119 DeliveryClass::Critical,
1120 payload,
1121 )
1122 .await
1123 }
1124
1125 #[allow(clippy::too_many_arguments)]
1126 pub async fn publish_to_with_class(
1127 &self,
1128 workspace: WorkspaceId,
1129 revision: u64,
1130 topic: Topic,
1131 endpoints: Vec<PeerId>,
1132 recipients: Vec<[u8; 32]>,
1133 delivery: DeliveryClass,
1134 payload: Vec<u8>,
1135 ) -> Result<AdmissionReport> {
1136 if payload.len() > MAX_PAYLOAD
1137 || recipients.is_empty()
1138 || recipients.len() > MAX_RECIPIENTS
1139 || recipients.windows(2).any(|pair| pair[0] >= pair[1])
1140 {
1141 return Err(Error::Rejected);
1142 }
1143 let peers = self.routing.lock().await.direct_recipients(
1144 workspace,
1145 revision,
1146 self.id(),
1147 &topic,
1148 &endpoints,
1149 )?;
1150 let skipped: Vec<_> = endpoints
1151 .into_iter()
1152 .filter(|peer| peers.binary_search(peer).is_err())
1153 .collect();
1154 let mut report = self
1155 .fanout(
1156 peers,
1157 Frame {
1158 workspace,
1159 revision,
1160 topic: topic.as_str().into(),
1161 delivery,
1162 operation: Operation::DirectPublish {
1163 payload,
1164 recipients,
1165 },
1166 },
1167 )
1168 .await?;
1169 report
1170 .failed
1171 .extend(skipped.into_iter().map(|peer| (peer, Error::NotSubscribed)));
1172 report.failed.sort_unstable_by_key(|(peer, _)| *peer);
1173 Ok(report)
1174 }
1175
1176 fn fanout(
1177 &self,
1178 peers: Vec<PeerId>,
1179 frame: Frame,
1180 ) -> impl std::future::Future<Output = Result<AdmissionReport>> + Send + 'static {
1181 let connections = self.connections.clone();
1182 let routing = self.routing.clone();
1183 let events = self.events.clone();
1184 let local = self.id();
1185 async move {
1186 let bytes = wire::encode(&frame)?;
1187 let mut report = AdmissionReport::default();
1188 let mut pending = stream::iter(peers.into_iter().map(|peer| {
1191 let frame = &frame;
1192 let bytes = &bytes;
1193 let connections = &connections;
1194 let routing = &routing;
1195 let events = &events;
1196 async move {
1197 let result = if peer == local {
1198 match wire::decode(bytes) {
1199 Ok(copy) => apply(routing, events, local, local, local, copy).await,
1200 Err(_) => Err(Error::InvalidFrame),
1201 }
1202 } else {
1203 send_frame(connections, routing, peer, frame, bytes).await
1204 };
1205 (peer, result)
1206 }
1207 }))
1208 .buffer_unordered(16);
1209 while let Some((peer, result)) = pending.next().await {
1210 match result {
1211 Ok(()) => report.admitted.push(peer),
1212 Err(error) => report.failed.push((peer, error)),
1213 }
1214 }
1215 report.admitted.sort_unstable();
1217 report.failed.sort_unstable_by_key(|(peer, _)| *peer);
1218 Ok(report)
1219 }
1220 }
1221
1222 pub async fn close(mut self) {
1223 self.resources.close().await;
1224 self.overlays.lock().await.clear();
1225 self.connections.close().await;
1226 self.listener.abort();
1227 let _ = (&mut self.listener).await;
1228
1229 }
1230}
1231
1232async fn receive_frame(
1233 connection: &iroh::endpoint::Connection,
1234 connections: &Connections,
1235 routing: &Mutex<RoutingTable>,
1236 output: &DeliveryQueue,
1237 address: Option<SocketAddr>,
1238 (send, recv): (
1239 &mut iroh::endpoint::SendStream,
1240 &mut iroh::endpoint::RecvStream,
1241 ),
1242) -> Result<()> {
1243 let sender = *connection.remote_id().as_bytes();
1244 let bytes = recv.read_to_end(MAX_FRAME).await.map_err(transport)?;
1245 tracing::info!(target: "data_fabric_transport", remote = %connection.remote_id(), connection_id = connection.stable_id(), bytes = bytes.len(), "TRANSPORT_FRAME_RECEIVED");
1246 let outcome = match wire::decode::<Frame>(&bytes) {
1247 Ok(frame) => apply(routing, output, connections.id(), sender, sender, frame).await,
1248 Err(_) => Err(Error::InvalidFrame),
1249 };
1250 if outcome.is_ok()
1253 && let Some(address) = address
1254 {
1255 connections.remember_observed(sender, address).await;
1256 }
1257 tracing::info!(target: "data_fabric_transport", remote = %connection.remote_id(), connection_id = connection.stable_id(), ?outcome, "TRANSPORT_ADMISSION");
1258 send.write_all(&[u8::from(outcome.is_ok())])
1259 .await
1260 .map_err(transport)?;
1261 send.finish().map_err(transport)?;
1262 Ok(())
1263}
1264
1265async fn send_frame(
1266 connections: &Connections,
1267 routing: &Mutex<RoutingTable>,
1268 peer: PeerId,
1269 frame: &Frame,
1270 bytes: &[u8],
1271) -> Result<()> {
1272 let key = iroh::PublicKey::from_bytes(&peer).map_err(transport)?;
1273 let mut stage = "connect";
1274 let mut observed_connection = None;
1275 tracing::info!(target: "data_fabric_transport", remote = %key, bytes = bytes.len(), "TRANSPORT_SEND_BEGIN");
1276 let result = tokio::time::timeout(TIMEOUT, async {
1277 let connection = connections.connect(peer, ALPN).await?;
1278 observed_connection = Some(connection.clone());
1279 tracing::info!(target: "data_fabric_transport", remote = %key, connection_id = connection.stable_id(), paths = ?connection.paths(), "TRANSPORT_SEND_CONNECTED");
1280 stage = "open stream";
1281 let (mut send, mut recv) = connection.open_bi().await.map_err(transport)?;
1282 stage = "recheck authorization";
1285 let topic = Topic::new(frame.topic.clone())?;
1286 let routing = routing.lock().await;
1287 let allowed = match &frame.operation {
1288 Operation::Publish(_) => {
1289 routing.recipients(frame.workspace, frame.revision, connections.id(), &topic)?
1290 }
1291 Operation::DirectPublish { .. } => routing.direct_recipients(
1292 frame.workspace,
1293 frame.revision,
1294 connections.id(),
1295 &topic,
1296 &[peer],
1297 )?,
1298 _ => {
1299 routing.publishers(frame.workspace, frame.revision, connections.id(), &topic)?
1300 }
1301 };
1302 if !allowed.contains(&peer) {
1303 return Err(Error::Routing(arachne_routing::Error::Denied));
1304 }
1305 drop(routing);
1306 stage = "write frame";
1307 send.write_all(&[0]).await.map_err(transport)?;
1308 send.write_all(bytes).await.map_err(transport)?;
1309 send.finish().map_err(transport)?;
1310 stage = "read acknowledgment";
1311 let ack = recv.read_to_end(1).await.map_err(transport)?;
1312 if ack == [1] {
1313 Ok(())
1314 } else {
1315 Err(Error::Rejected)
1316 }
1317 })
1318 .await
1319 .map_err(|_| Error::Timeout(stage))
1320 .and_then(|result| result);
1321 tracing::info!(target: "data_fabric_transport", remote = %key, stage, ?result, "TRANSPORT_SEND_END");
1322 if result.is_err() {
1323 tracing::info!(target: "data_fabric_transport", remote = %key,
1324 stats = ?observed_connection.as_ref().map(|connection| connection.stats()),
1325 "TRANSPORT_SEND_STATS");
1326 }
1327 result
1328}
1329
1330impl Drop for Node {
1331 fn drop(&mut self) {
1332 self.resources.stop();
1333 self.events.close();
1334 self.listener.abort();
1335 }
1336}
1337
1338fn transport(error: impl std::fmt::Display) -> Error {
1339 Error::Transport(error.to_string())
1340}
1341
1342async fn apply(
1343 routing: &Mutex<RoutingTable>,
1344 events: &DeliveryQueue,
1345 local: PeerId,
1346 sender: PeerId,
1347 received_from: PeerId,
1348 frame: Frame,
1349) -> Result<()> {
1350 let topic = Topic::new(frame.topic)?;
1351 let delivery = frame.delivery;
1352 let mut routing = routing.lock().await;
1353 match frame.operation {
1354 Operation::Subscribe => {
1355 routing.subscribe(frame.workspace, frame.revision, sender, topic)?
1356 }
1357 Operation::Unsubscribe => {
1358 routing.unsubscribe(frame.workspace, frame.revision, sender, &topic)?
1359 }
1360 Operation::Publish(payload) => {
1361 if payload.len() > MAX_PAYLOAD {
1362 return Err(Error::TooLarge);
1363 }
1364 let recipients = routing.recipients(frame.workspace, frame.revision, sender, &topic)?;
1365 if !recipients.contains(&local) {
1366 return Err(Error::Rejected);
1367 }
1368 events.push(Message {
1369 workspace: frame.workspace,
1370 revision: frame.revision,
1371 sender,
1372 received_from,
1373 topic,
1374 payload,
1375 recipients: Vec::new(),
1376 delivery,
1377 })?;
1378 }
1379 Operation::DirectPublish {
1380 payload,
1381 recipients,
1382 } => {
1383 if payload.len() > MAX_PAYLOAD
1384 || recipients.is_empty()
1385 || recipients.len() > MAX_RECIPIENTS
1386 || recipients.windows(2).any(|pair| pair[0] >= pair[1])
1387 {
1388 return Err(Error::Rejected);
1389 }
1390 let recipient_endpoints = routing.direct_recipients(
1391 frame.workspace,
1392 frame.revision,
1393 sender,
1394 &topic,
1395 &[local],
1396 )?;
1397 if recipient_endpoints != [local] {
1398 return Err(Error::Rejected);
1399 }
1400 events.push(Message {
1401 workspace: frame.workspace,
1402 revision: frame.revision,
1403 sender,
1404 received_from,
1405 topic,
1406 payload,
1407 recipients,
1408 delivery,
1409 })?;
1410 }
1411 }
1412 Ok(())
1413}
1414
1415#[tokio::test]
1416async fn receiver_checks_local_interest_even_if_sender_routes_a_direct_frame() {
1417 let routing = Mutex::new(RoutingTable::default());
1418 let events = DeliveryQueue::default();
1419 let mut received = MessageReceiver {
1420 queue: events.clone(),
1421 };
1422 let topic = Topic::new("streams/opaque").unwrap();
1423 routing
1424 .lock()
1425 .await
1426 .install_verified_policy(
1427 [9; 32],
1428 1,
1429 BTreeMap::from([
1430 ([1; 32], Permissions::AllTopics),
1431 ([2; 32], Permissions::AllTopics),
1432 ]),
1433 )
1434 .unwrap();
1435 let frame = || Frame {
1436 workspace: [9; 32],
1437 revision: 1,
1438 topic: topic.as_str().into(),
1439 delivery: DeliveryClass::Critical,
1440 operation: Operation::DirectPublish {
1441 payload: vec![0, 255],
1442 recipients: vec![[3; 32]],
1443 },
1444 };
1445 assert!(
1446 apply(&routing, &events, [2; 32], [1; 32], [1; 32], frame())
1447 .await
1448 .is_err(),
1449 "receiver admitted a direct frame without local interest"
1450 );
1451 routing
1452 .lock()
1453 .await
1454 .subscribe([9; 32], 1, [2; 32], topic.clone())
1455 .unwrap();
1456 apply(&routing, &events, [2; 32], [1; 32], [1; 32], frame())
1457 .await
1458 .unwrap();
1459 assert_eq!(received.recv().await.unwrap().payload, vec![0, 255]);
1460 routing
1461 .lock()
1462 .await
1463 .unsubscribe([9; 32], 1, [2; 32], &topic)
1464 .unwrap();
1465 assert!(
1466 apply(&routing, &events, [2; 32], [1; 32], [1; 32], frame())
1467 .await
1468 .is_err(),
1469 "stale sender interest defeated local unsubscribe"
1470 );
1471 assert!(received.try_recv().is_err());
1472}