Skip to main content

arachne_node/
budget.rs

1//! Host-owned capacity shared by independent workspace endpoints. Permits are
2//! resources, not authorization; cancellation and connection close return them.
3use std::{
4    collections::HashMap,
5    sync::{Arc, Mutex as StdMutex},
6    time::{Duration, Instant},
7};
8
9use iroh::endpoint::{AfterHandshakeOutcome, Connection, EndpointHooks};
10use tokio::sync::{OwnedSemaphorePermit, Semaphore};
11
12use crate::{Error, Result};
13
14/// Clone one budget into all endpoints on a device. Workspace identities and
15/// connection caches remain independent. Data cannot consume the control reserve.
16#[derive(Clone, Debug)]
17pub struct ConnectionBudget {
18    connections: [Arc<Semaphore>; 2],
19    dials: [Arc<Semaphore>; 2],
20    /// Gossip's own dial slots. Shared with data, dials to offline peers held
21    /// every slot and a gossip link to a live member waited behind them
22    /// (tablet HEWN, 50 closed joiners, 2026-09-18).
23    gossip_dials: Arc<Semaphore>,
24    /// Established connections, for making room when a plane is full.
25    tracked: Arc<StdMutex<HashMap<usize, Tracked>>>,
26    evicted: Arc<std::sync::atomic::AtomicU64>,
27    refused: Arc<std::sync::atomic::AtomicU64>,
28    pub(crate) handshakes: Arc<Semaphore>,
29    pub(crate) exchanges: [Arc<Semaphore>; 2],
30}
31
32const GOSSIP_DIALS: usize = 8;
33/// A connection must be this long without an exchange before it can be
34/// closed to make room.
35const EVICTABLE_AFTER: Duration = Duration::from_secs(1);
36/// How long a new connection waits for an evicted one's slot.
37const EVICTION_WAIT: Duration = Duration::from_millis(500);
38
39/// Connections closed to make room, and connections refused for lack of it,
40/// since this device started.
41#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize)]
42pub struct CapacityCounts {
43    pub evicted: u64,
44    pub refused: u64,
45}
46
47#[derive(Debug)]
48struct Tracked {
49    plane: usize,
50    handle: iroh::endpoint::WeakConnectionHandle,
51    last_active: Instant,
52    /// Exchanges in progress on this connection. One is never closed.
53    busy: usize,
54    /// Only incoming exchange connections: never gossip links (HyParView
55    /// manages those) or this node's own outgoing cache.
56    evictable: bool,
57}
58
59/// Marks one exchange on a connection; the connection is busy until it drops.
60pub(crate) struct ExchangeGuard {
61    tracked: Arc<StdMutex<HashMap<usize, Tracked>>>,
62    id: usize,
63}
64
65impl Drop for ExchangeGuard {
66    fn drop(&mut self) {
67        if let Some(entry) = self.tracked.lock().unwrap().get_mut(&self.id) {
68            entry.busy = entry.busy.saturating_sub(1);
69            entry.last_active = Instant::now();
70        }
71    }
72}
73
74impl Default for ConnectionBudget {
75    fn default() -> Self {
76        // Control is short-lived admission/chat coordination traffic. Keep it
77        // bounded, but large enough to queue a shared-link burst without
78        // rejecting peers before the application can drain the inbox.
79        // Control dials: 16. With 4, dials to offline members (up to the dial
80        // timeout each) filled every slot and a live owner got Backpressure on
81        // ~4 of 5 catch-up queries (tablets, 2026-09-18).
82        Self::new([64, 512], [16, 16])
83    }
84}
85
86impl ConnectionBudget {
87    #[cfg(test)]
88    pub(crate) fn control_dials_available(&self) -> usize {
89        self.dials[1].available_permits()
90    }
91
92    pub(crate) fn gossip_dials(&self) -> Arc<Semaphore> {
93        self.gossip_dials.clone()
94    }
95
96    #[cfg(test)]
97    fn with_gossip_dials(mut self, permits: usize) -> Self {
98        self.gossip_dials = Arc::new(Semaphore::new(permits));
99        self
100    }
101
102    fn new(connections: [usize; 2], dials: [usize; 2]) -> Self {
103        Self {
104            connections: connections.map(|n| Arc::new(Semaphore::new(n))),
105            dials: dials.map(|n| Arc::new(Semaphore::new(n))),
106            gossip_dials: Arc::new(Semaphore::new(GOSSIP_DIALS)),
107            tracked: Arc::default(),
108            evicted: Arc::default(),
109            refused: Arc::default(),
110            handshakes: Arc::new(Semaphore::new(512)),
111            exchanges: [32, 512].map(|n| Arc::new(Semaphore::new(n))),
112        }
113    }
114
115    pub(crate) fn capacity_counts(&self) -> CapacityCounts {
116        CapacityCounts {
117            evicted: self.evicted.load(std::sync::atomic::Ordering::Relaxed),
118            refused: self.refused.load(std::sync::atomic::Ordering::Relaxed),
119        }
120    }
121
122    /// An incoming exchange connection may be closed to make room when idle.
123    pub(crate) fn mark_evictable(&self, id: usize) {
124        if let Some(entry) = self.tracked.lock().unwrap().get_mut(&id) {
125            entry.evictable = true;
126        }
127    }
128
129    pub(crate) fn exchange(&self, id: usize) -> ExchangeGuard {
130        if let Some(entry) = self.tracked.lock().unwrap().get_mut(&id) {
131            entry.busy += 1;
132            entry.last_active = Instant::now();
133        }
134        ExchangeGuard {
135            tracked: self.tracked.clone(),
136            id,
137        }
138    }
139
140    /// Close the longest-idle evictable connection of this plane. A device
141    /// full of idle joiner links refused its members (500-joiner run, HEWN,
142    /// 2026-09-19): an idle link now gives way instead.
143    fn evict_idle(&self, plane: usize) -> bool {
144        let now = Instant::now();
145        let victim = {
146            let tracked = self.tracked.lock().unwrap();
147            tracked
148                .values()
149                .filter(|entry| entry.plane == plane && entry.evictable && entry.busy == 0)
150                .filter(|entry| now.saturating_duration_since(entry.last_active) >= EVICTABLE_AFTER)
151                .min_by_key(|entry| entry.last_active)
152                .and_then(|entry| entry.handle.upgrade())
153        };
154        let Some(connection) = victim else {
155            return false;
156        };
157        self.evicted
158            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
159        tracing::info!(target: "data_fabric_transport", remote = %connection.remote_id().fmt_short(), "CONNECTION_EVICTED_IDLE");
160        connection.close(0u32.into(), b"idle; device connection capacity");
161        true
162    }
163
164    pub(crate) fn dial(&self, alpn: &[u8]) -> Result<OwnedSemaphorePermit> {
165        self.dials[plane(alpn)]
166            .clone()
167            .try_acquire_owned()
168            .map_err(|_| Error::Backpressure)
169    }
170}
171
172fn plane(alpn: &[u8]) -> usize {
173    usize::from(alpn == crate::control::ALPN)
174}
175
176impl EndpointHooks for ConnectionBudget {
177    async fn after_handshake<'a>(&'a self, connection: &'a Connection) -> AfterHandshakeOutcome {
178        let plane = plane(connection.alpn());
179        let slots = self.connections[plane].clone();
180        let permit = match slots.clone().try_acquire_owned() {
181            Ok(permit) => permit,
182            Err(_) => {
183                let waited = if self.evict_idle(plane) {
184                    tokio::time::timeout(EVICTION_WAIT, slots.acquire_owned())
185                        .await
186                        .ok()
187                        .and_then(|permit| permit.ok())
188                } else {
189                    None
190                };
191                match waited {
192                    Some(permit) => permit,
193                    None => {
194                        self.refused
195                            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
196                        tracing::info!(target: "data_fabric_transport", remote = %connection.remote_id().fmt_short(), plane, "CONNECTION_REFUSED_CAPACITY");
197                        return AfterHandshakeOutcome::Reject {
198                            error_code: 429u32.into(),
199                            reason: b"device connection capacity".to_vec(),
200                        };
201                    }
202                }
203            }
204        };
205        let id = connection.stable_id();
206        let handle = connection.weak_handle();
207        self.tracked.lock().unwrap().insert(
208            id,
209            Tracked {
210                plane,
211                handle: handle.clone(),
212                last_active: Instant::now(),
213                busy: 0,
214                evictable: false,
215            },
216        );
217        // Iroh's weak close future does not keep an otherwise idle connection
218        // alive. No endpoint reference cycle or strong observer ownership.
219        let closed = handle.closed();
220        let tracked = self.tracked.clone();
221        tokio::spawn(async move {
222            let _permit = permit;
223            closed.await;
224            tracked.lock().unwrap().remove(&id);
225        });
226        AfterHandshakeOutcome::Accept
227    }
228}
229
230#[cfg(test)]
231mod tests {
232    use super::*;
233    use crate::{NetworkProfile, connections::Connections};
234    use iroh::{Endpoint, EndpointAddr, endpoint::presets};
235    use std::time::Duration;
236
237    async fn full_server() -> crate::Node {
238        // One incoming control connection fits; the next must make room.
239        crate::Node::bind_with_profile(
240            "127.0.0.1:0".parse().unwrap(),
241            None,
242            NetworkProfile::Direct,
243            ConnectionBudget::new([4, 1], [4, 4]),
244        )
245        .await
246        .unwrap()
247        .0
248    }
249
250    async fn serve_one(server: &mut crate::Node) {
251        loop {
252            if let Some(request) = server.poll_control() {
253                request.respond(vec![1]).unwrap();
254                return;
255            }
256            tokio::time::sleep(Duration::from_millis(5)).await;
257        }
258    }
259
260    /// A full device refused every new peer while idle joiner links held all
261    /// control slots: members JOSA and BIG RED could not reach the owner during
262    /// a 500-joiner run (2026-09-19). The longest-idle link now makes room.
263    #[tokio::test]
264    async fn a_new_peer_takes_the_slot_of_the_longest_idle_connection() {
265        tokio::time::timeout(Duration::from_secs(20), async {
266            let mut server = full_server().await;
267            let (idle, _) = crate::Node::bind("127.0.0.1:0".parse().unwrap())
268                .await
269                .unwrap();
270            let (newcomer, _) = crate::Node::bind("127.0.0.1:0".parse().unwrap())
271                .await
272                .unwrap();
273            for client in [&idle, &newcomer] {
274                client
275                    .add_address_hint(server.id(), server.address())
276                    .await
277                    .unwrap();
278            }
279            let first = tokio::spawn(idle.request_control(server.id(), b"first"));
280            serve_one(&mut server).await;
281            first.await.unwrap().unwrap();
282            tokio::time::sleep(Duration::from_millis(1300)).await;
283            let second = tokio::spawn(newcomer.request_control(server.id(), b"second"));
284            serve_one(&mut server).await;
285            assert_eq!(
286                second.await.unwrap().unwrap(),
287                vec![1],
288                "the new peer was refused"
289            );
290            // A device run must be able to tell that eviction happened.
291            assert_eq!(
292                server.connection_capacity(),
293                crate::CapacityCounts {
294                    evicted: 1,
295                    refused: 0
296                }
297            );
298        })
299        .await
300        .unwrap();
301    }
302
303    /// A link that still owes a reply is never closed to make room, however
304    /// long the host takes: the owner held joiner requests for up to 10 s.
305    #[tokio::test]
306    async fn a_connection_waiting_for_its_reply_is_never_evicted() {
307        tokio::time::timeout(Duration::from_secs(20), async {
308            let mut server = full_server().await;
309            let (waiting, _) = crate::Node::bind("127.0.0.1:0".parse().unwrap())
310                .await
311                .unwrap();
312            let (newcomer, _) = crate::Node::bind("127.0.0.1:0".parse().unwrap())
313                .await
314                .unwrap();
315            for client in [&waiting, &newcomer] {
316                client
317                    .add_address_hint(server.id(), server.address())
318                    .await
319                    .unwrap();
320            }
321            let pending = tokio::spawn(waiting.request_control(server.id(), b"pending"));
322            let request = loop {
323                if let Some(request) = server.poll_control() {
324                    break request;
325                }
326                tokio::time::sleep(Duration::from_millis(5)).await;
327            };
328            tokio::time::sleep(Duration::from_millis(1300)).await;
329            assert!(
330                newcomer
331                    .request_control(server.id(), b"second")
332                    .await
333                    .is_err(),
334                "a busy link was closed to make room"
335            );
336            assert_eq!(server.connection_capacity().refused, 1);
337            request.respond(vec![7]).unwrap();
338            assert_eq!(pending.await.unwrap().unwrap(), vec![7]);
339        })
340        .await
341        .unwrap();
342    }
343
344    #[tokio::test]
345    async fn native_gossip_dials_use_their_own_capacity_and_release_on_cancel_and_success() {
346        tokio::time::timeout(Duration::from_secs(10), async {
347            let budget = ConnectionBudget::new([4, 1], [1, 1]).with_gossip_dials(1);
348            let node = Connections::bind(
349                "127.0.0.1:0".parse().unwrap(),
350                NetworkProfile::Direct,
351                None,
352                budget.clone(),
353                vec![],
354                None,
355            )
356            .await
357            .unwrap();
358            let alpn = b"arachne/workspace-gossip/1/dial-test";
359            let peer = Endpoint::builder(presets::Minimal)
360                .clear_relay_transports()
361                .clear_ip_transports()
362                .bind_addr("127.0.0.1:0")
363                .unwrap()
364                .alpns(vec![alpn.to_vec()])
365                .bind()
366                .await
367                .unwrap();
368            let id = *peer.id().as_bytes();
369            node.add_address_hint(id, peer.bound_sockets()[0])
370                .await
371                .unwrap();
372            let blackhole = std::net::UdpSocket::bind("127.0.0.1:0").unwrap();
373            let unreachable = iroh::SecretKey::from_bytes(&[223; 32]).public();
374            node.add_address_hint(*unreachable.as_bytes(), blackhole.local_addr().unwrap())
375                .await
376                .unwrap();
377            node.authorize_gossip(alpn.to_vec(), vec![id, *unreachable.as_bytes()])
378                .await;
379            let spawn_gossip = || {
380                iroh_gossip::net::Gossip::builder()
381                    .alpn(alpn)
382                    .dial_capacity(node.gossip_dial_capacity())
383                    .spawn(node.endpoint())
384            };
385            let topic = iroh_gossip::TopicId::from_bytes([4; 32]);
386            let reserved = budget.gossip_dials().try_acquire_owned().unwrap();
387            // Data dials never wait on, or take, gossip slots.
388            let data = budget.dial(crate::ALPN).unwrap();
389            let queued = spawn_gossip();
390            let queued_topic = queued.subscribe(topic, vec![peer.id()]).await.unwrap();
391            assert!(
392                tokio::time::timeout(Duration::from_millis(50), peer.accept())
393                    .await
394                    .is_err(),
395                "gossip cannot emit a handshake while its own dial slot is held"
396            );
397            queued.shutdown().await.unwrap();
398            drop(queued_topic);
399            drop(queued);
400            drop(reserved);
401            drop(data);
402            let gossip = spawn_gossip();
403            let subscription = gossip.subscribe(topic, vec![peer.id()]).await.unwrap();
404            let connected = peer.accept().await.unwrap().await.unwrap();
405            while budget.gossip_dials.available_permits() == 0 {
406                tokio::task::yield_now().await;
407            }
408            assert!(connected.close_reason().is_none());
409            gossip.shutdown().await.unwrap();
410            drop(subscription);
411            drop(gossip);
412            let cancelled = spawn_gossip();
413            let subscription = cancelled.subscribe(topic, vec![unreachable]).await.unwrap();
414            while budget.gossip_dials.available_permits() != 0 {
415                tokio::task::yield_now().await;
416            }
417            // A gossip dial to an offline peer leaves data and control free.
418            assert!(budget.dial(crate::ALPN).is_ok());
419            assert!(budget.dial(crate::control::ALPN).is_ok());
420            cancelled.shutdown().await.unwrap();
421            drop(subscription);
422            drop(cancelled);
423            while budget.gossip_dials.available_permits() == 0 {
424                tokio::task::yield_now().await;
425            }
426            node.close().await;
427            peer.close().await;
428            assert_eq!(budget.gossip_dials.available_permits(), 1);
429        })
430        .await
431        .unwrap();
432    }
433
434    #[tokio::test]
435    async fn independent_endpoints_share_capacity_and_control_reserve_without_retaining_links() {
436        tokio::time::timeout(Duration::from_secs(10), async {
437            let budget = ConnectionBudget::new([1, 1], [1, 1]);
438            let dial = budget.dial(crate::ALPN).unwrap();
439            assert!(budget.clone().dial(crate::ALPN).is_err());
440            assert!(budget.dial(crate::control::ALPN).is_ok());
441            drop(dial);
442            let first = Connections::bind(
443                "127.0.0.1:0".parse().unwrap(),
444                NetworkProfile::Direct,
445                None,
446                budget.clone(),
447                vec![],
448                None,
449            )
450            .await
451            .unwrap();
452            let second = Connections::bind(
453                "127.0.0.1:0".parse().unwrap(),
454                NetworkProfile::Direct,
455                None,
456                budget.clone(),
457                vec![],
458                None,
459            )
460            .await
461            .unwrap();
462            assert_ne!(
463                first.id(),
464                second.id(),
465                "capacity must not unify identities"
466            );
467            let gossip = b"arachne/workspace-gossip/1/budget-test";
468            let peer = Endpoint::builder(presets::Minimal)
469                .clear_relay_transports()
470                .clear_ip_transports()
471                .bind_addr("127.0.0.1:0")
472                .unwrap()
473                .alpns(vec![
474                    gossip.to_vec(),
475                    crate::ALPN.to_vec(),
476                    crate::control::ALPN.to_vec(),
477                ])
478                .bind()
479                .await
480                .unwrap();
481            let id = *peer.id().as_bytes();
482            first.authorize_gossip(gossip.to_vec(), vec![id]).await;
483            second
484                .add_address_hint(id, peer.bound_sockets()[0])
485                .await
486                .unwrap();
487            // Native gossip bypasses the request cache, but not the endpoint's
488            // shared established-connection gate.
489            let endpoint = first.endpoint();
490            let (gossip_link, remote) = tokio::join!(
491                endpoint.connect(
492                    EndpointAddr::new(peer.id()).with_ip_addr(peer.bound_sockets()[0]),
493                    gossip
494                ),
495                async { peer.accept().await.unwrap().await }
496            );
497            let gossip_link = gossip_link.unwrap();
498            let remote = remote.unwrap();
499            assert_eq!(budget.connections[0].available_permits(), 0);
500            let (rejected, _) = tokio::join!(second.connect(id, crate::ALPN), async {
501                peer.accept().await.unwrap().await
502            });
503            assert!(
504                rejected.is_err(),
505                "another endpoint must not get a fresh budget"
506            );
507            let (control, remote_control) =
508                tokio::join!(second.connect(id, crate::control::ALPN), async {
509                    peer.accept().await.unwrap().await
510                });
511            let control = control.unwrap();
512            let _remote_control = remote_control.unwrap();
513            assert!(
514                control.close_reason().is_none(),
515                "gossip/data cannot consume control capacity"
516            );
517            // Drop, do not explicitly close: the budget observer must not keep
518            // this native link alive merely to account for it.
519            drop(gossip_link);
520            remote.closed().await;
521            while budget.connections[0].available_permits() == 0 {
522                tokio::task::yield_now().await;
523            }
524            // A cancelled unreachable dial returns capacity immediately, even
525            // though Iroh's own connection timeout has not fired.
526            let blackhole = std::net::UdpSocket::bind("127.0.0.1:0").unwrap();
527            let unreachable = *iroh::SecretKey::from_bytes(&[222; 32]).public().as_bytes();
528            first
529                .add_address_hint(unreachable, blackhole.local_addr().unwrap())
530                .await
531                .unwrap();
532            let caller = first.clone();
533            let pending =
534                tokio::spawn(async move { caller.connect(unreachable, crate::ALPN).await });
535            while budget.dials[0].available_permits() != 0 {
536                tokio::task::yield_now().await;
537            }
538            assert!(matches!(
539                second.connect(id, crate::ALPN).await,
540                Err(Error::Backpressure)
541            ));
542            pending.abort();
543            assert!(pending.await.unwrap_err().is_cancelled());
544            assert_eq!(budget.dials[0].available_permits(), 1);
545            first
546                .add_address_hint(first.id(), first.address())
547                .await
548                .unwrap();
549            assert!(first.connect(first.id(), crate::ALPN).await.is_err());
550            assert_eq!(
551                budget.dials[0].available_permits(),
552                1,
553                "failed self-dial returns its slot"
554            );
555            let (data, remote_data) = tokio::join!(second.connect(id, crate::ALPN), async {
556                peer.accept().await.unwrap().await
557            });
558            let data = data.unwrap();
559            let _remote_data = remote_data.unwrap();
560            first.close().await;
561            assert!(
562                data.close_reason().is_none(),
563                "closing one workspace must not close another"
564            );
565            second.close().await;
566            peer.close().await;
567            while budget
568                .connections
569                .iter()
570                .any(|s| s.available_permits() == 0)
571            {
572                tokio::task::yield_now().await;
573            }
574            assert_eq!(budget.dials[0].available_permits(), 1);
575            assert_eq!(budget.dials[1].available_permits(), 1);
576        })
577        .await
578        .unwrap();
579    }
580}