Skip to main content

arachne_node/
control.rs

1//! Bounded host-handled request/reply over authenticated Iroh transport.
2//! Transport identity is not workspace authorization; the host validates payloads.
3use super::*;
4use std::sync::atomic::{AtomicU64, Ordering};
5use tokio::sync::oneshot;
6
7pub(super) const ALPN: &[u8] = b"data-fabric/control/1";
8// Durable host control includes queueing and persistence; data/dial budgets stay short.
9pub(super) const CONTROL_TIMEOUT: Duration = Duration::from_secs(30);
10const MAX_REQUEST: usize = 32 * 1024;
11/// Maximum encoded control reply, for preflight before committing a side effect.
12pub const MAX_CONTROL_REPLY: usize = MAX_FRAME;
13
14/// Answers an inquiry: a control request that asks for committed state and
15/// changes nothing. `None` means "not an inquiry" and passes the request to the
16/// host queue unchanged. It runs off the network workers and may run many at
17/// once, so it must only read. This layer does not know what is being asked;
18/// the responder validates the peer and the payload.
19pub type InquiryResponder = Arc<dyn Fn(PeerId, &[u8]) -> Option<Vec<u8>> + Send + Sync>;
20
21/// One duration series, in microseconds.
22#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize)]
23pub struct Timing {
24    pub count: u64,
25    pub total_us: u64,
26    pub max_us: u64,
27}
28
29/// Where control requests spent their time. `inquiry`: answered by the
30/// inquiry responder, whole service time. `host_wait`: queued until the host took it.
31/// `host_service`: taken by the host until it replied.
32#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize)]
33pub struct ControlTiming {
34    pub inquiry: Timing,
35    pub host_wait: Timing,
36    pub host_service: Timing,
37}
38
39#[derive(Default)]
40struct Series {
41    count: AtomicU64,
42    total_us: AtomicU64,
43    max_us: AtomicU64,
44}
45impl Series {
46    fn add(&self, elapsed: Duration) {
47        let us = elapsed.as_micros().min(u64::MAX as u128) as u64;
48        self.count.fetch_add(1, Ordering::Relaxed);
49        self.total_us.fetch_add(us, Ordering::Relaxed);
50        self.max_us.fetch_max(us, Ordering::Relaxed);
51    }
52    fn get(&self) -> Timing {
53        Timing {
54            count: self.count.load(Ordering::Relaxed),
55            total_us: self.total_us.load(Ordering::Relaxed),
56            max_us: self.max_us.load(Ordering::Relaxed),
57        }
58    }
59}
60
61#[derive(Default)]
62pub(super) struct TimingCounters {
63    inquiry: Series,
64    host_wait: Series,
65    host_service: Series,
66}
67impl TimingCounters {
68    fn snapshot(&self) -> ControlTiming {
69        ControlTiming {
70            inquiry: self.inquiry.get(),
71            host_wait: self.host_wait.get(),
72            host_service: self.host_service.get(),
73        }
74    }
75}
76
77pub struct ControlRequest {
78    peer: PeerId,
79    remote_address: Option<SocketAddr>,
80    payload: Vec<u8>,
81    reply: oneshot::Sender<Vec<u8>>,
82    timing: Arc<TimingCounters>,
83    queued: std::time::Instant,
84    taken: Option<std::time::Instant>,
85}
86impl ControlRequest {
87    /// Observed path only. The caller must validate workspace authorization
88    /// before retaining this as a dial hint.
89    pub fn remote_address(&self) -> Option<SocketAddr> {
90        self.remote_address
91    }
92    pub fn peer(&self) -> PeerId {
93        self.peer
94    }
95    pub fn payload(&self) -> &[u8] {
96        &self.payload
97    }
98    /// Time since the request reached this node's queue.
99    pub fn waited(&self) -> Duration {
100        self.queued.elapsed()
101    }
102
103    pub fn expired(&self) -> bool {
104        self.reply.is_closed()
105    }
106    /// Respond only after application authorization and required durable writes.
107    pub fn respond(self, bytes: Vec<u8>) -> Result<()> {
108        if bytes.len() > MAX_FRAME {
109            return Err(Error::TooLarge);
110        }
111        if let Some(taken) = self.taken {
112            self.timing.host_service.add(taken.elapsed());
113        }
114        self.reply.send(bytes).map_err(|_| Error::Rejected)
115    }
116    /// The host took this request from the queue: its wait ends, its service starts.
117    fn take(mut self) -> Self {
118        if self.taken.is_none() {
119            self.timing.host_wait.add(self.queued.elapsed());
120            self.taken = Some(std::time::Instant::now());
121        }
122        self
123    }
124}
125
126/// The control inbox and its arrival signal travel together, so a request can
127/// never be queued without waking the host.
128#[derive(Clone)]
129pub(super) struct ControlInbox {
130    sender: mpsc::Sender<ControlRequest>,
131    signal: Arc<tokio::sync::Notify>,
132    responder: Arc<std::sync::RwLock<Option<InquiryResponder>>>,
133    timing: Arc<TimingCounters>,
134}
135impl ControlInbox {
136    pub(super) fn new(
137        capacity: usize,
138    ) -> (
139        Self,
140        mpsc::Receiver<ControlRequest>,
141        Arc<tokio::sync::Notify>,
142    ) {
143        let (sender, receiver) = mpsc::channel(capacity);
144        let signal = Arc::new(tokio::sync::Notify::new());
145        (
146            Self {
147                sender,
148                signal: signal.clone(),
149                responder: Arc::default(),
150                timing: Arc::default(),
151            },
152            receiver,
153            signal,
154        )
155    }
156    pub(super) fn set_responder(&self, responder: InquiryResponder) {
157        *self
158            .responder
159            .write()
160            .unwrap_or_else(|error| error.into_inner()) = Some(responder);
161    }
162    pub(super) fn timing(&self) -> ControlTiming {
163        self.timing.snapshot()
164    }
165    /// Let the inquiry responder answer first. It runs on the blocking pool: a
166    /// slow inquiry (history verification) must not hold a network worker.
167    async fn inquire(&self, peer: PeerId, payload: Vec<u8>) -> (Vec<u8>, Option<Vec<u8>>) {
168        let responder = self
169            .responder
170            .read()
171            .unwrap_or_else(|error| error.into_inner())
172            .clone();
173        let Some(responder) = responder else {
174            return (payload, None);
175        };
176        let started = std::time::Instant::now();
177        let answered = tokio::task::spawn_blocking(move || {
178            let answer = responder(peer, &payload);
179            (payload, answer)
180        })
181        .await;
182        match answered {
183            Ok((payload, answer)) => {
184                if answer.is_some() {
185                    self.timing.inquiry.add(started.elapsed());
186                }
187                (payload, answer)
188            }
189            // A responder that panicked answered nothing; the request is lost with it.
190            Err(_) => (Vec::new(), Some(Vec::new())),
191        }
192    }
193    /// `notify_one` stores one permit when nobody waits. One permit is enough:
194    /// the host drains until empty before it waits again.
195    fn offer(&self, request: ControlRequest) -> Result<()> {
196        self.sender
197            .try_send(request)
198            .map_err(|_| Error::Backpressure)?;
199        self.signal.notify_one();
200        Ok(())
201    }
202}
203
204/// A new connection often opens on the relay and moves to a direct path once
205/// holepunching succeeds, tens of milliseconds later. On tablets a reply of
206/// several packets that crossed that move was never read (a 7 KB join reply
207/// written 11 ms after the request, the joiner timing out at 30 s, on every
208/// retry; 2026-09-18). Give a relay-only connection a short window to settle on
209/// a direct path before sending. A peer reachable only through the relay costs
210/// this window once per new connection.
211/// Both ends call it: the requester before sending, the responder before
212/// writing a reply (on tablets the responder's own view of the connection
213/// switched to direct 22 ms before it wrote a 7 KB reply that never arrived).
214/// When a switch happens while waiting, a short grace lets both ends finish
215/// the path handoff before several packets go out on the new path.
216async fn settle_on_direct_path(connection: &iroh::endpoint::Connection) {
217    use futures_util::StreamExt;
218    let direct = |paths: &iroh::endpoint::PathList<'_>| {
219        paths.iter().any(|path| path.is_selected() && path.is_ip())
220    };
221    if direct(&connection.paths()) {
222        return;
223    }
224    let mut stream = connection.paths_stream();
225    let switched = tokio::time::timeout(DIRECT_PATH_SETTLE, async {
226        while let Some(paths) = stream.next().await {
227            if direct(&paths) {
228                return true;
229            }
230        }
231        false
232    })
233    .await
234    .unwrap_or(false);
235    if switched {
236        tokio::time::sleep(PATH_SWITCH_GRACE).await;
237    }
238}
239
240const DIRECT_PATH_SETTLE: Duration = Duration::from_millis(750);
241const PATH_SWITCH_GRACE: Duration = Duration::from_millis(250);
242
243pub(super) async fn receive(
244    connection: &iroh::endpoint::Connection,
245    inbox: &ControlInbox,
246    remote_address: Option<SocketAddr>,
247    (send, recv): (
248        &mut iroh::endpoint::SendStream,
249        &mut iroh::endpoint::RecvStream,
250    ),
251) -> Result<()> {
252    tracing::info!(target: "data_fabric_transport", remote = %connection.remote_id(), "CONTROL_ACCEPT_STREAM");
253    let payload = recv.read_to_end(MAX_REQUEST).await.map_err(transport)?;
254    tracing::info!(target: "data_fabric_transport", bytes = payload.len(), "CONTROL_REQUEST_READ");
255    let (reply, response) = oneshot::channel();
256    // Settle while the reply is prepared, not after: it costs no extra
257    // time when the reply takes longer than the path switch.
258    let settled = settle_on_direct_path(connection);
259    let peer = *connection.remote_id().as_bytes();
260    let (payload, answer) = inbox.inquire(peer, payload).await;
261    let bytes = if let Some(bytes) = answer {
262        if bytes.len() > MAX_FRAME {
263            return Err(Error::TooLarge);
264        }
265        tracing::info!(target: "data_fabric_transport", "CONTROL_INQUIRY_ANSWERED");
266        bytes
267    } else {
268        inbox.offer(ControlRequest {
269            peer,
270            remote_address,
271            payload,
272            reply,
273            timing: inbox.timing.clone(),
274            queued: std::time::Instant::now(),
275            taken: None,
276        })?;
277        tokio::select! {
278            response = response => response.map_err(|_| Error::Rejected)?,
279            _ = send.stopped() => return Err(Error::Rejected),
280        }
281    };
282    settled.await;
283    tracing::info!(target: "data_fabric_transport", bytes = bytes.len(), paths = ?connection.paths(), "CONTROL_REPLY_WRITE");
284    send.write_all(&bytes).await.map_err(transport)?;
285    send.finish().map_err(transport)?;
286    tracing::info!(target: "data_fabric_transport", "CONTROL_REPLY_FINISHED");
287    Ok(())
288}
289
290/// Cloneable outbound control handle. It owns only the connection and
291/// cancellation handles, so a runtime task can finish a multi-request
292/// exchange without borrowing the mutable node queue.
293#[derive(Clone)]
294pub struct ControlClient {
295    connections: super::connections::Connections,
296    control_cancel: watch::Sender<bool>,
297}
298
299impl ControlClient {
300    pub fn request_control(
301        self,
302        peer: PeerId,
303        payload: &[u8],
304    ) -> impl std::future::Future<Output = Result<Vec<u8>>> + Send + 'static {
305        let payload = (payload.len() <= MAX_REQUEST).then(|| payload.to_vec());
306        let connections = self.connections.clone();
307        let mut cancelled = self.control_cancel.subscribe();
308        async move {
309            if *cancelled.borrow() {
310                return Err(Error::Cancelled);
311            }
312            tokio::select! {
313            _ = cancelled.changed() => Err(Error::Cancelled),
314            outcome = async {
315            let payload = payload.ok_or(Error::TooLarge)?;
316            let mut stage = "connect";
317            let mut observed = None;
318            let outcome = tokio::time::timeout(CONTROL_TIMEOUT, async {
319            let connection = tokio::time::timeout(TIMEOUT, connections.connect(peer, ALPN))
320                .await.map_err(|_| Error::Timeout("control connect"))?
321                ?;
322            observed = Some(connection.clone());
323            settle_on_direct_path(&connection).await;
324            stage = "open stream";
325            tracing::info!(target: "data_fabric_transport", paths = ?connection.paths(), "CONTROL_CLIENT_CONNECTED");
326            let (mut send, mut recv) = connection.open_bi().await.map_err(transport)?;
327            stage = "write request";
328            send.write_all(&payload).await.map_err(transport)?;
329            send.finish().map_err(transport)?;
330            stage = "read response";
331            tracing::info!(target: "data_fabric_transport", bytes = payload.len(), "CONTROL_REQUEST_SENT");
332            let reply = recv.read_to_end(MAX_FRAME).await.map_err(transport)?;
333            tracing::info!(target: "data_fabric_transport", bytes = reply.len(), "CONTROL_REPLY_READ");
334            Ok(reply)
335        })
336        .await
337        .map_err(|_| Error::Timeout("control response")).and_then(|result| result);
338            tracing::info!(target: "data_fabric_transport", stage, success = outcome.is_ok(),
339                error = outcome.as_ref().err().map(ToString::to_string), "CONTROL_CLIENT_END");
340            if outcome.is_err()
341                && let Some(connection) = observed
342            {
343                tracing::info!(target: "data_fabric_transport", paths = ?connection.paths(), stats = ?connection.stats(), "CONTROL_CLIENT_FAILURE");
344            }
345            // Sent but no reply: do not reuse this connection for the retry.
346            if outcome.is_err() && matches!(stage, "read response" | "write request") {
347                connections.discard(peer, ALPN).await;
348            }
349            // No control bytes can have left before write_all is entered. Once
350            // writing starts, preserve uncertainty even for a partial write.
351            match outcome {
352                Err(_) if matches!(stage, "connect" | "open stream") => {
353                    Err(Error::ControlNotSent(stage))
354                }
355                other => other,
356            }
357            } => outcome,
358            }
359        }
360    }
361}
362
363impl Node {
364    /// Poll at most eight queued requests, dropping expired requests. No workspace
365    /// membership is implied; the peer is derived from the TLS connection.
366    /// Raised once per queued control request. It carries no data; the caller
367    /// still polls to learn what arrived.
368    pub fn control_signal(&self) -> Arc<tokio::sync::Notify> {
369        self.control_signal.clone()
370    }
371
372    /// True while requests set aside by `poll_control_matching` wait.
373    pub fn has_deferred_controls(&self) -> bool {
374        !self.deferred_controls.is_empty()
375    }
376
377    /// Wake the host for requests already queued, for example ones set aside
378    /// while a membership commit was pending. Arrivals raise this themselves.
379    pub fn rearm_control_signal(&self) {
380        self.control_signal.notify_one();
381    }
382
383    pub fn poll_control(&mut self) -> Option<ControlRequest> {
384        for _ in 0..8 {
385            let request = self
386                .deferred_controls
387                .pop_front()
388                .or_else(|| self.controls.try_recv().ok())?;
389            if !request.expired() {
390                return Some(request.take());
391            }
392        }
393        None
394    }
395
396    /// Answer inquiries from committed state before they reach the host queue.
397    pub fn set_inquiry_responder(&self, responder: InquiryResponder) {
398        self.control_inbox.set_responder(responder);
399    }
400
401    /// Connections closed to make room or refused, device-wide.
402    pub fn connection_capacity(&self) -> super::budget::CapacityCounts {
403        self.connections.capacity_counts()
404    }
405
406    /// Where control requests spent their time since this node started.
407    pub fn control_timing(&self) -> ControlTiming {
408        self.control_inbox.timing()
409    }
410
411    /// Find a request without consuming unrelated control traffic. Used while a
412    /// membership commit is pending so admission intake can continue without
413    /// reordering recovery or membership-control messages.
414    pub fn poll_control_matching(
415        &mut self,
416        matches: impl Fn(&[u8]) -> bool,
417    ) -> Option<ControlRequest> {
418        self.poll_control_first(matches, 8)
419    }
420
421    /// Take the first matching request from up to `depth` newly queued ones
422    /// plus those already set aside; the rest keep their order. For short
423    /// requests that must not wait behind a long queue (ADR 0009).
424    pub fn poll_control_first(
425        &mut self,
426        matches: impl Fn(&[u8]) -> bool,
427        depth: usize,
428    ) -> Option<ControlRequest> {
429        let mut scanned = Vec::with_capacity(16);
430        while let Some(request) = self.deferred_controls.pop_front() {
431            scanned.push(request);
432        }
433        for _ in 0..depth {
434            let Ok(request) = self.controls.try_recv() else {
435                break;
436            };
437            scanned.push(request);
438        }
439        let selected = scanned
440            .iter()
441            .position(|request| !request.expired() && matches(request.payload()))
442            .map(|index| scanned.remove(index).take());
443        self.deferred_controls.extend(scanned);
444        selected
445    }
446
447    pub fn control_client(&self) -> ControlClient {
448        ControlClient {
449            connections: self.connections.clone(),
450            control_cancel: self.control_cancel.clone(),
451        }
452    }
453
454    /// Own the bounded request so a host can spawn it without borrowing its node
455    /// or holding an application session lock across the network wait.
456    pub fn request_control(
457        &self,
458        peer: PeerId,
459        payload: &[u8],
460    ) -> impl std::future::Future<Output = Result<Vec<u8>>> + Send + 'static {
461        self.control_client().request_control(peer, payload)
462    }
463}