Skip to main content

sapphire_framework_server/sync/
live.rs

1//! The live session tables of a SyncRuntime, one per synced workspace.
2//!
3//! One `LivePeers` per synced workspace: it holds the open sessions to peers, pushes a local
4//! commit to all of them, and forwards what a session receives to every session whose peer
5//! lacks it. The dialing loop and the inbound accept path both end here, at `insert`, so the
6//! two sides of one pair agree on which stream survives — the one rule that keeps two
7//! scheduled dialers from keeping two different streams alive.
8
9use std::collections::{HashMap, HashSet};
10use std::sync::Arc;
11use std::time::Duration;
12
13use grain_id::GrainId;
14use sapphire_framework_session::LiveSession;
15use sapphire_sync::{PathUpdate, Replica};
16use tokio::sync::{Mutex, broadcast};
17
18/// How often the dialer walks the peers.
19pub const DIAL_INTERVAL: Duration = Duration::from_secs(5);
20
21/// How long a dial waits for the bridge to open a stream to a peer.
22///
23/// The bridge waits on iroh's own connect, which against a node that is offline takes far
24/// longer than this; one such peer must not hold up a dial to every other (#228).
25pub const OPEN_TIMEOUT: Duration = Duration::from_secs(10);
26
27/// The longest the dialer stays away from a peer that keeps refusing.
28pub const DIAL_BACKOFF_MAX: Duration = Duration::from_secs(5 * 60);
29
30/// How long the dialer waits after the `attempt`-th failure to a peer.
31///
32/// Doubling from one second: 1, 2, 4, 8, … capped at five minutes. The counter resets when a
33/// session opens, so a peer that is merely slow is not punished for long.
34pub(crate) fn dial_backoff(attempt: u32) -> Duration {
35    let secs = 1u64 << attempt.min(9);
36    Duration::from_secs(secs).min(DIAL_BACKOFF_MAX)
37}
38
39/// The live session table of one workspace.
40pub(crate) struct LivePeers {
41    /// This host's device id, so a pair's two ends can agree on which of their streams
42    /// survives — see [`LivePeers::insert`].
43    me: GrainId,
44    /// One session per peer device, while it is open.
45    sessions: Mutex<HashMap<GrainId, Arc<LiveSession>>>,
46    /// The peers this host is dialing right now.
47    ///
48    /// A dial is started from two places — `sync_now` when a workspace is enabled, and the
49    /// dial loop on its interval — and they can overlap. Two outbound streams to one peer
50    /// would give the pair's two ends a second stream whose fate the one `insert` rule
51    /// cannot settle (both ends would see `outbound`), so a dialer claims a peer before
52    /// opening its stream and releases the claim when the attempt is over; the second
53    /// dialer skips it until the next walk.
54    dialing: Mutex<HashSet<GrainId>>,
55}
56
57impl LivePeers {
58    /// The session table of a host with device id `me`.
59    pub(crate) fn new(me: GrainId) -> LivePeers {
60        LivePeers {
61            me,
62            sessions: Mutex::new(HashMap::new()),
63            dialing: Mutex::new(HashSet::new()),
64        }
65    }
66
67    /// Claim the right to dial `device`, or `false` if a dial is already under way.
68    pub(crate) async fn begin_dial(&self, device: GrainId) -> bool {
69        self.dialing.lock().await.insert(device)
70    }
71
72    /// Release the claim [`begin_dial`](Self::begin_dial) took.
73    pub(crate) async fn end_dial(&self, device: GrainId) {
74        self.dialing.lock().await.remove(&device);
75    }
76
77    /// Send `updates` to every open session whose peer lacks them.
78    ///
79    /// `from` is the device a batch arrived from, when forwarding: it is never sent back
80    /// there. Two rules keep a triangle quiet:
81    ///
82    /// 1. never back to the source (`from` is skipped);
83    /// 2. never what the peer already has (the session's `peer_vv` covers it).
84    ///
85    /// A push that fails is not retried here: the session is dead or the peer is gone, the
86    /// next fan-out or dial pass notices, and the batch rides the next exchange. One slow
87    /// peer does not stall the rest — each push is its own await.
88    pub(crate) async fn fan_out(&self, updates: &[PathUpdate], from: Option<GrainId>) {
89        if updates.is_empty() {
90            return;
91        }
92        let mut sessions = self.sessions.lock().await;
93        // Closed sessions go now: nobody is behind them waiting to be told.
94        sessions.retain(|_, session| session.is_open());
95        for (device, session) in sessions.iter() {
96            if Some(device) == from.as_ref() {
97                continue;
98            }
99            let seen: Vec<_> = updates.iter().map(|u| u.seen.clone()).collect();
100            if seen.iter().all(|v| session.peer_vv().covers(v)) {
101                continue;
102            }
103            if let Err(err) = session.push(updates.to_vec()).await {
104                tracing::debug!(%device, "a live push failed: {err}");
105            }
106        }
107    }
108
109    /// Remember `session` as the open session to `device`, and return its update stream.
110    ///
111    /// `outbound` says which side dialled. Both ends of a pair dial on a schedule, so two
112    /// streams between the same two devices routinely come up at once — and if each end kept
113    /// whichever it stored last, the two ends could keep *different* streams, each one
114    /// holding a session whose far half the peer had already dropped. Both would then read
115    /// EOF and go quiet, and the pair would never settle.
116    ///
117    /// So the two ends agree on a rule both can compute: **the stream whose initiator has
118    /// the lower device id survives**. For the stream A dialled, A sees `outbound` with
119    /// `A < B` — keeps. For the stream B dialled, A sees `!outbound` with `A < B` — drops.
120    /// Same stream, same verdict, either end.
121    ///
122    /// `None` means this stream lost the tie-break: it is dropped here, closing it, and the
123    /// surviving one is already in the table. Subscribing *before* the session is stored is
124    /// what guarantees the caller's reader sees every batch announced from now on.
125    pub(crate) async fn insert(
126        &self,
127        device: GrainId,
128        session: LiveSession,
129        outbound: bool,
130    ) -> Option<broadcast::Receiver<Vec<PathUpdate>>> {
131        if (self.me < device) != outbound {
132            return None;
133        }
134        let updates = session.updates();
135        self.sessions.lock().await.insert(device, Arc::new(session));
136        Some(updates)
137    }
138
139    /// Push to `device`'s session what `replica` has committed and the peer has not seen.
140    ///
141    /// Called once a session has joined the table. A commit that landed after the session's
142    /// exchange but before it joined was carried by no fan-out — there was nothing to fan
143    /// out to yet — and no dial either, since the peer counts as caught up by its live
144    /// session. Without this it would wait for the next commit.
145    pub(crate) async fn catch_up(&self, device: GrainId, replica: &Mutex<Replica>) {
146        let Some(session) = self.sessions.lock().await.get(&device).cloned() else {
147            return;
148        };
149        let delta = match replica.lock().await.delta_for(&session.peer_vv()) {
150            Ok(delta) => delta,
151            Err(err) => {
152                tracing::debug!(%device, "no catch-up for a new live session: {err}");
153                return;
154            }
155        };
156        if delta.is_empty() {
157            return;
158        }
159        if let Err(err) = session.push(delta).await {
160            tracing::debug!(%device, "a live push failed: {err}");
161        }
162    }
163
164    /// The devices with an open session.
165    pub(crate) async fn devices(&self) -> Vec<GrainId> {
166        let sessions = self.sessions.lock().await;
167        sessions
168            .iter()
169            .filter(|(_, session)| session.is_open())
170            .map(|(device, _)| *device)
171            .collect()
172    }
173
174    /// How many open sessions have a peer that acknowledges what it is pushed.
175    pub(crate) async fn acknowledging(&self) -> usize {
176        self.sessions
177            .lock()
178            .await
179            .values()
180            .filter(|session| session.is_open() && session.peer_acknowledges())
181            .count()
182    }
183
184    /// Whether `device` has an open session.
185    pub(crate) async fn has(&self, device: &GrainId) -> bool {
186        self.sessions
187            .lock()
188            .await
189            .get(device)
190            .is_some_and(|session| session.is_open())
191    }
192
193    /// Close every session, as if the connections had been cut.
194    ///
195    /// Dropping a `LiveSession` stops its reader task; the dialer's next walk opens fresh
196    /// ones, which is how "a peer that goes away is redialled" holds.
197    pub(crate) async fn drop_connections(&self) {
198        self.sessions.lock().await.clear();
199    }
200
201    /// Close every session whose device `keep` rejects, as `drop_connections` does for all.
202    pub(crate) async fn retain(&self, keep: impl Fn(&GrainId) -> bool) {
203        self.sessions.lock().await.retain(|device, _| keep(device));
204    }
205}
206
207#[cfg(test)]
208mod tests {
209    use super::*;
210
211    #[test]
212    fn the_dial_backoff_doubles_and_caps_at_five_minutes() {
213        assert_eq!(dial_backoff(0), Duration::from_secs(1));
214        assert_eq!(dial_backoff(1), Duration::from_secs(2));
215        assert_eq!(dial_backoff(2), Duration::from_secs(4));
216        assert_eq!(
217            dial_backoff(3),
218            Duration::from_secs(8),
219            "each failure doubles the wait"
220        );
221        assert_eq!(
222            dial_backoff(4),
223            Duration::from_secs(16),
224            "each failure doubles the wait"
225        );
226        assert_eq!(
227            dial_backoff(9),
228            DIAL_BACKOFF_MAX,
229            "doubling reaches the cap at nine failures"
230        );
231        assert_eq!(
232            dial_backoff(40),
233            DIAL_BACKOFF_MAX,
234            "the cap holds however many failures"
235        );
236    }
237}