sapphire_framework_server/sync/live.rs
1//! The live session tables of a SyncRuntime, one per synced workspace.
2//!
3//! One `LivePeers` per synced workspace: it holds the open sessions to peers, pushes a local
4//! commit to all of them, and forwards what a session receives to every session whose peer
5//! lacks it. The dialing loop and the inbound accept path both end here, at `insert`, so the
6//! two sides of one pair agree on which stream survives — the one rule that keeps two
7//! scheduled dialers from keeping two different streams alive.
8
9use std::collections::{HashMap, HashSet};
10use std::sync::Arc;
11use std::time::Duration;
12
13use grain_id::GrainId;
14use sapphire_framework_session::LiveSession;
15use sapphire_sync::{PathUpdate, Replica};
16use tokio::sync::{Mutex, broadcast};
17
18/// How often the dialer walks the peers.
19pub const DIAL_INTERVAL: Duration = Duration::from_secs(5);
20
21/// How long a dial waits for the bridge to open a stream to a peer.
22///
23/// The bridge waits on iroh's own connect, which against a node that is offline takes far
24/// longer than this; one such peer must not hold up a dial to every other (#228).
25pub const OPEN_TIMEOUT: Duration = Duration::from_secs(10);
26
27/// The longest the dialer stays away from a peer that keeps refusing.
28pub const DIAL_BACKOFF_MAX: Duration = Duration::from_secs(5 * 60);
29
30/// How long the dialer waits after the `attempt`-th failure to a peer.
31///
32/// Doubling from one second: 1, 2, 4, 8, … capped at five minutes. The counter resets when a
33/// session opens, so a peer that is merely slow is not punished for long.
34pub(crate) fn dial_backoff(attempt: u32) -> Duration {
35 let secs = 1u64 << attempt.min(9);
36 Duration::from_secs(secs).min(DIAL_BACKOFF_MAX)
37}
38
39/// The live session table of one workspace.
40pub(crate) struct LivePeers {
41 /// This host's device id, so a pair's two ends can agree on which of their streams
42 /// survives — see [`LivePeers::insert`].
43 me: GrainId,
44 /// One session per peer device, while it is open.
45 sessions: Mutex<HashMap<GrainId, Arc<LiveSession>>>,
46 /// The peers this host is dialing right now.
47 ///
48 /// A dial is started from two places — `sync_now` when a workspace is enabled, and the
49 /// dial loop on its interval — and they can overlap. Two outbound streams to one peer
50 /// would give the pair's two ends a second stream whose fate the one `insert` rule
51 /// cannot settle (both ends would see `outbound`), so a dialer claims a peer before
52 /// opening its stream and releases the claim when the attempt is over; the second
53 /// dialer skips it until the next walk.
54 dialing: Mutex<HashSet<GrainId>>,
55}
56
57impl LivePeers {
58 /// The session table of a host with device id `me`.
59 pub(crate) fn new(me: GrainId) -> LivePeers {
60 LivePeers {
61 me,
62 sessions: Mutex::new(HashMap::new()),
63 dialing: Mutex::new(HashSet::new()),
64 }
65 }
66
67 /// Claim the right to dial `device`, or `false` if a dial is already under way.
68 pub(crate) async fn begin_dial(&self, device: GrainId) -> bool {
69 self.dialing.lock().await.insert(device)
70 }
71
72 /// Release the claim [`begin_dial`](Self::begin_dial) took.
73 pub(crate) async fn end_dial(&self, device: GrainId) {
74 self.dialing.lock().await.remove(&device);
75 }
76
77 /// Send `updates` to every open session whose peer lacks them.
78 ///
79 /// `from` is the device a batch arrived from, when forwarding: it is never sent back
80 /// there. Two rules keep a triangle quiet:
81 ///
82 /// 1. never back to the source (`from` is skipped);
83 /// 2. never what the peer already has (the session's `peer_vv` covers it).
84 ///
85 /// A push that fails is not retried here: the session is dead or the peer is gone, the
86 /// next fan-out or dial pass notices, and the batch rides the next exchange. One slow
87 /// peer does not stall the rest — each push is its own await.
88 pub(crate) async fn fan_out(&self, updates: &[PathUpdate], from: Option<GrainId>) {
89 if updates.is_empty() {
90 return;
91 }
92 let mut sessions = self.sessions.lock().await;
93 // Closed sessions go now: nobody is behind them waiting to be told.
94 sessions.retain(|_, session| session.is_open());
95 for (device, session) in sessions.iter() {
96 if Some(device) == from.as_ref() {
97 continue;
98 }
99 let seen: Vec<_> = updates.iter().map(|u| u.seen.clone()).collect();
100 if seen.iter().all(|v| session.peer_vv().covers(v)) {
101 continue;
102 }
103 if let Err(err) = session.push(updates.to_vec()).await {
104 tracing::debug!(%device, "a live push failed: {err}");
105 }
106 }
107 }
108
109 /// Remember `session` as the open session to `device`, and return its update stream.
110 ///
111 /// `outbound` says which side dialled. Both ends of a pair dial on a schedule, so two
112 /// streams between the same two devices routinely come up at once — and if each end kept
113 /// whichever it stored last, the two ends could keep *different* streams, each one
114 /// holding a session whose far half the peer had already dropped. Both would then read
115 /// EOF and go quiet, and the pair would never settle.
116 ///
117 /// So the two ends agree on a rule both can compute: **the stream whose initiator has
118 /// the lower device id survives**. For the stream A dialled, A sees `outbound` with
119 /// `A < B` — keeps. For the stream B dialled, A sees `!outbound` with `A < B` — drops.
120 /// Same stream, same verdict, either end.
121 ///
122 /// `None` means this stream lost the tie-break: it is dropped here, closing it, and the
123 /// surviving one is already in the table. Subscribing *before* the session is stored is
124 /// what guarantees the caller's reader sees every batch announced from now on.
125 pub(crate) async fn insert(
126 &self,
127 device: GrainId,
128 session: LiveSession,
129 outbound: bool,
130 ) -> Option<broadcast::Receiver<Vec<PathUpdate>>> {
131 if (self.me < device) != outbound {
132 return None;
133 }
134 let updates = session.updates();
135 self.sessions.lock().await.insert(device, Arc::new(session));
136 Some(updates)
137 }
138
139 /// Push to `device`'s session what `replica` has committed and the peer has not seen.
140 ///
141 /// Called once a session has joined the table. A commit that landed after the session's
142 /// exchange but before it joined was carried by no fan-out — there was nothing to fan
143 /// out to yet — and no dial either, since the peer counts as caught up by its live
144 /// session. Without this it would wait for the next commit.
145 pub(crate) async fn catch_up(&self, device: GrainId, replica: &Mutex<Replica>) {
146 let Some(session) = self.sessions.lock().await.get(&device).cloned() else {
147 return;
148 };
149 let delta = match replica.lock().await.delta_for(&session.peer_vv()) {
150 Ok(delta) => delta,
151 Err(err) => {
152 tracing::debug!(%device, "no catch-up for a new live session: {err}");
153 return;
154 }
155 };
156 if delta.is_empty() {
157 return;
158 }
159 if let Err(err) = session.push(delta).await {
160 tracing::debug!(%device, "a live push failed: {err}");
161 }
162 }
163
164 /// The devices with an open session.
165 pub(crate) async fn devices(&self) -> Vec<GrainId> {
166 let sessions = self.sessions.lock().await;
167 sessions
168 .iter()
169 .filter(|(_, session)| session.is_open())
170 .map(|(device, _)| *device)
171 .collect()
172 }
173
174 /// How many open sessions have a peer that acknowledges what it is pushed.
175 pub(crate) async fn acknowledging(&self) -> usize {
176 self.sessions
177 .lock()
178 .await
179 .values()
180 .filter(|session| session.is_open() && session.peer_acknowledges())
181 .count()
182 }
183
184 /// Whether `device` has an open session.
185 pub(crate) async fn has(&self, device: &GrainId) -> bool {
186 self.sessions
187 .lock()
188 .await
189 .get(device)
190 .is_some_and(|session| session.is_open())
191 }
192
193 /// Close every session, as if the connections had been cut.
194 ///
195 /// Dropping a `LiveSession` stops its reader task; the dialer's next walk opens fresh
196 /// ones, which is how "a peer that goes away is redialled" holds.
197 pub(crate) async fn drop_connections(&self) {
198 self.sessions.lock().await.clear();
199 }
200
201 /// Close every session whose device `keep` rejects, as `drop_connections` does for all.
202 pub(crate) async fn retain(&self, keep: impl Fn(&GrainId) -> bool) {
203 self.sessions.lock().await.retain(|device, _| keep(device));
204 }
205}
206
207#[cfg(test)]
208mod tests {
209 use super::*;
210
211 #[test]
212 fn the_dial_backoff_doubles_and_caps_at_five_minutes() {
213 assert_eq!(dial_backoff(0), Duration::from_secs(1));
214 assert_eq!(dial_backoff(1), Duration::from_secs(2));
215 assert_eq!(dial_backoff(2), Duration::from_secs(4));
216 assert_eq!(
217 dial_backoff(3),
218 Duration::from_secs(8),
219 "each failure doubles the wait"
220 );
221 assert_eq!(
222 dial_backoff(4),
223 Duration::from_secs(16),
224 "each failure doubles the wait"
225 );
226 assert_eq!(
227 dial_backoff(9),
228 DIAL_BACKOFF_MAX,
229 "doubling reaches the cap at nine failures"
230 );
231 assert_eq!(
232 dial_backoff(40),
233 DIAL_BACKOFF_MAX,
234 "the cap holds however many failures"
235 );
236 }
237}