sapphire_framework_session/live.rs
1//! The phase after `Done`: a session that stays open and pushes what is committed.
2//!
3//! [`open_live_session`] runs the same initial exchange as [`crate::run_session`] and then,
4//! instead of closing the stream, keeps reading it. `Live` messages are applied as they
5//! arrive and published on a broadcast, and [`LiveSession::push`] sends a batch through the
6//! same writer task the exchange used. The two ends of one session are two independent
7//! `LiveSession`s: each side pushes what it commits and applies what arrives, and neither is
8//! a client.
9//!
10//! Content does not travel inside a `Live` message. A batch that names a file this side does
11//! not have is answered with `Want` on the same stream, and the peer's content settles it
12//! through the machinery the exchange already uses: the same writer streams it, and the
13//! same `Incoming` stages it. A batch is published only once every
14//! hash it was waiting on has been answered, so a host that forwards what it receives never
15//! announces an entry it cannot serve.
16//!
17//! # Acknowledgements
18//!
19//! A `Live` push returns once it is queued, which says nothing about whether it arrived. So
20//! a peer that said `acks` in its `Hello` answers every batch it has applied — content
21//! included — with [`Message::Ack`], and [`LiveSession::acknowledged`] follows what the peer
22//! is known to have. A peer from before acknowledgements never sends one; its session still
23//! works, and only the exchange and its own pushes advance what it is known to have.
24//!
25//! # A dead peer must not stall the others
26//!
27//! The write half belongs to one task fed by unbounded queues (see `writer.rs` for why), so
28//! queueing for a peer that stopped reading never waits: [`LiveSession::push`] returns at
29//! once, and fails once the session has closed, so a caller looping over peers moves on to
30//! the next one instead of waiting on one that went away.
31
32use std::collections::HashSet;
33use std::sync::Arc;
34use std::sync::atomic::{AtomicBool, Ordering};
35
36use grain_id::GrainId;
37use sapphire_sync::{ContentHash, PathUpdate, Replica, VersionVector};
38use tokio::io::{AsyncRead, AsyncWrite};
39use tokio::sync::{broadcast, watch};
40use tokio::task::JoinHandle;
41
42use crate::error::{Error, Result};
43use crate::frame::{Frame, read_frame};
44use crate::message::Message;
45use crate::session::{self, Exchange, NoContent, SessionOutcome, Stop, send};
46use crate::writer::{Out, Outbox};
47
48/// How many batches may wait for a subscriber before the oldest is dropped. Applies to the
49/// broadcast of received updates, not to the wire.
50const LIVE_QUEUE: usize = 64;
51
52/// A session that stays open after the initial exchange.
53///
54/// Dropping it closes the session; `close` is the same thing said out loud.
55pub struct LiveSession {
56 out: Outbox,
57 updates: broadcast::Sender<Vec<PathUpdate>>,
58 peer_vv: Arc<watch::Sender<VersionVector>>,
59 peer_acks: bool,
60 open: Arc<AtomicBool>,
61 reader: JoinHandle<()>,
62}
63
64impl LiveSession {
65 /// Send updates to the peer.
66 ///
67 /// Fails rather than waiting once the session has closed, so a caller looping over
68 /// peers does not stall on one that went away.
69 pub async fn push(&self, updates: Vec<PathUpdate>) -> Result<()> {
70 if !self.is_open() {
71 return Err(Error::Protocol("the session is closed".to_owned()));
72 }
73 self.out
74 .send(Out::Control(Message::Live(updates)))
75 .map_err(|_| Error::Protocol("the session is closed".to_owned()))
76 }
77
78 /// Updates that arrived from the peer, after they were applied.
79 ///
80 /// A batch is published once every hash it was waiting on has been answered — with the
81 /// bytes, or with `Missing` — so a subscriber forwarding it forwards what this host can
82 /// answer for.
83 pub fn updates(&self) -> broadcast::Receiver<Vec<PathUpdate>> {
84 self.updates.subscribe()
85 }
86
87 /// Everything the peer is known to have.
88 ///
89 /// Consulted before forwarding: an entry the peer already covers must not be sent again,
90 /// or three connected hosts will pass one edit round in a circle. It advances as the
91 /// peer's batches arrive, because each one carries what the peer had when it sent it.
92 ///
93 /// It also advances when the peer acknowledges a push (see the module docs), so it
94 /// covers this side's own entries once they have reached the peer.
95 pub fn peer_vv(&self) -> VersionVector {
96 self.peer_vv.borrow().clone()
97 }
98
99 /// [`peer_vv`](Self::peer_vv) as it changes: marked changed whenever an arriving batch or
100 /// an acknowledgement advances it. Closed once the session is.
101 ///
102 /// Starts at what the initial exchange left: what the peer said it had, plus everything
103 /// this side sent it then — the exchange completes only once the peer has all of that.
104 pub fn acknowledged(&self) -> watch::Receiver<VersionVector> {
105 self.peer_vv.subscribe()
106 }
107
108 /// Whether the peer acknowledges pushes. `false` for a peer from before
109 /// acknowledgements: what this side pushes to it never shows in
110 /// [`peer_vv`](Self::peer_vv) until a later exchange.
111 pub fn peer_acknowledges(&self) -> bool {
112 self.peer_acks
113 }
114
115 /// Is the session still usable?
116 pub fn is_open(&self) -> bool {
117 self.open.load(Ordering::Relaxed)
118 }
119
120 /// Close it.
121 pub fn close(self) {
122 drop(self);
123 }
124}
125
126impl Drop for LiveSession {
127 fn drop(&mut self) {
128 self.open.store(false, Ordering::Relaxed);
129 self.reader.abort();
130 }
131}
132
133/// A batch that arrived and is waiting for content before it can be announced.
134struct Pending {
135 /// What the peer sent.
136 batch: Vec<PathUpdate>,
137 /// The hashes this batch is still waiting on. Empty means it is ready to publish.
138 outstanding: HashSet<ContentHash>,
139}
140
141/// Run the initial exchange and then keep the session open.
142///
143/// Both sides call this: each sends what the other lacks during the exchange and afterwards
144/// pushes what it commits, so the returned [`LiveSession`] is the same handle on either end
145/// of the stream. The [`SessionOutcome`] is returned as soon as both sides have said `Done`
146/// and `Settled` — "we are caught up" — while the session itself keeps going until a side
147/// closes it or the stream ends.
148///
149/// The replica is locked per step — never while waiting for the peer's `Hello` (issue #163)
150/// or for content in flight: for each page and `Want` of the exchange, and then for each
151/// arriving batch — so a scan or a local commit on this side waits behind one step rather
152/// than behind the session. The exchange gives up on a peer that makes no progress for
153/// [`IDLE_TIMEOUT`](crate::IDLE_TIMEOUT); the live phase after it has no such bound.
154/// Nothing is committed when the peer goes away or breaks the protocol: the error says which
155/// happened, and the replica's version vector is left where it was.
156///
157/// `S` must be `'static` because both halves of the stream are moved into tasks; a stream
158/// owned by the caller (a socket, a spliced pipe, a duplex) is.
159pub async fn open_live_session<S>(
160 stream: S,
161 replica: Arc<tokio::sync::Mutex<Replica>>,
162 workspace_id: GrainId,
163) -> Result<(SessionOutcome, LiveSession)>
164where
165 S: AsyncRead + AsyncWrite + Send + Unpin + 'static,
166{
167 let exchange = session::initial_exchange(stream, &replica, workspace_id).await?;
168
169 // A session is only live once the exchange reached its end markers. Anything else is
170 // reported as the failure it is: `run_session` may treat a peer that simply left as
171 // "nothing happened", but a caller asking for a live session cannot.
172 if !matches!(&exchange.stop, Stop::Complete) {
173 let why = match &exchange.stop {
174 Stop::Gone => Error::Protocol("the peer left before the exchange finished".to_owned()),
175 Stop::Refused(why) => Error::Refused(why.clone()),
176 Stop::Protocol(why) => Error::Protocol(why.clone()),
177 Stop::Complete => unreachable!("checked above"),
178 };
179 session::abort(exchange.out, exchange.writer);
180 return Err(why);
181 }
182
183 let Exchange {
184 reader,
185 out,
186 writer: _,
187 progress: _,
188 peer_vv,
189 peer_acks,
190 sent_vv,
191 mut incoming,
192 outcome,
193 stop: _,
194 } = exchange;
195
196 // Caught up: materialise what was waiting on content and commit, exactly as
197 // `run_session` does at this point.
198 let outcome = {
199 let mut locked = replica.lock().await;
200 session::materialise(&mut locked, &peer_vv, outcome)?
201 };
202
203 // The peer has what it said it had and everything this side sent it. Starting from both,
204 // rather than from its `Hello` alone, is what lets a caller push exactly what this side
205 // committed after the exchange — nothing it already sent.
206 let mut peer_vv = peer_vv;
207 peer_vv.merge(&sent_vv);
208 let peer_vv = Arc::new(watch::Sender::new(peer_vv));
209 let open = Arc::new(AtomicBool::new(true));
210 let (updates, _) = broadcast::channel(LIVE_QUEUE);
211
212 let reader_task = {
213 let replica = Arc::clone(&replica);
214 let peer_vv = Arc::clone(&peer_vv);
215 let open = Arc::clone(&open);
216 let updates = updates.clone();
217 let out = out.clone();
218 let mut reader = reader;
219 // Hashes asked for and not yet answered. One `Want` for a hash however many batches
220 // need it.
221 let mut wanted: HashSet<ContentHash> = HashSet::new();
222 // Batches joined but not yet announced; each is held until its own hashes are in.
223 let mut pending: Vec<Pending> = Vec::new();
224 tokio::spawn(async move {
225 loop {
226 // Set when content landed, so the files behind it are written before the
227 // batches waiting on it are announced.
228 let mut wrote_content = false;
229 match read_frame(&mut reader).await {
230 Ok(Some(Frame::Control(Message::Live(batch)))) => {
231 // The peer had all of this when it sent the batch, so its own
232 // version vector covers it. Merging now is what keeps a forwarder
233 // from sending the same entries back for ever.
234 peer_vv.send_modify(|known| {
235 for update in &batch {
236 known.merge(&update.seen);
237 }
238 });
239 let missing = {
240 let mut locked = replica.lock().await;
241 // The join is idempotent, so a repeated batch costs nothing.
242 if let Err(err) = locked.apply(&batch, &NoContent) {
243 tracing::warn!("a live batch was not applied: {err}");
244 break;
245 }
246 match session::needed(&locked, &batch) {
247 Ok(missing) => missing,
248 Err(err) => {
249 tracing::warn!("a live batch could not be checked: {err}");
250 break;
251 }
252 }
253 };
254 // Content travels by hash, so a batch naming a file this side does
255 // not have is completed by asking for it on this same stream.
256 let mut outstanding = HashSet::new();
257 for hash in missing {
258 if wanted.insert(hash)
259 && send(&out, Out::Control(Message::Want(hash))).is_err()
260 {
261 return;
262 }
263 outstanding.insert(hash);
264 }
265 pending.push(Pending { batch, outstanding });
266 }
267 // The peer has what this side pushed, content and all.
268 Ok(Some(Frame::Control(Message::Ack(vv)))) => {
269 peer_vv.send_modify(|known| known.merge(&vv));
270 }
271 Ok(Some(Frame::Control(Message::Want(hash)))) => {
272 let answer = session::answer(&*replica.lock().await, hash);
273 if send(&out, answer).is_err() {
274 break;
275 }
276 }
277 // Content is verified before it is staged: it is addressed by hash, so a
278 // peer that sent the wrong bytes under one must not be able to plant
279 // them.
280 Ok(Some(Frame::BlobStart { hash, len })) => {
281 if let Err(err) = incoming.start(hash, len) {
282 tracing::warn!("{err}");
283 break;
284 }
285 }
286 Ok(Some(Frame::BlobChunk(bytes))) => {
287 if let Err(err) = incoming.chunk(&bytes) {
288 tracing::warn!("{err}");
289 break;
290 }
291 }
292 Ok(Some(Frame::BlobEnd)) => match incoming.end() {
293 Ok(hash) => {
294 remember(&mut wanted, &mut pending, hash);
295 wrote_content = true;
296 }
297 Err(err) => {
298 tracing::warn!("{err}");
299 break;
300 }
301 },
302 // The peer does not have it either. Another peer may; the path simply
303 // stays unmaterialised until one does, and the batches that wanted it
304 // can be announced without it.
305 Ok(Some(Frame::Control(Message::Missing(hash)))) => {
306 remember(&mut wanted, &mut pending, hash);
307 }
308 // The peer's file changed under it mid-send: as good as `Missing`.
309 Ok(Some(Frame::Control(Message::Aborted(hash)))) => {
310 if let Err(err) = incoming.aborted(hash) {
311 tracing::warn!("{err}");
312 break;
313 }
314 remember(&mut wanted, &mut pending, hash);
315 }
316 // The exchange is over and frames arrive in order, so `Done`, `Settled`,
317 // `Updates`, a second `Hello` or a `Refused` cannot follow here.
318 Ok(Some(other)) => {
319 tracing::warn!("a live session got an unexpected frame: {other:?}");
320 break;
321 }
322 Ok(None) => break,
323 Err(err) => {
324 tracing::debug!("a live session stopped reading: {err}");
325 break;
326 }
327 }
328
329 if wrote_content {
330 // Write the bytes to disk before any batch that names them is
331 // announced: a subscriber reads the file, not the store.
332 let mut locked = replica.lock().await;
333 if let Err(err) = locked.fetch_missing(&NoContent) {
334 tracing::warn!("fetched content did not settle: {err}");
335 break;
336 }
337 }
338 // Announce every batch that is no longer waiting on anything. Failing only
339 // means nobody is listening.
340 let mut applied: Option<VersionVector> = None;
341 pending.retain(|entry| {
342 if entry.outstanding.is_empty() {
343 let _ = updates.send(entry.batch.clone());
344 let applied = applied.get_or_insert_with(VersionVector::new);
345 for update in &entry.batch {
346 applied.merge(&update.seen);
347 }
348 false
349 } else {
350 true
351 }
352 });
353 // Tell the peer what has landed — only now, with the content on disk, so an
354 // `Ack` means the edit survives the pushing device going away.
355 if peer_acks
356 && let Some(applied) = applied
357 && send(&out, Out::Control(Message::Ack(applied))).is_err()
358 {
359 break;
360 }
361 }
362 open.store(false, Ordering::Relaxed);
363 })
364 };
365
366 let live = LiveSession {
367 out,
368 updates,
369 peer_vv,
370 peer_acks,
371 open,
372 reader: reader_task,
373 };
374 Ok((outcome, live))
375}
376
377/// Record that `hash` is answered: it is no longer wanted, and no pending batch waits on it.
378fn remember(wanted: &mut HashSet<ContentHash>, pending: &mut [Pending], hash: ContentHash) {
379 wanted.remove(&hash);
380 for entry in pending.iter_mut() {
381 entry.outstanding.remove(&hash);
382 }
383}