Skip to main content

sapphire_framework_session/
live.rs

1//! The phase after `Done`: a session that stays open and pushes what is committed.
2//!
3//! [`open_live_session`] runs the same initial exchange as [`crate::run_session`] and then,
4//! instead of closing the stream, keeps reading it. `Live` messages are applied as they
5//! arrive and published on a broadcast, and [`LiveSession::push`] sends a batch through the
6//! same writer task the exchange used. The two ends of one session are two independent
7//! `LiveSession`s: each side pushes what it commits and applies what arrives, and neither is
8//! a client.
9//!
10//! Content does not travel inside a `Live` message. A batch that names a file this side does
11//! not have is answered with `Want` on the same stream, and the peer's content settles it
12//! through the machinery the exchange already uses: the same writer streams it, and the
13//! same `Incoming` stages it. A batch is published only once every
14//! hash it was waiting on has been answered, so a host that forwards what it receives never
15//! announces an entry it cannot serve.
16//!
17//! # Acknowledgements
18//!
19//! A `Live` push returns once it is queued, which says nothing about whether it arrived. So
20//! a peer that said `acks` in its `Hello` answers every batch it has applied — content
21//! included — with [`Message::Ack`], and [`LiveSession::acknowledged`] follows what the peer
22//! is known to have. A peer from before acknowledgements never sends one; its session still
23//! works, and only the exchange and its own pushes advance what it is known to have.
24//!
25//! # A dead peer must not stall the others
26//!
27//! The write half belongs to one task fed by unbounded queues (see `writer.rs` for why), so
28//! queueing for a peer that stopped reading never waits: [`LiveSession::push`] returns at
29//! once, and fails once the session has closed, so a caller looping over peers moves on to
30//! the next one instead of waiting on one that went away.
31
32use std::collections::HashSet;
33use std::sync::Arc;
34use std::sync::atomic::{AtomicBool, Ordering};
35
36use grain_id::GrainId;
37use sapphire_sync::{ContentHash, PathUpdate, Replica, VersionVector};
38use tokio::io::{AsyncRead, AsyncWrite};
39use tokio::sync::{broadcast, watch};
40use tokio::task::JoinHandle;
41
42use crate::error::{Error, Result};
43use crate::frame::{Frame, read_frame};
44use crate::message::Message;
45use crate::session::{self, Exchange, NoContent, SessionOutcome, Stop, send};
46use crate::writer::{Out, Outbox};
47
48/// How many batches may wait for a subscriber before the oldest is dropped. Applies to the
49/// broadcast of received updates, not to the wire.
50const LIVE_QUEUE: usize = 64;
51
52/// A session that stays open after the initial exchange.
53///
54/// Dropping it closes the session; `close` is the same thing said out loud.
55pub struct LiveSession {
56    out: Outbox,
57    updates: broadcast::Sender<Vec<PathUpdate>>,
58    peer_vv: Arc<watch::Sender<VersionVector>>,
59    peer_acks: bool,
60    open: Arc<AtomicBool>,
61    reader: JoinHandle<()>,
62}
63
64impl LiveSession {
65    /// Send updates to the peer.
66    ///
67    /// Fails rather than waiting once the session has closed, so a caller looping over
68    /// peers does not stall on one that went away.
69    pub async fn push(&self, updates: Vec<PathUpdate>) -> Result<()> {
70        if !self.is_open() {
71            return Err(Error::Protocol("the session is closed".to_owned()));
72        }
73        self.out
74            .send(Out::Control(Message::Live(updates)))
75            .map_err(|_| Error::Protocol("the session is closed".to_owned()))
76    }
77
78    /// Updates that arrived from the peer, after they were applied.
79    ///
80    /// A batch is published once every hash it was waiting on has been answered — with the
81    /// bytes, or with `Missing` — so a subscriber forwarding it forwards what this host can
82    /// answer for.
83    pub fn updates(&self) -> broadcast::Receiver<Vec<PathUpdate>> {
84        self.updates.subscribe()
85    }
86
87    /// Everything the peer is known to have.
88    ///
89    /// Consulted before forwarding: an entry the peer already covers must not be sent again,
90    /// or three connected hosts will pass one edit round in a circle. It advances as the
91    /// peer's batches arrive, because each one carries what the peer had when it sent it.
92    ///
93    /// It also advances when the peer acknowledges a push (see the module docs), so it
94    /// covers this side's own entries once they have reached the peer.
95    pub fn peer_vv(&self) -> VersionVector {
96        self.peer_vv.borrow().clone()
97    }
98
99    /// [`peer_vv`](Self::peer_vv) as it changes: marked changed whenever an arriving batch or
100    /// an acknowledgement advances it. Closed once the session is.
101    ///
102    /// Starts at what the initial exchange left: what the peer said it had, plus everything
103    /// this side sent it then — the exchange completes only once the peer has all of that.
104    pub fn acknowledged(&self) -> watch::Receiver<VersionVector> {
105        self.peer_vv.subscribe()
106    }
107
108    /// Whether the peer acknowledges pushes. `false` for a peer from before
109    /// acknowledgements: what this side pushes to it never shows in
110    /// [`peer_vv`](Self::peer_vv) until a later exchange.
111    pub fn peer_acknowledges(&self) -> bool {
112        self.peer_acks
113    }
114
115    /// Is the session still usable?
116    pub fn is_open(&self) -> bool {
117        self.open.load(Ordering::Relaxed)
118    }
119
120    /// Close it.
121    pub fn close(self) {
122        drop(self);
123    }
124}
125
126impl Drop for LiveSession {
127    fn drop(&mut self) {
128        self.open.store(false, Ordering::Relaxed);
129        self.reader.abort();
130    }
131}
132
133/// A batch that arrived and is waiting for content before it can be announced.
134struct Pending {
135    /// What the peer sent.
136    batch: Vec<PathUpdate>,
137    /// The hashes this batch is still waiting on. Empty means it is ready to publish.
138    outstanding: HashSet<ContentHash>,
139}
140
141/// Run the initial exchange and then keep the session open.
142///
143/// Both sides call this: each sends what the other lacks during the exchange and afterwards
144/// pushes what it commits, so the returned [`LiveSession`] is the same handle on either end
145/// of the stream. The [`SessionOutcome`] is returned as soon as both sides have said `Done`
146/// and `Settled` — "we are caught up" — while the session itself keeps going until a side
147/// closes it or the stream ends.
148///
149/// The replica is locked per step — never while waiting for the peer's `Hello` (issue #163)
150/// or for content in flight: for each page and `Want` of the exchange, and then for each
151/// arriving batch — so a scan or a local commit on this side waits behind one step rather
152/// than behind the session. The exchange gives up on a peer that makes no progress for
153/// [`IDLE_TIMEOUT`](crate::IDLE_TIMEOUT); the live phase after it has no such bound.
154/// Nothing is committed when the peer goes away or breaks the protocol: the error says which
155/// happened, and the replica's version vector is left where it was.
156///
157/// `S` must be `'static` because both halves of the stream are moved into tasks; a stream
158/// owned by the caller (a socket, a spliced pipe, a duplex) is.
159pub async fn open_live_session<S>(
160    stream: S,
161    replica: Arc<tokio::sync::Mutex<Replica>>,
162    workspace_id: GrainId,
163) -> Result<(SessionOutcome, LiveSession)>
164where
165    S: AsyncRead + AsyncWrite + Send + Unpin + 'static,
166{
167    let exchange = session::initial_exchange(stream, &replica, workspace_id).await?;
168
169    // A session is only live once the exchange reached its end markers. Anything else is
170    // reported as the failure it is: `run_session` may treat a peer that simply left as
171    // "nothing happened", but a caller asking for a live session cannot.
172    if !matches!(&exchange.stop, Stop::Complete) {
173        let why = match &exchange.stop {
174            Stop::Gone => Error::Protocol("the peer left before the exchange finished".to_owned()),
175            Stop::Refused(why) => Error::Refused(why.clone()),
176            Stop::Protocol(why) => Error::Protocol(why.clone()),
177            Stop::Complete => unreachable!("checked above"),
178        };
179        session::abort(exchange.out, exchange.writer);
180        return Err(why);
181    }
182
183    let Exchange {
184        reader,
185        out,
186        writer: _,
187        progress: _,
188        peer_vv,
189        peer_acks,
190        sent_vv,
191        mut incoming,
192        outcome,
193        stop: _,
194    } = exchange;
195
196    // Caught up: materialise what was waiting on content and commit, exactly as
197    // `run_session` does at this point.
198    let outcome = {
199        let mut locked = replica.lock().await;
200        session::materialise(&mut locked, &peer_vv, outcome)?
201    };
202
203    // The peer has what it said it had and everything this side sent it. Starting from both,
204    // rather than from its `Hello` alone, is what lets a caller push exactly what this side
205    // committed after the exchange — nothing it already sent.
206    let mut peer_vv = peer_vv;
207    peer_vv.merge(&sent_vv);
208    let peer_vv = Arc::new(watch::Sender::new(peer_vv));
209    let open = Arc::new(AtomicBool::new(true));
210    let (updates, _) = broadcast::channel(LIVE_QUEUE);
211
212    let reader_task = {
213        let replica = Arc::clone(&replica);
214        let peer_vv = Arc::clone(&peer_vv);
215        let open = Arc::clone(&open);
216        let updates = updates.clone();
217        let out = out.clone();
218        let mut reader = reader;
219        // Hashes asked for and not yet answered. One `Want` for a hash however many batches
220        // need it.
221        let mut wanted: HashSet<ContentHash> = HashSet::new();
222        // Batches joined but not yet announced; each is held until its own hashes are in.
223        let mut pending: Vec<Pending> = Vec::new();
224        tokio::spawn(async move {
225            loop {
226                // Set when content landed, so the files behind it are written before the
227                // batches waiting on it are announced.
228                let mut wrote_content = false;
229                match read_frame(&mut reader).await {
230                    Ok(Some(Frame::Control(Message::Live(batch)))) => {
231                        // The peer had all of this when it sent the batch, so its own
232                        // version vector covers it. Merging now is what keeps a forwarder
233                        // from sending the same entries back for ever.
234                        peer_vv.send_modify(|known| {
235                            for update in &batch {
236                                known.merge(&update.seen);
237                            }
238                        });
239                        let missing = {
240                            let mut locked = replica.lock().await;
241                            // The join is idempotent, so a repeated batch costs nothing.
242                            if let Err(err) = locked.apply(&batch, &NoContent) {
243                                tracing::warn!("a live batch was not applied: {err}");
244                                break;
245                            }
246                            match session::needed(&locked, &batch) {
247                                Ok(missing) => missing,
248                                Err(err) => {
249                                    tracing::warn!("a live batch could not be checked: {err}");
250                                    break;
251                                }
252                            }
253                        };
254                        // Content travels by hash, so a batch naming a file this side does
255                        // not have is completed by asking for it on this same stream.
256                        let mut outstanding = HashSet::new();
257                        for hash in missing {
258                            if wanted.insert(hash)
259                                && send(&out, Out::Control(Message::Want(hash))).is_err()
260                            {
261                                return;
262                            }
263                            outstanding.insert(hash);
264                        }
265                        pending.push(Pending { batch, outstanding });
266                    }
267                    // The peer has what this side pushed, content and all.
268                    Ok(Some(Frame::Control(Message::Ack(vv)))) => {
269                        peer_vv.send_modify(|known| known.merge(&vv));
270                    }
271                    Ok(Some(Frame::Control(Message::Want(hash)))) => {
272                        let answer = session::answer(&*replica.lock().await, hash);
273                        if send(&out, answer).is_err() {
274                            break;
275                        }
276                    }
277                    // Content is verified before it is staged: it is addressed by hash, so a
278                    // peer that sent the wrong bytes under one must not be able to plant
279                    // them.
280                    Ok(Some(Frame::BlobStart { hash, len })) => {
281                        if let Err(err) = incoming.start(hash, len) {
282                            tracing::warn!("{err}");
283                            break;
284                        }
285                    }
286                    Ok(Some(Frame::BlobChunk(bytes))) => {
287                        if let Err(err) = incoming.chunk(&bytes) {
288                            tracing::warn!("{err}");
289                            break;
290                        }
291                    }
292                    Ok(Some(Frame::BlobEnd)) => match incoming.end() {
293                        Ok(hash) => {
294                            remember(&mut wanted, &mut pending, hash);
295                            wrote_content = true;
296                        }
297                        Err(err) => {
298                            tracing::warn!("{err}");
299                            break;
300                        }
301                    },
302                    // The peer does not have it either. Another peer may; the path simply
303                    // stays unmaterialised until one does, and the batches that wanted it
304                    // can be announced without it.
305                    Ok(Some(Frame::Control(Message::Missing(hash)))) => {
306                        remember(&mut wanted, &mut pending, hash);
307                    }
308                    // The peer's file changed under it mid-send: as good as `Missing`.
309                    Ok(Some(Frame::Control(Message::Aborted(hash)))) => {
310                        if let Err(err) = incoming.aborted(hash) {
311                            tracing::warn!("{err}");
312                            break;
313                        }
314                        remember(&mut wanted, &mut pending, hash);
315                    }
316                    // The exchange is over and frames arrive in order, so `Done`, `Settled`,
317                    // `Updates`, a second `Hello` or a `Refused` cannot follow here.
318                    Ok(Some(other)) => {
319                        tracing::warn!("a live session got an unexpected frame: {other:?}");
320                        break;
321                    }
322                    Ok(None) => break,
323                    Err(err) => {
324                        tracing::debug!("a live session stopped reading: {err}");
325                        break;
326                    }
327                }
328
329                if wrote_content {
330                    // Write the bytes to disk before any batch that names them is
331                    // announced: a subscriber reads the file, not the store.
332                    let mut locked = replica.lock().await;
333                    if let Err(err) = locked.fetch_missing(&NoContent) {
334                        tracing::warn!("fetched content did not settle: {err}");
335                        break;
336                    }
337                }
338                // Announce every batch that is no longer waiting on anything. Failing only
339                // means nobody is listening.
340                let mut applied: Option<VersionVector> = None;
341                pending.retain(|entry| {
342                    if entry.outstanding.is_empty() {
343                        let _ = updates.send(entry.batch.clone());
344                        let applied = applied.get_or_insert_with(VersionVector::new);
345                        for update in &entry.batch {
346                            applied.merge(&update.seen);
347                        }
348                        false
349                    } else {
350                        true
351                    }
352                });
353                // Tell the peer what has landed — only now, with the content on disk, so an
354                // `Ack` means the edit survives the pushing device going away.
355                if peer_acks
356                    && let Some(applied) = applied
357                    && send(&out, Out::Control(Message::Ack(applied))).is_err()
358                {
359                    break;
360                }
361            }
362            open.store(false, Ordering::Relaxed);
363        })
364    };
365
366    let live = LiveSession {
367        out,
368        updates,
369        peer_vv,
370        peer_acks,
371        open,
372        reader: reader_task,
373    };
374    Ok((outcome, live))
375}
376
377/// Record that `hash` is answered: it is no longer wanted, and no pending batch waits on it.
378fn remember(wanted: &mut HashSet<ContentHash>, pending: &mut [Pending], hash: ContentHash) {
379    wanted.remove(&hash);
380    for entry in pending.iter_mut() {
381        entry.outstanding.remove(&hash);
382    }
383}