Skip to main content

kunobi_daemon/readiness/
channel.rs

1//! A readiness channel from a daemon to the process that started it.
2//!
3//! A launcher that only polls a probe learns that its daemon died during
4//! startup when the deadline passes, and a fixed deadline can fail a daemon
5//! that is still making progress on a slow machine. Over this channel the
6//! daemon says when it is ready and may report progress on the way. The
7//! channel ending before `ready` tells the launcher at once that the daemon
8//! died.
9//!
10//! The launcher asks for the channel with `launch::DaemonCommand::readiness_channel`
11//! (`launch` feature) and takes a [`Channel`] from the started daemon. The
12//! daemon takes its end with `Notifier::from_env` (`local` feature), reports
13//! progress, and sends ready.
14//!
15//! The launcher's deadline is the longest silence it accepts. Every message
16//! starts it again, so a daemon that keeps reporting progress is waited for as
17//! long as it takes.
18//!
19//! `ready` is not a proof. It says when a probe is worth making; a fresh probe
20//! still decides, as everywhere in [`crate::readiness`] and [`crate::selection`].
21//! [`Signaled`] adds a channel to selection evidence without changing the
22//! selection rules.
23//!
24//! # Wire format
25//!
26//! One message per line, in ASCII, each ending in a newline (`\n`) and at most
27//! [`MAX_MESSAGE`] bytes long with it:
28//!
29//! - `ready`: the daemon is serving. The launcher reads nothing after it.
30//! - `progress`, or `progress ` followed by a detail of at most [`MAX_DETAIL`]
31//!   printable ASCII characters (space to `~`): the daemon is still starting.
32//!
33//! Anything else, including a line that grows past the limit, ends the channel
34//! as malformed, and nothing after it is read. The channel ending without
35//! `ready`, with or without part of a message, means the daemon died. A later
36//! version of this protocol will use a new environment variable, so a daemon
37//! never sends a launcher a message it does not know.
38//!
39//! # Passing the channel
40//!
41//! [`ENV`] tells the daemon where its end is.
42//!
43//! - On Unix it is the number of the write end of a pipe the launcher
44//!   creates, the only descriptor above stderr that survives the exec. The
45//!   daemon adopts that descriptor, which is why `Notifier::from_env` is
46//!   `unsafe`, as adopting descriptors passed by `LISTEN_FDS` is.
47//! - On Windows it is the name of a named pipe the launcher creates: 128
48//!   random bits, one inbound instance, a DACL that admits only the current
49//!   user, remote clients refused. The daemon opens it by name, so it
50//!   inherits no handle for it, and neither can any other child the launcher
51//!   starts. The launcher watches the daemon's process as well as the pipe,
52//!   so a daemon that exits before it connects is reported at once.
53//!
54//! Taking the channel removes [`ENV`] from the daemon's environment, and its
55//! end is not inherited by programs the daemon starts.
56
57// Without `launch` nothing in the crate reads a channel; the daemon's side is
58// still used.
59#![cfg_attr(not(feature = "launch"), allow(dead_code))]
60
61use std::fmt;
62use std::io::{self, Read, Write};
63use std::sync::atomic::{AtomicBool, Ordering};
64use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError};
65use std::time::{Duration, Instant};
66
67#[cfg(feature = "async")]
68use crate::selection::AsyncEvidence;
69use crate::selection::Evidence;
70
71/// The environment variable that names the daemon's end of the channel.
72pub const ENV: &str = "KUNOBI_DAEMON_READY";
73
74/// Longest message in bytes, including its newline.
75pub const MAX_MESSAGE: usize = 256;
76
77/// Longest progress detail in bytes.
78pub const MAX_DETAIL: usize = MAX_MESSAGE - "progress \n".len();
79
80/// Why waiting on a channel ended without `ready`.
81#[derive(Clone, Copy, Debug, PartialEq, Eq)]
82#[non_exhaustive]
83pub enum NotReady {
84    /// The channel ended before `ready`: the daemon exited, or closed its end
85    /// without saying it was ready. The process that was started will not
86    /// serve; another one may, which only a probe can tell.
87    Died,
88    /// The daemon sent something that is not a message of this protocol.
89    /// Nothing after it is read.
90    Malformed,
91    /// No message arrived within the silence the caller allowed.
92    Silent,
93}
94
95impl fmt::Display for NotReady {
96    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
97        f.write_str(match self {
98            Self::Died => "the daemon's readiness channel ended before it was ready",
99            Self::Malformed => "the daemon sent a malformed readiness message",
100            Self::Silent => "the daemon's readiness channel stayed silent too long",
101        })
102    }
103}
104
105impl std::error::Error for NotReady {}
106
107// ── Wire format ─────────────────────────────────────────────────────
108
109/// One message from the daemon.
110#[derive(Clone, Debug, PartialEq, Eq)]
111enum Message {
112    Ready,
113    Progress(Option<String>),
114}
115
116/// Space to `~`: no control characters, no escape sequences, no newline.
117fn printable(bytes: &[u8]) -> bool {
118    bytes.iter().all(|byte| (b' '..=b'~').contains(byte))
119}
120
121/// The message a complete line (without its newline) holds, if any.
122fn parse(line: &[u8]) -> Option<Message> {
123    match line {
124        b"ready" => Some(Message::Ready),
125        b"progress" => Some(Message::Progress(None)),
126        _ => {
127            let detail = line.strip_prefix(b"progress ")?;
128            if detail.is_empty() || !printable(detail) {
129                return None;
130            }
131            String::from_utf8(detail.to_vec())
132                .ok()
133                .map(|detail| Message::Progress(Some(detail)))
134        }
135    }
136}
137
138/// The line for a progress message; an empty detail sends bare `progress`.
139fn progress_line(detail: &str) -> io::Result<Vec<u8>> {
140    if detail.len() > MAX_DETAIL || !printable(detail.as_bytes()) {
141        return Err(io::Error::new(
142            io::ErrorKind::InvalidInput,
143            "a progress detail is printable ASCII of at most MAX_DETAIL bytes",
144        ));
145    }
146    let mut line = Vec::with_capacity(MAX_MESSAGE);
147    line.extend_from_slice(b"progress");
148    if !detail.is_empty() {
149        line.push(b' ');
150        line.extend_from_slice(detail.as_bytes());
151    }
152    line.push(b'\n');
153    Ok(line)
154}
155
156/// What one more byte of the stream completed.
157#[derive(Debug, PartialEq, Eq)]
158enum Step {
159    More,
160    Message(Message),
161    Malformed,
162}
163
164/// Splits the stream into messages, one byte at a time, so how the reads
165/// happen to split it never matters.
166#[derive(Debug, Default)]
167struct Decoder {
168    line: Vec<u8>,
169}
170
171impl Decoder {
172    fn push(&mut self, byte: u8) -> Step {
173        if byte == b'\n' {
174            let message = parse(&self.line);
175            self.line.clear();
176            return message.map_or(Step::Malformed, Step::Message);
177        }
178        // No room left for this byte and the newline: fail now rather than
179        // buffer a line that can never be valid.
180        if self.line.len() >= MAX_MESSAGE - 1 {
181            return Step::Malformed;
182        }
183        self.line.push(byte);
184        Step::More
185    }
186}
187
188// ── State machine ───────────────────────────────────────────────────
189
190/// What the daemon has said so far.
191#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
192enum State {
193    #[default]
194    Waiting,
195    Progressing,
196    Ready,
197    Died,
198    Malformed,
199}
200
201/// One thing the launcher learned from the channel.
202#[derive(Clone, Copy, Debug, PartialEq, Eq)]
203#[cfg_attr(kani, derive(kani::Arbitrary))]
204enum Event {
205    Progress,
206    Ready,
207    Malformed,
208    /// The channel ended: every write end is closed, or reading failed.
209    Closed,
210}
211
212/// What a waiter does now.
213#[derive(Clone, Copy, Debug, PartialEq, Eq)]
214enum Decision {
215    Ready,
216    NotReady(NotReady),
217    /// Nothing to report for this long.
218    Wait(Duration),
219}
220
221/// The channel's state and the time of its last message, kept apart from any
222/// I/O so it can be model-checked. Times are measured from the channel's
223/// opening.
224#[derive(Clone, Copy, Debug, Default)]
225struct Tracker {
226    state: State,
227    last_message: Duration,
228}
229
230impl Tracker {
231    /// Record `event`, received at `at`. Ready, died and malformed are final.
232    fn record(&mut self, at: Duration, event: Event) {
233        if matches!(self.state, State::Ready | State::Died | State::Malformed) {
234            return;
235        }
236        self.last_message = self.last_message.max(at);
237        self.state = match event {
238            Event::Progress => State::Progressing,
239            Event::Ready => State::Ready,
240            Event::Malformed => State::Malformed,
241            Event::Closed => State::Died,
242        };
243    }
244
245    /// Decide at `now` for a waiter that accepts `silence` between messages.
246    fn decide(&self, now: Duration, silence: Duration) -> Decision {
247        match self.state {
248            State::Ready => Decision::Ready,
249            State::Died => Decision::NotReady(NotReady::Died),
250            State::Malformed => Decision::NotReady(NotReady::Malformed),
251            State::Waiting | State::Progressing => {
252                let until = self.last_message.saturating_add(silence);
253                if now >= until {
254                    Decision::NotReady(NotReady::Silent)
255                } else {
256                    Decision::Wait(until - now)
257                }
258            }
259        }
260    }
261}
262
263// ── The launcher's end ──────────────────────────────────────────────
264
265#[derive(Debug, Default)]
266struct Inner {
267    tracker: Tracker,
268    /// Counts every event, so a waiter can tell news from what it has seen.
269    version: u64,
270    progress: u64,
271    last_progress: Option<String>,
272}
273
274#[derive(Debug)]
275struct Shared {
276    opened: Instant,
277    inner: Mutex<Inner>,
278    changed: Condvar,
279    #[cfg(feature = "async")]
280    notify: tokio::sync::Notify,
281    /// Set when the launcher drops its [`Channel`]: the reader stops at its
282    /// next read and closes the launcher's end.
283    closed: AtomicBool,
284}
285
286impl Shared {
287    fn new() -> Self {
288        Self {
289            opened: Instant::now(),
290            inner: Mutex::default(),
291            changed: Condvar::new(),
292            #[cfg(feature = "async")]
293            notify: tokio::sync::Notify::new(),
294            closed: AtomicBool::new(false),
295        }
296    }
297
298    fn lock(&self) -> MutexGuard<'_, Inner> {
299        self.inner.lock().unwrap_or_else(PoisonError::into_inner)
300    }
301
302    fn post(&self, event: Event, detail: Option<String>) {
303        let at = self.opened.elapsed();
304        {
305            let mut inner = self.lock();
306            inner.tracker.record(at, event);
307            inner.version += 1;
308            if event == Event::Progress {
309                inner.progress += 1;
310                inner.last_progress = detail;
311            }
312        }
313        self.changed.notify_all();
314        #[cfg(feature = "async")]
315        self.notify.notify_one();
316    }
317}
318
319/// Whether a failed read ends the channel. An interrupted read is retried.
320fn ends_channel(error: &io::Error) -> bool {
321    error.kind() != io::ErrorKind::Interrupted
322}
323
324/// Read messages until `ready`, the end of the channel, a malformed message,
325/// or the launcher dropping its [`Channel`].
326fn read_messages(mut reader: impl Read, shared: &Shared) {
327    let mut decoder = Decoder::default();
328    let mut buffer = [0u8; MAX_MESSAGE];
329    loop {
330        let read = reader.read(&mut buffer);
331        if shared.closed.load(Ordering::Acquire) {
332            return;
333        }
334        let count = match read {
335            Ok(0) => return shared.post(Event::Closed, None),
336            Ok(count) => count,
337            Err(error) => {
338                if ends_channel(&error) {
339                    return shared.post(Event::Closed, None);
340                }
341                continue;
342            }
343        };
344        for &byte in &buffer[..count] {
345            match decoder.push(byte) {
346                Step::More => {}
347                Step::Message(Message::Progress(detail)) => shared.post(Event::Progress, detail),
348                Step::Message(Message::Ready) => return shared.post(Event::Ready, None),
349                Step::Malformed => return shared.post(Event::Malformed, None),
350            }
351        }
352    }
353}
354
355/// The launcher's end of a daemon's readiness channel.
356///
357/// A thread reads the channel from the moment the daemon starts, so the
358/// silence bound runs from then even before anyone waits. It stops at
359/// `ready`, at the end of the channel, at a malformed message, or at the
360/// first message after this handle is dropped; the daemon's next send then
361/// fails instead of blocking.
362#[derive(Debug)]
363pub struct Channel {
364    shared: Arc<Shared>,
365}
366
367impl Channel {
368    /// Start reading the launcher's end of a channel.
369    pub(crate) fn start(reader: impl Read + Send + 'static) -> io::Result<Self> {
370        let shared = Arc::new(Shared::new());
371        let reading = Arc::clone(&shared);
372        std::thread::Builder::new()
373            .name("kunobi-daemon-readiness".into())
374            .spawn(move || read_messages(reader, &reading))?;
375        Ok(Self { shared })
376    }
377
378    /// Block until the daemon says it is ready, its channel ends, or
379    /// `silence` passes without a message.
380    ///
381    /// `silence` is the longest wait between two messages, counted from the
382    /// daemon's start for the first one. A daemon that keeps reporting
383    /// progress is waited for however long it takes in total.
384    ///
385    /// `Ok` means the daemon said it was ready, not that it is: probe it
386    /// before relying on it. Once the answer is ready, died or malformed,
387    /// every later call returns it at once. After [`NotReady::Silent`] a later
388    /// call can still see the daemon's next message.
389    pub fn wait(&mut self, silence: Duration) -> Result<(), NotReady> {
390        let mut inner = self.shared.lock();
391        loop {
392            let now = self.shared.opened.elapsed();
393            match inner.tracker.decide(now, silence) {
394                Decision::Ready => return Ok(()),
395                Decision::NotReady(reason) => return Err(reason),
396                Decision::Wait(remaining) => {
397                    inner = self
398                        .shared
399                        .changed
400                        .wait_timeout(inner, remaining)
401                        .unwrap_or_else(PoisonError::into_inner)
402                        .0;
403                }
404            }
405        }
406    }
407
408    /// Async counterpart of [`Channel::wait`], on a Tokio runtime with its
409    /// timer enabled. Cancelling it loses nothing: the channel is still read.
410    #[cfg(feature = "async")]
411    pub async fn wait_async(&mut self, silence: Duration) -> Result<(), NotReady> {
412        loop {
413            let notified = self.shared.notify.notified();
414            let now = self.shared.opened.elapsed();
415            let decision = self.shared.lock().tracker.decide(now, silence);
416            match decision {
417                Decision::Ready => return Ok(()),
418                Decision::NotReady(reason) => return Err(reason),
419                Decision::Wait(remaining) => {
420                    let _ = tokio::time::timeout(remaining, notified).await;
421                }
422            }
423        }
424    }
425
426    /// How many progress messages the daemon has sent so far.
427    pub fn progress_reports(&self) -> u64 {
428        self.shared.lock().progress
429    }
430
431    /// The detail of the daemon's latest progress message, or `None` if it
432    /// had none or nothing was reported yet. Useful in an error message when
433    /// the daemon does not become ready.
434    pub fn last_progress(&self) -> Option<String> {
435        self.shared.lock().last_progress.clone()
436    }
437
438    fn version(&self) -> u64 {
439        self.shared.lock().version
440    }
441
442    fn is_ready(&self) -> bool {
443        self.shared.lock().tracker.state == State::Ready
444    }
445
446    /// Why waiting is over, if it is and not because the daemon is ready.
447    fn failure(&self, silence: Duration) -> Option<NotReady> {
448        let now = self.shared.opened.elapsed();
449        match self.shared.lock().tracker.decide(now, silence) {
450            Decision::NotReady(reason) => Some(reason),
451            Decision::Ready | Decision::Wait(_) => None,
452        }
453    }
454
455    /// Block until the channel has news since version `since`, the silence
456    /// bound passes, or `until`.
457    fn wait_for_news(&self, since: u64, until: Instant, silence: Duration) {
458        let mut inner = self.shared.lock();
459        loop {
460            let now = self.shared.opened.elapsed();
461            let Some(quiet) = quiet_for(&inner, since, now, silence) else {
462                return;
463            };
464            let remaining = quiet.min(until.saturating_duration_since(Instant::now()));
465            if remaining.is_zero() {
466                return;
467            }
468            inner = self
469                .shared
470                .changed
471                .wait_timeout(inner, remaining)
472                .unwrap_or_else(PoisonError::into_inner)
473                .0;
474        }
475    }
476
477    /// Async counterpart of [`Channel::wait_for_news`].
478    #[cfg(feature = "async")]
479    async fn wait_for_news_async(
480        &self,
481        since: u64,
482        until: tokio::time::Instant,
483        silence: Duration,
484    ) {
485        loop {
486            let notified = self.shared.notify.notified();
487            let now = self.shared.opened.elapsed();
488            let quiet = quiet_for(&self.shared.lock(), since, now, silence);
489            let Some(quiet) = quiet else {
490                return;
491            };
492            let remaining = quiet.min(until.saturating_duration_since(tokio::time::Instant::now()));
493            if remaining.is_zero() {
494                return;
495            }
496            if tokio::time::timeout(remaining, notified).await.is_err() {
497                return;
498            }
499        }
500    }
501}
502
503impl Drop for Channel {
504    fn drop(&mut self) {
505        self.shared.closed.store(true, Ordering::Release);
506    }
507}
508
509/// How long nothing is worth a new round: `None` once there is news since
510/// `since`, an answer, or silence past the bound.
511fn quiet_for(inner: &Inner, since: u64, now: Duration, silence: Duration) -> Option<Duration> {
512    if inner.version != since {
513        return None;
514    }
515    match inner.tracker.decide(now, silence) {
516        Decision::Wait(remaining) => Some(remaining),
517        Decision::Ready | Decision::NotReady(_) => None,
518    }
519}
520
521// ── The daemon's end ────────────────────────────────────────────────
522
523/// The daemon's end of the readiness channel its launcher asked for.
524///
525/// Report progress while starting, then call [`Notifier::ready`] once the
526/// daemon serves. Dropping it without `ready` tells the launcher the daemon
527/// will not become ready, as exiting does.
528///
529/// A failed send means the launcher stopped listening, usually because it
530/// gave up or exited. The daemon should carry on: whether it serves is not
531/// the launcher's decision.
532#[derive(Debug)]
533pub struct Notifier {
534    pipe: std::io::PipeWriter,
535}
536
537impl Notifier {
538    #[cfg(feature = "local")]
539    pub(crate) fn from_pipe(pipe: std::io::PipeWriter) -> Self {
540        Self { pipe }
541    }
542
543    /// Report progress, which restarts the launcher's silence bound.
544    ///
545    /// `detail` is shown to whoever waits, for example in an error message;
546    /// pass `""` for none. It must be printable ASCII (space to `~`) of at
547    /// most [`MAX_DETAIL`] bytes, or this fails with
548    /// [`io::ErrorKind::InvalidInput`] and sends nothing.
549    pub fn progress(&mut self, detail: &str) -> io::Result<()> {
550        let line = progress_line(detail)?;
551        self.pipe.write_all(&line)
552    }
553
554    /// Say the daemon is serving, and close the channel.
555    ///
556    /// Send it once the daemon answers on its endpoint: the launcher probes
557    /// it next.
558    pub fn ready(mut self) -> io::Result<()> {
559        self.pipe.write_all(b"ready\n")
560    }
561}
562
563// ── Selection ───────────────────────────────────────────────────────
564
565/// Selection evidence woken by a daemon's readiness channel.
566///
567/// Wraps the evidence a caller already has and changes only when a round is
568/// worth making, never what it decides:
569///
570/// - Before the daemon says it is ready, the wait between rounds lasts until
571///   the channel has news (a message, or its end) or the silence bound
572///   passes, instead of a poll interval.
573/// - After `ready`, the wrapped evidence's own wait paces the rounds.
574/// - A probe still decides. When one does not prove the daemon and the
575///   channel has died, turned malformed or stayed silent past the bound, the
576///   probe fails with [`SignaledError::NotReady`], which ends the wait at once
577///   instead of at the budget.
578///
579/// A fresh proof wins over a dead channel: when another instance won the
580/// endpoint and the daemon this launcher started exited, a probe that reaches
581/// the winner still reports it current.
582///
583/// The selection's commit budget still bounds the total wait. With a channel
584/// it can be generous; the silence bound catches a daemon that stopped
585/// reporting.
586#[derive(Debug)]
587pub struct Signaled<E> {
588    evidence: E,
589    channel: Channel,
590    silence: Duration,
591    /// The channel's version when the current round began.
592    seen: u64,
593}
594
595impl<E> Signaled<E> {
596    /// Wrap `evidence` with the daemon's `channel`, allowing at most
597    /// `silence` between the daemon's messages.
598    pub fn new(evidence: E, channel: Channel, silence: Duration) -> Self {
599        Self {
600            evidence,
601            channel,
602            silence,
603            seen: 0,
604        }
605    }
606
607    /// The wrapped evidence and the channel.
608    pub fn into_parts(self) -> (E, Channel) {
609        (self.evidence, self.channel)
610    }
611}
612
613/// Why a [`Signaled`] wait failed.
614#[derive(Clone, Debug, PartialEq, Eq)]
615#[non_exhaustive]
616pub enum SignaledError<E> {
617    /// The wrapped evidence failed.
618    Evidence(E),
619    /// No fresh probe proved the daemon, and its channel says it will not
620    /// become ready.
621    NotReady(NotReady),
622}
623
624impl<E: fmt::Display> fmt::Display for SignaledError<E> {
625    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
626        match self {
627            Self::Evidence(error) => fmt::Display::fmt(error, f),
628            Self::NotReady(reason) => fmt::Display::fmt(reason, f),
629        }
630    }
631}
632
633impl<E: std::error::Error + 'static> std::error::Error for SignaledError<E> {
634    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
635        match self {
636            Self::Evidence(error) => Some(error),
637            Self::NotReady(reason) => Some(reason),
638        }
639    }
640}
641
642impl<E: Evidence> Evidence for Signaled<E> {
643    type Proof = E::Proof;
644    type Error = SignaledError<E::Error>;
645
646    fn committed(&mut self) -> Result<bool, Self::Error> {
647        self.evidence.committed().map_err(SignaledError::Evidence)
648    }
649
650    fn probe(&mut self, deadline: Instant) -> Result<Option<Self::Proof>, Self::Error> {
651        // News from here on wakes the wait after this round.
652        self.seen = self.channel.version();
653        let proof = self
654            .evidence
655            .probe(deadline)
656            .map_err(SignaledError::Evidence)?;
657        if proof.is_some() {
658            return Ok(proof);
659        }
660        match self.channel.failure(self.silence) {
661            Some(reason) => Err(SignaledError::NotReady(reason)),
662            None => Ok(None),
663        }
664    }
665
666    fn wait(&mut self, until: Instant) {
667        // News during the round, such as a `ready` that arrived while an
668        // unproven probe ran, is worth a new round at once.
669        if self.channel.version() != self.seen {
670            return;
671        }
672        if self.channel.is_ready() {
673            self.evidence.wait(until);
674        } else {
675            self.channel.wait_for_news(self.seen, until, self.silence);
676        }
677    }
678}
679
680#[cfg(feature = "async")]
681impl<E> AsyncEvidence for Signaled<E>
682where
683    E: AsyncEvidence + Send,
684    E::Proof: Send,
685    E::Error: Send,
686{
687    type Proof = E::Proof;
688    type Error = SignaledError<E::Error>;
689
690    async fn committed(&mut self) -> Result<bool, Self::Error> {
691        self.evidence
692            .committed()
693            .await
694            .map_err(SignaledError::Evidence)
695    }
696
697    async fn probe(&mut self) -> Result<Option<Self::Proof>, Self::Error> {
698        self.seen = self.channel.version();
699        let proof = self
700            .evidence
701            .probe()
702            .await
703            .map_err(SignaledError::Evidence)?;
704        if proof.is_some() {
705            return Ok(proof);
706        }
707        match self.channel.failure(self.silence) {
708            Some(reason) => Err(SignaledError::NotReady(reason)),
709            None => Ok(None),
710        }
711    }
712
713    async fn wait(&mut self, until: tokio::time::Instant) {
714        if self.channel.version() != self.seen {
715            return;
716        }
717        if self.channel.is_ready() {
718            self.evidence.wait(until).await;
719        } else {
720            self.channel
721                .wait_for_news_async(self.seen, until, self.silence)
722                .await;
723        }
724    }
725}
726
727/// The channel's states for every sequence of messages and silences, not
728/// only the ones a test was written for.
729#[cfg(kani)]
730mod proofs {
731    use super::*;
732
733    /// Steps explored. Durations are small so every ordering of messages,
734    /// silence bound and waiting time is reachable within them.
735    const STEPS: usize = 5;
736
737    fn small() -> Duration {
738        Duration::from_millis(u64::from(kani::any::<u8>() % 8))
739    }
740
741    /// Waiting, progressing, ready, died and malformed: the launcher's answer
742    /// follows the first final message, never anything after it, and until
743    /// then only the silence since the last message can end the wait.
744    #[kani::proof]
745    #[kani::unwind(6)]
746    fn a_launcher_learns_only_what_its_daemon_said() {
747        let silence = small();
748        let mut tracker = Tracker::default();
749        let mut now = Duration::ZERO;
750        let mut last_message = Duration::ZERO;
751        let mut progressed = false;
752        let mut end: Option<Event> = None;
753        let mut settled: Option<Decision> = None;
754
755        for _ in 0..STEPS {
756            // Time never runs backwards; a message may or may not arrive.
757            now += small();
758            if kani::any() {
759                let event: Event = kani::any();
760                tracker.record(now, event);
761                if end.is_none() {
762                    last_message = now;
763                    if event == Event::Progress {
764                        progressed = true;
765                    } else {
766                        end = Some(event);
767                    }
768                }
769            }
770
771            let state = match end {
772                Some(Event::Ready) => State::Ready,
773                Some(Event::Closed) => State::Died,
774                Some(Event::Malformed) => State::Malformed,
775                Some(Event::Progress) | None if progressed => State::Progressing,
776                Some(Event::Progress) | None => State::Waiting,
777            };
778            assert!(tracker.state == state);
779
780            let decision = tracker.decide(now, silence);
781            match end {
782                Some(Event::Ready) => assert!(decision == Decision::Ready),
783                // The end of the channel before ready fails at once,
784                // whatever the silence bound.
785                Some(Event::Closed) => assert!(decision == Decision::NotReady(NotReady::Died)),
786                Some(Event::Malformed) => {
787                    assert!(decision == Decision::NotReady(NotReady::Malformed));
788                }
789                // Every message restarts the silence bound.
790                Some(Event::Progress) | None => {
791                    let until = last_message + silence;
792                    if now >= until {
793                        assert!(decision == Decision::NotReady(NotReady::Silent));
794                    } else {
795                        assert!(decision == Decision::Wait(until - now));
796                    }
797                }
798            }
799
800            // A final answer never changes.
801            if let Some(earlier) = settled {
802                assert!(decision == earlier);
803            }
804            if matches!(
805                decision,
806                Decision::Ready | Decision::NotReady(NotReady::Died | NotReady::Malformed)
807            ) {
808                settled = Some(decision);
809            }
810        }
811    }
812}
813
814/// The promises of the Kani harness over long runs, and the wire format over
815/// generated streams.
816#[cfg(test)]
817mod properties {
818    use super::*;
819    use proptest::prelude::*;
820
821    fn config() -> ProptestConfig {
822        ProptestConfig {
823            cases: 256,
824            failure_persistence: None,
825            ..ProptestConfig::default()
826        }
827    }
828
829    fn event() -> impl Strategy<Value = Event> {
830        prop_oneof![
831            6 => Just(Event::Progress),
832            1 => Just(Event::Ready),
833            1 => Just(Event::Malformed),
834            1 => Just(Event::Closed),
835        ]
836    }
837
838    fn step() -> impl Strategy<Value = Duration> {
839        (0u64..5_000_000).prop_map(Duration::from_micros)
840    }
841
842    /// Pieces of a stream: messages, near misses and noise.
843    fn piece() -> impl Strategy<Value = Vec<u8>> {
844        prop_oneof![
845            3 => Just(b"ready\n".to_vec()),
846            3 => Just(b"progress\n".to_vec()),
847            3 => "progress [ -~]{1,40}\n".prop_map(String::into_bytes),
848            1 => "progress [ -~]{240,250}\n".prop_map(String::into_bytes),
849            1 => "[ -~]{0,300}".prop_map(String::into_bytes),
850            1 => proptest::collection::vec(any::<u8>(), 0..8),
851            1 => Just(b"\n".to_vec()),
852        ]
853    }
854
855    /// Whether `line`, with its newline, is exactly what a daemon sends.
856    fn sent_by_a_daemon(line: &[u8]) -> bool {
857        if line == b"ready\n" {
858            return true;
859        }
860        let Some(detail) = line
861            .strip_prefix(b"progress")
862            .and_then(|rest| rest.strip_suffix(b"\n"))
863        else {
864            return false;
865        };
866        let detail = match detail.strip_prefix(b" ") {
867            Some(detail) if !detail.is_empty() => detail,
868            Some(_) => return false,
869            None if detail.is_empty() => detail,
870            None => return false,
871        };
872        std::str::from_utf8(detail)
873            .ok()
874            .and_then(|detail| progress_line(detail).ok())
875            .is_some_and(|encoded| encoded == line)
876    }
877
878    proptest! {
879        #![proptest_config(config())]
880
881        #[test]
882        fn a_launcher_learns_only_what_its_daemon_said(
883            silence in step(),
884            steps in proptest::collection::vec((step(), proptest::option::of(event())), 1..64),
885        ) {
886            let mut tracker = Tracker::default();
887            let mut now = Duration::ZERO;
888            let mut last_message = Duration::ZERO;
889            let mut end: Option<Event> = None;
890            for (elapsed, event) in steps {
891                now += elapsed;
892                if let Some(event) = event {
893                    tracker.record(now, event);
894                    if end.is_none() {
895                        last_message = now;
896                        if event != Event::Progress {
897                            end = Some(event);
898                        }
899                    }
900                }
901                let decision = tracker.decide(now, silence);
902                let expected = match end {
903                    Some(Event::Ready) => Decision::Ready,
904                    Some(Event::Closed) => Decision::NotReady(NotReady::Died),
905                    Some(Event::Malformed) => Decision::NotReady(NotReady::Malformed),
906                    Some(Event::Progress) | None if now >= last_message + silence => {
907                        Decision::NotReady(NotReady::Silent)
908                    }
909                    Some(Event::Progress) | None => Decision::Wait(last_message + silence - now),
910                };
911                prop_assert_eq!(decision, expected);
912            }
913        }
914
915        #[test]
916        fn every_message_decoded_is_exactly_what_a_daemon_sent(
917            pieces in proptest::collection::vec(piece(), 0..24),
918        ) {
919            let mut decoder = Decoder::default();
920            let mut line = Vec::new();
921            for byte in pieces.concat() {
922                line.push(byte);
923                match decoder.push(byte) {
924                    Step::More => prop_assert!(line.len() < MAX_MESSAGE),
925                    Step::Message(_) => {
926                        prop_assert!(sent_by_a_daemon(&line), "{}", line.escape_ascii());
927                        line.clear();
928                    }
929                    Step::Malformed => {
930                        prop_assert!(
931                            line.len() == MAX_MESSAGE || !sent_by_a_daemon(&line),
932                            "{}",
933                            line.escape_ascii()
934                        );
935                        break;
936                    }
937                }
938            }
939        }
940
941        #[test]
942        fn whatever_a_daemon_sends_is_decoded_as_sent(
943            details in proptest::collection::vec(
944                prop_oneof![Just(String::new()), "[ -~]{1,246}"],
945                0..8,
946            ),
947        ) {
948            let mut stream = Vec::new();
949            for detail in &details {
950                stream.extend(progress_line(detail).unwrap());
951            }
952            stream.extend_from_slice(b"ready\n");
953            let mut decoder = Decoder::default();
954            let messages: Vec<Message> = stream
955                .into_iter()
956                .filter_map(|byte| match decoder.push(byte) {
957                    Step::Message(message) => Some(message),
958                    Step::More | Step::Malformed => None,
959                })
960                .collect();
961            let mut expected: Vec<Message> = details
962                .into_iter()
963                .map(|detail| Message::Progress((!detail.is_empty()).then_some(detail)))
964                .collect();
965            expected.push(Message::Ready);
966            prop_assert_eq!(messages, expected);
967        }
968    }
969}
970
971#[cfg(test)]
972mod tests {
973    use super::*;
974    use crate::selection::{Budget, Selection, await_selection};
975    use std::io::PipeWriter;
976
977    /// Long enough never to pass in a test that expects an answer first.
978    const LONG: Duration = Duration::from_secs(20);
979
980    fn open() -> (Channel, PipeWriter) {
981        let (reader, writer) = std::io::pipe().unwrap();
982        (Channel::start(reader).unwrap(), writer)
983    }
984
985    fn ms(millis: u64) -> Duration {
986        Duration::from_millis(millis)
987    }
988
989    // ── State machine ───────────────────────────────────────────────
990
991    #[test]
992    fn signaled_errors_preserve_the_failed_probe_or_channel() {
993        let probe = SignaledError::Evidence(io::Error::other("unreadable record"));
994        assert_eq!(
995            std::error::Error::source(&probe).unwrap().to_string(),
996            "unreadable record"
997        );
998        let channel: SignaledError<io::Error> = SignaledError::NotReady(NotReady::Died);
999        assert_eq!(
1000            std::error::Error::source(&channel).unwrap().to_string(),
1001            NotReady::Died.to_string()
1002        );
1003    }
1004
1005    #[test]
1006    fn progress_restarts_the_silence_bound_so_a_slow_start_is_not_cut_off() {
1007        // A total deadline of one second would fail this daemon at 1 s; it
1008        // reports progress every 0.9 s and is ready at 3.6 s.
1009        let silence = ms(1000);
1010        let mut tracker = Tracker::default();
1011        for step in 1..=3 {
1012            let at = ms(900 * step);
1013            tracker.record(at, Event::Progress);
1014            assert_eq!(tracker.decide(at, silence), Decision::Wait(silence));
1015            assert_eq!(
1016                tracker.decide(at + ms(899), silence),
1017                Decision::Wait(ms(101))
1018            );
1019        }
1020        tracker.record(ms(3600), Event::Ready);
1021        assert_eq!(tracker.decide(ms(3600), silence), Decision::Ready);
1022        assert_eq!(tracker.decide(ms(100_000), silence), Decision::Ready);
1023    }
1024
1025    #[test]
1026    fn silence_ends_the_wait_exactly_at_the_bound() {
1027        let silence = ms(1000);
1028        let mut tracker = Tracker::default();
1029        assert_eq!(tracker.state, State::Waiting);
1030        assert_eq!(tracker.decide(ms(300), silence), Decision::Wait(ms(700)));
1031        assert_eq!(
1032            tracker.decide(silence, silence),
1033            Decision::NotReady(NotReady::Silent)
1034        );
1035        tracker.record(ms(2000), Event::Progress);
1036        assert_eq!(tracker.state, State::Progressing);
1037        assert_eq!(tracker.decide(ms(2600), silence), Decision::Wait(ms(400)));
1038        assert_eq!(
1039            tracker.decide(ms(3000), silence),
1040            Decision::NotReady(NotReady::Silent)
1041        );
1042        // An unbounded silence never overflows.
1043        assert_eq!(
1044            tracker.decide(ms(3000), Duration::MAX),
1045            Decision::Wait(Duration::MAX - ms(3000))
1046        );
1047    }
1048
1049    #[test]
1050    fn the_first_final_message_decides_for_good() {
1051        let mut tracker = Tracker::default();
1052        tracker.record(ms(1000), Event::Closed);
1053        tracker.record(ms(2000), Event::Ready);
1054        assert_eq!(tracker.state, State::Died);
1055        assert_eq!(
1056            tracker.decide(Duration::ZERO, Duration::ZERO),
1057            Decision::NotReady(NotReady::Died)
1058        );
1059
1060        let mut tracker = Tracker::default();
1061        tracker.record(ms(1000), Event::Malformed);
1062        tracker.record(ms(2000), Event::Ready);
1063        assert_eq!(
1064            tracker.decide(ms(3000), LONG),
1065            Decision::NotReady(NotReady::Malformed)
1066        );
1067
1068        let mut tracker = Tracker::default();
1069        tracker.record(ms(1000), Event::Ready);
1070        tracker.record(ms(2000), Event::Closed);
1071        assert_eq!(tracker.decide(ms(3000), Duration::ZERO), Decision::Ready);
1072    }
1073
1074    // ── Wire format ─────────────────────────────────────────────────
1075
1076    #[test]
1077    fn messages_parse_strictly() {
1078        assert_eq!(parse(b"ready"), Some(Message::Ready));
1079        assert_eq!(parse(b"progress"), Some(Message::Progress(None)));
1080        assert_eq!(
1081            parse(b"progress opening the cache"),
1082            Some(Message::Progress(Some("opening the cache".into())))
1083        );
1084        assert_eq!(
1085            parse(b"progress  ~"),
1086            Some(Message::Progress(Some(" ~".into())))
1087        );
1088        let refused: [&[u8]; 11] = [
1089            b"",
1090            b"READY",
1091            b"ready ",
1092            b"ready\r",
1093            b"progress ",
1094            b"progressing",
1095            b"progress\tx",
1096            b"progress caf\xc3\xa9",
1097            b"progress x\x7f",
1098            b"progress x\x1b[2J",
1099            b"hello",
1100        ];
1101        for line in refused {
1102            assert_eq!(parse(line), None, "{}", line.escape_ascii());
1103        }
1104    }
1105
1106    #[test]
1107    fn a_line_may_fill_the_limit_but_not_pass_it() {
1108        let longest = progress_line(&"x".repeat(MAX_DETAIL)).unwrap();
1109        assert_eq!(longest.len(), MAX_MESSAGE);
1110        let mut decoder = Decoder::default();
1111        let steps: Vec<Step> = longest.iter().map(|&byte| decoder.push(byte)).collect();
1112        assert_eq!(
1113            steps.last(),
1114            Some(&Step::Message(Message::Progress(Some(
1115                "x".repeat(MAX_DETAIL)
1116            ))))
1117        );
1118
1119        let mut decoder = Decoder::default();
1120        for _ in 0..MAX_MESSAGE - 1 {
1121            assert_eq!(decoder.push(b'x'), Step::More);
1122        }
1123        assert_eq!(decoder.push(b'x'), Step::Malformed);
1124    }
1125
1126    #[test]
1127    fn a_progress_detail_is_checked_before_anything_is_sent() {
1128        assert_eq!(progress_line("").unwrap(), b"progress\n");
1129        assert_eq!(progress_line("loading").unwrap(), b"progress loading\n");
1130        assert!(progress_line(&"x".repeat(MAX_DETAIL)).is_ok());
1131        for detail in [
1132            "x".repeat(MAX_DETAIL + 1),
1133            "two\nlines".into(),
1134            "café".into(),
1135            "\u{1b}[2J".into(),
1136        ] {
1137            let error = progress_line(&detail).unwrap_err();
1138            assert_eq!(error.kind(), io::ErrorKind::InvalidInput, "{detail:?}");
1139        }
1140    }
1141
1142    #[test]
1143    fn only_an_interrupted_read_is_retried() {
1144        assert!(!ends_channel(&io::ErrorKind::Interrupted.into()));
1145        assert!(ends_channel(&io::ErrorKind::BrokenPipe.into()));
1146        assert!(ends_channel(&io::Error::other("read failed")));
1147    }
1148
1149    #[test]
1150    fn errors_say_what_happened() {
1151        assert!(NotReady::Died.to_string().contains("ended before"));
1152        assert!(NotReady::Malformed.to_string().contains("malformed"));
1153        assert!(NotReady::Silent.to_string().contains("silent"));
1154        let error: SignaledError<NotReady> = SignaledError::NotReady(NotReady::Died);
1155        assert_eq!(error.to_string(), NotReady::Died.to_string());
1156        let error = SignaledError::Evidence(io::Error::other("record unreadable"));
1157        assert_eq!(error.to_string(), "record unreadable");
1158    }
1159
1160    // ── A channel over a real pipe ──────────────────────────────────
1161
1162    #[test]
1163    fn progress_then_ready() {
1164        let (mut channel, mut daemon) = open();
1165        daemon
1166            .write_all(b"progress\nprogress opening the cache\nready\n")
1167            .unwrap();
1168        assert_eq!(channel.wait(LONG), Ok(()));
1169        assert_eq!(channel.progress_reports(), 2);
1170        assert_eq!(
1171            channel.last_progress().as_deref(),
1172            Some("opening the cache")
1173        );
1174        // The answer stays, whatever the silence bound.
1175        assert_eq!(channel.wait(Duration::ZERO), Ok(()));
1176    }
1177
1178    #[test]
1179    fn a_notifier_speaks_the_protocol() {
1180        let (reader, writer) = std::io::pipe().unwrap();
1181        let mut channel = Channel::start(reader).unwrap();
1182        let mut notifier = Notifier { pipe: writer };
1183        notifier.progress("").unwrap();
1184        notifier.progress("loading").unwrap();
1185        assert_eq!(
1186            notifier.progress("bad\ndetail").unwrap_err().kind(),
1187            io::ErrorKind::InvalidInput
1188        );
1189        notifier.ready().unwrap();
1190        assert_eq!(channel.wait(LONG), Ok(()));
1191        assert_eq!(channel.progress_reports(), 2);
1192        assert_eq!(channel.last_progress().as_deref(), Some("loading"));
1193    }
1194
1195    #[test]
1196    fn the_end_of_the_channel_before_ready_fails_at_once() {
1197        let (mut channel, mut daemon) = open();
1198        daemon.write_all(b"progress\nrea").unwrap();
1199        drop(daemon);
1200        assert_eq!(channel.wait(LONG), Err(NotReady::Died));
1201        assert_eq!(channel.progress_reports(), 1);
1202
1203        let (reader, writer) = std::io::pipe().unwrap();
1204        let mut channel = Channel::start(reader).unwrap();
1205        drop(Notifier { pipe: writer });
1206        assert_eq!(channel.wait(LONG), Err(NotReady::Died));
1207    }
1208
1209    #[test]
1210    fn a_malformed_message_fails_closed() {
1211        let lines: [&[u8]; 6] = [
1212            b"READY\n",
1213            b"ready \n",
1214            b"ready\r\n",
1215            b"progress \n",
1216            b"\n",
1217            b"progress ok\nnonsense\nready\n",
1218        ];
1219        for line in lines {
1220            let (mut channel, mut daemon) = open();
1221            daemon.write_all(line).unwrap();
1222            assert_eq!(
1223                channel.wait(LONG),
1224                Err(NotReady::Malformed),
1225                "{}",
1226                line.escape_ascii()
1227            );
1228        }
1229    }
1230
1231    #[test]
1232    fn an_overlong_line_fails_without_waiting_for_its_end() {
1233        let (mut channel, mut daemon) = open();
1234        daemon.write_all(&[b'x'; MAX_MESSAGE]).unwrap();
1235        assert_eq!(channel.wait(LONG), Err(NotReady::Malformed));
1236        drop(daemon);
1237    }
1238
1239    #[test]
1240    fn nothing_after_ready_is_read() {
1241        let (mut channel, mut daemon) = open();
1242        daemon.write_all(b"ready\nnonsense\n").unwrap();
1243        drop(daemon);
1244        assert_eq!(channel.wait(LONG), Ok(()));
1245    }
1246
1247    #[test]
1248    fn a_silent_wait_can_still_see_a_later_message() {
1249        let (mut channel, mut daemon) = open();
1250        assert_eq!(channel.wait(Duration::ZERO), Err(NotReady::Silent));
1251        daemon.write_all(b"ready\n").unwrap();
1252        assert_eq!(channel.wait(LONG), Ok(()));
1253    }
1254
1255    #[test]
1256    fn dropping_the_channel_closes_the_launcher_s_end() {
1257        let (channel, mut daemon) = open();
1258        drop(channel);
1259        // The reader stops after the next message instead of reading on, so
1260        // a daemon's send fails rather than waiting on a full pipe.
1261        let error = loop {
1262            if let Err(error) = daemon.write_all(b"progress\n") {
1263                break error;
1264            }
1265        };
1266        assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
1267    }
1268
1269    // ── Selection ───────────────────────────────────────────────────
1270
1271    /// A probe that never proves the daemon, and must not be polled before
1272    /// the channel says the daemon is ready.
1273    struct Unproven {
1274        probes: usize,
1275    }
1276
1277    impl Evidence for Unproven {
1278        type Proof = ();
1279        type Error = ();
1280        fn committed(&mut self) -> Result<bool, ()> {
1281            Ok(false)
1282        }
1283        fn probe(&mut self, _: Instant) -> Result<Option<()>, ()> {
1284            self.probes += 1;
1285            Ok(None)
1286        }
1287        fn wait(&mut self, _: Instant) {
1288            panic!("polled although the channel had nothing to say");
1289        }
1290    }
1291
1292    const SELECTION: Budget = Budget {
1293        commit: Duration::from_secs(10),
1294        proof: Duration::ZERO,
1295    };
1296
1297    #[test]
1298    fn a_dead_channel_ends_the_selection_at_once_unless_a_probe_proves_the_daemon() {
1299        let (channel, daemon) = open();
1300        drop(daemon);
1301        let mut signaled = Signaled::new(Unproven { probes: 0 }, channel, LONG);
1302        assert_eq!(
1303            await_selection(SELECTION, &mut signaled),
1304            Err(SignaledError::NotReady(NotReady::Died))
1305        );
1306
1307        // Another instance serves: the proof wins over the dead channel.
1308        struct Proven;
1309        impl Evidence for Proven {
1310            type Proof = &'static str;
1311            type Error = ();
1312            fn committed(&mut self) -> Result<bool, ()> {
1313                Ok(false)
1314            }
1315            fn probe(&mut self, _: Instant) -> Result<Option<&'static str>, ()> {
1316                Ok(Some("winner"))
1317            }
1318        }
1319        let (channel, daemon) = open();
1320        drop(daemon);
1321        let mut signaled = Signaled::new(Proven, channel, LONG);
1322        assert_eq!(
1323            await_selection(SELECTION, &mut signaled),
1324            Ok(Selection::Current("winner"))
1325        );
1326    }
1327
1328    #[test]
1329    fn silence_past_the_bound_ends_the_selection() {
1330        let (channel, _daemon) = open();
1331        let mut signaled = Signaled::new(Unproven { probes: 0 }, channel, Duration::ZERO);
1332        assert_eq!(
1333            await_selection(SELECTION, &mut signaled),
1334            Err(SignaledError::NotReady(NotReady::Silent))
1335        );
1336    }
1337
1338    /// Probes made while waiting `budget` for a daemon that says nothing more.
1339    fn probes_while_quiet(channel: Channel) -> usize {
1340        let budget = Budget {
1341            commit: Duration::from_millis(300),
1342            proof: Duration::ZERO,
1343        };
1344        let mut signaled = Signaled::new(Unproven { probes: 0 }, channel, LONG);
1345        assert_eq!(
1346            await_selection(budget, &mut signaled),
1347            Ok(Selection::NotCommitted)
1348        );
1349        signaled.into_parts().0.probes
1350    }
1351
1352    #[test]
1353    fn before_the_signal_a_round_waits_for_news_not_a_poll_interval() {
1354        // Nothing arrives, so after its first probe the wait lasts until the
1355        // commit budget: one more probe at most, however slow the machine is.
1356        // Polling would probe every 25 ms.
1357        let (channel, _daemon) = open();
1358        assert!(probes_while_quiet(channel) <= 2);
1359
1360        // A message that arrived before the round began is not news to it.
1361        let (channel, mut daemon) = open();
1362        daemon.write_all(b"progress\n").unwrap();
1363        wait_for_version(&channel.shared, 1);
1364        assert!(probes_while_quiet(channel) <= 2);
1365    }
1366
1367    #[test]
1368    fn a_quiet_channel_holds_the_wait_until_its_deadline() {
1369        let (channel, _daemon) = open();
1370        let until = Instant::now() + ms(50);
1371        channel.wait_for_news(channel.version(), until, LONG);
1372        assert!(Instant::now() >= until, "returned early without news");
1373    }
1374
1375    #[test]
1376    fn news_is_whatever_arrived_since_the_round_began() {
1377        let (channel, mut daemon) = open();
1378        assert_eq!(channel.version(), 0);
1379        daemon.write_all(b"progress\n").unwrap();
1380        wait_for_version(&channel.shared, 1);
1381        assert_eq!(channel.version(), 1);
1382
1383        let mut inner = Inner {
1384            version: 1,
1385            ..Inner::default()
1386        };
1387        inner.tracker.record(ms(100), Event::Progress);
1388        let silence = ms(1000);
1389        // Unseen news: a new round now.
1390        assert_eq!(quiet_for(&inner, 0, ms(300), silence), None);
1391        // Nothing new: quiet until the silence bound.
1392        assert_eq!(quiet_for(&inner, 1, ms(300), silence), Some(ms(800)));
1393        // The bound has passed: a new round now, which reports it.
1394        assert_eq!(quiet_for(&inner, 1, ms(1100), silence), None);
1395        inner.tracker.record(ms(200), Event::Ready);
1396        assert_eq!(quiet_for(&inner, 1, ms(300), silence), None);
1397    }
1398
1399    #[test]
1400    fn errors_from_the_wrapped_evidence_end_the_wait() {
1401        struct Failing(bool);
1402        impl Evidence for Failing {
1403            type Proof = ();
1404            type Error = &'static str;
1405            fn committed(&mut self) -> Result<bool, Self::Error> {
1406                if self.0 {
1407                    Err("record unreadable")
1408                } else {
1409                    Ok(false)
1410                }
1411            }
1412            fn probe(&mut self, _: Instant) -> Result<Option<()>, Self::Error> {
1413                Err("peer is another user")
1414            }
1415        }
1416        for (record_fails, error) in [(true, "record unreadable"), (false, "peer is another user")]
1417        {
1418            let (channel, _daemon) = open();
1419            let mut signaled = Signaled::new(Failing(record_fails), channel, LONG);
1420            assert_eq!(
1421                await_selection(SELECTION, &mut signaled),
1422                Err(SignaledError::Evidence(error))
1423            );
1424        }
1425    }
1426
1427    /// Block until `shared` has seen `version` events.
1428    fn wait_for_version(shared: &Shared, version: u64) {
1429        let mut inner = shared.lock();
1430        while inner.version < version {
1431            inner = shared
1432                .changed
1433                .wait(inner)
1434                .unwrap_or_else(PoisonError::into_inner);
1435        }
1436    }
1437
1438    /// The daemon sends `message` while the first probe runs, which fails
1439    /// anyway; the second probe proves it.
1440    struct NewsDuringProbe {
1441        daemon: PipeWriter,
1442        shared: Arc<Shared>,
1443        message: &'static [u8],
1444        probes: usize,
1445    }
1446
1447    impl Evidence for NewsDuringProbe {
1448        type Proof = usize;
1449        type Error = ();
1450        fn committed(&mut self) -> Result<bool, ()> {
1451            Ok(false)
1452        }
1453        fn probe(&mut self, _: Instant) -> Result<Option<usize>, ()> {
1454            self.probes += 1;
1455            if self.probes == 1 {
1456                self.daemon.write_all(self.message).unwrap();
1457                wait_for_version(&self.shared, 1);
1458                return Ok(None);
1459            }
1460            Ok(Some(self.probes))
1461        }
1462        fn wait(&mut self, _: Instant) {
1463            panic!("waited although the channel had news");
1464        }
1465    }
1466
1467    #[test]
1468    fn news_during_an_unproven_probe_starts_the_next_round_at_once() {
1469        // A `ready` that arrives while a probe fails is news too: waiting on
1470        // the evidence's own pace would delay the probe it makes worthwhile.
1471        let messages: [&'static [u8]; 2] = [b"progress\n", b"ready\n"];
1472        for message in messages {
1473            let (channel, daemon) = open();
1474            let evidence = NewsDuringProbe {
1475                daemon,
1476                shared: Arc::clone(&channel.shared),
1477                message,
1478                probes: 0,
1479            };
1480            let mut signaled = Signaled::new(evidence, channel, LONG);
1481            assert_eq!(
1482                await_selection(SELECTION, &mut signaled),
1483                Ok(Selection::Current(2)),
1484                "{}",
1485                message.escape_ascii()
1486            );
1487        }
1488    }
1489
1490    #[test]
1491    fn after_ready_the_wrapped_evidence_paces_the_rounds() {
1492        struct Paced {
1493            probes: usize,
1494            waits: usize,
1495        }
1496        impl Evidence for Paced {
1497            type Proof = ();
1498            type Error = ();
1499            fn committed(&mut self) -> Result<bool, ()> {
1500                Ok(true)
1501            }
1502            fn probe(&mut self, _: Instant) -> Result<Option<()>, ()> {
1503                self.probes += 1;
1504                Ok((self.probes == 2).then_some(()))
1505            }
1506            fn wait(&mut self, _: Instant) {
1507                self.waits += 1;
1508            }
1509        }
1510        let (mut channel, mut daemon) = open();
1511        daemon.write_all(b"ready\n").unwrap();
1512        assert_eq!(channel.wait(LONG), Ok(()));
1513        let budget = Budget {
1514            commit: LONG,
1515            proof: LONG,
1516        };
1517        let mut signaled = Signaled::new(
1518            Paced {
1519                probes: 0,
1520                waits: 0,
1521            },
1522            channel,
1523            LONG,
1524        );
1525        // Ready is not a proof: the first probe fails and a second is made.
1526        assert_eq!(
1527            await_selection(budget, &mut signaled),
1528            Ok(Selection::Current(()))
1529        );
1530        let (evidence, _channel) = signaled.into_parts();
1531        assert_eq!(evidence.waits, 1);
1532    }
1533
1534    #[test]
1535    fn a_ready_daemon_is_still_not_current_without_a_proof() {
1536        struct Never;
1537        impl Evidence for Never {
1538            type Proof = ();
1539            type Error = ();
1540            fn committed(&mut self) -> Result<bool, ()> {
1541                Ok(false)
1542            }
1543            fn probe(&mut self, _: Instant) -> Result<Option<()>, ()> {
1544                Ok(None)
1545            }
1546        }
1547        let (mut channel, mut daemon) = open();
1548        daemon.write_all(b"ready\n").unwrap();
1549        drop(daemon);
1550        assert_eq!(channel.wait(LONG), Ok(()));
1551        let budget = Budget {
1552            commit: Duration::from_millis(100),
1553            proof: Duration::ZERO,
1554        };
1555        let mut signaled = Signaled::new(Never, channel, LONG);
1556        assert_eq!(
1557            await_selection(budget, &mut signaled),
1558            Ok(Selection::NotCommitted)
1559        );
1560    }
1561
1562    #[cfg(feature = "async")]
1563    mod asynchronous {
1564        use super::*;
1565        use crate::selection::await_selection_async;
1566
1567        #[tokio::test]
1568        async fn progress_then_ready() {
1569            let (mut channel, mut daemon) = open();
1570            daemon.write_all(b"progress loading\nready\n").unwrap();
1571            assert_eq!(channel.wait_async(LONG).await, Ok(()));
1572            assert_eq!(channel.progress_reports(), 1);
1573            assert_eq!(channel.last_progress().as_deref(), Some("loading"));
1574        }
1575
1576        #[tokio::test]
1577        async fn the_end_of_the_channel_before_ready_fails_at_once() {
1578            let (mut channel, daemon) = open();
1579            drop(daemon);
1580            assert_eq!(channel.wait_async(LONG).await, Err(NotReady::Died));
1581        }
1582
1583        #[tokio::test]
1584        async fn silence_past_the_bound() {
1585            let (mut channel, _daemon) = open();
1586            assert_eq!(
1587                channel.wait_async(Duration::ZERO).await,
1588                Err(NotReady::Silent)
1589            );
1590        }
1591
1592        struct AsyncNewsDuringProbe {
1593            daemon: PipeWriter,
1594            shared: Arc<Shared>,
1595            message: &'static [u8],
1596            probes: usize,
1597        }
1598
1599        impl AsyncEvidence for AsyncNewsDuringProbe {
1600            type Proof = usize;
1601            type Error = ();
1602            async fn committed(&mut self) -> Result<bool, ()> {
1603                Ok(false)
1604            }
1605            async fn probe(&mut self) -> Result<Option<usize>, ()> {
1606                self.probes += 1;
1607                if self.probes == 1 {
1608                    self.daemon.write_all(self.message).unwrap();
1609                    wait_for_version(&self.shared, 1);
1610                    return Ok(None);
1611                }
1612                Ok(Some(self.probes))
1613            }
1614            async fn wait(&mut self, _: tokio::time::Instant) {
1615                panic!("waited although the channel had news");
1616            }
1617        }
1618
1619        #[tokio::test]
1620        async fn news_during_an_unproven_probe_starts_the_next_round_at_once() {
1621            let messages: [&'static [u8]; 2] = [b"progress\n", b"ready\n"];
1622            for message in messages {
1623                let (channel, daemon) = open();
1624                let evidence = AsyncNewsDuringProbe {
1625                    daemon,
1626                    shared: Arc::clone(&channel.shared),
1627                    message,
1628                    probes: 0,
1629                };
1630                let mut signaled = Signaled::new(evidence, channel, LONG);
1631                assert_eq!(
1632                    await_selection_async(SELECTION, &mut signaled).await,
1633                    Ok(Selection::Current(2)),
1634                    "{}",
1635                    message.escape_ascii()
1636                );
1637            }
1638        }
1639
1640        struct AsyncUnproven;
1641
1642        impl AsyncEvidence for AsyncUnproven {
1643            type Proof = ();
1644            type Error = ();
1645            async fn committed(&mut self) -> Result<bool, ()> {
1646                Ok(false)
1647            }
1648            async fn probe(&mut self) -> Result<Option<()>, ()> {
1649                Ok(None)
1650            }
1651        }
1652
1653        struct AsyncQuiet {
1654            probes: usize,
1655        }
1656
1657        impl AsyncEvidence for AsyncQuiet {
1658            type Proof = ();
1659            type Error = ();
1660            async fn committed(&mut self) -> Result<bool, ()> {
1661                Ok(false)
1662            }
1663            async fn probe(&mut self) -> Result<Option<()>, ()> {
1664                self.probes += 1;
1665                Ok(None)
1666            }
1667            async fn wait(&mut self, _: tokio::time::Instant) {
1668                panic!("polled although the channel had nothing to say");
1669            }
1670        }
1671
1672        /// Probes made while waiting out a 300 ms budget, in paused time,
1673        /// for a daemon that says nothing more.
1674        async fn probes_while_quiet(channel: Channel) -> usize {
1675            let budget = Budget {
1676                commit: Duration::from_millis(300),
1677                proof: Duration::ZERO,
1678            };
1679            let mut signaled = Signaled::new(AsyncQuiet { probes: 0 }, channel, LONG);
1680            assert_eq!(
1681                await_selection_async(budget, &mut signaled).await,
1682                Ok(Selection::NotCommitted)
1683            );
1684            signaled.into_parts().0.probes
1685        }
1686
1687        #[tokio::test(start_paused = true)]
1688        async fn before_the_signal_a_round_waits_for_news_not_a_poll_interval() {
1689            // Paused time moves only when every task waits, so the first round
1690            // cannot outlast the budget: exactly one probe, then one at its end.
1691            let (channel, _daemon) = open();
1692            assert_eq!(probes_while_quiet(channel).await, 2);
1693
1694            // A message that arrived before the round began is not news to it.
1695            let (channel, mut daemon) = open();
1696            daemon.write_all(b"progress\n").unwrap();
1697            wait_for_version(&channel.shared, 1);
1698            assert_eq!(probes_while_quiet(channel).await, 2);
1699        }
1700
1701        #[tokio::test(start_paused = true)]
1702        async fn a_quiet_channel_holds_the_wait_until_its_deadline() {
1703            let (channel, _daemon) = open();
1704            let until = tokio::time::Instant::now() + ms(300);
1705            channel
1706                .wait_for_news_async(channel.version(), until, LONG)
1707                .await;
1708            assert!(tokio::time::Instant::now() >= until);
1709        }
1710
1711        #[tokio::test]
1712        async fn errors_from_the_wrapped_evidence_end_the_wait() {
1713            struct Failing;
1714            impl AsyncEvidence for Failing {
1715                type Proof = ();
1716                type Error = &'static str;
1717                async fn committed(&mut self) -> Result<bool, Self::Error> {
1718                    Err("record unreadable")
1719                }
1720                async fn probe(&mut self) -> Result<Option<()>, Self::Error> {
1721                    Ok(None)
1722                }
1723            }
1724            let (channel, _daemon) = open();
1725            let mut signaled = Signaled::new(Failing, channel, LONG);
1726            assert_eq!(
1727                await_selection_async(SELECTION, &mut signaled).await,
1728                Err(SignaledError::Evidence("record unreadable"))
1729            );
1730        }
1731
1732        #[tokio::test]
1733        async fn a_dead_channel_ends_the_selection_at_once() {
1734            let (channel, daemon) = open();
1735            drop(daemon);
1736            let mut signaled = Signaled::new(AsyncUnproven, channel, LONG);
1737            assert_eq!(
1738                await_selection_async(SELECTION, &mut signaled).await,
1739                Err(SignaledError::NotReady(NotReady::Died))
1740            );
1741        }
1742
1743        #[tokio::test]
1744        async fn after_ready_the_wrapped_evidence_paces_the_rounds() {
1745            struct Paced {
1746                probes: usize,
1747                waits: usize,
1748            }
1749            impl AsyncEvidence for Paced {
1750                type Proof = ();
1751                type Error = ();
1752                async fn committed(&mut self) -> Result<bool, ()> {
1753                    Ok(true)
1754                }
1755                async fn probe(&mut self) -> Result<Option<()>, ()> {
1756                    self.probes += 1;
1757                    Ok((self.probes == 2).then_some(()))
1758                }
1759                async fn wait(&mut self, _: tokio::time::Instant) {
1760                    self.waits += 1;
1761                }
1762            }
1763            let (mut channel, mut daemon) = open();
1764            daemon.write_all(b"ready\n").unwrap();
1765            assert_eq!(channel.wait_async(LONG).await, Ok(()));
1766            let budget = Budget {
1767                commit: LONG,
1768                proof: LONG,
1769            };
1770            let mut signaled = Signaled::new(
1771                Paced {
1772                    probes: 0,
1773                    waits: 0,
1774                },
1775                channel,
1776                LONG,
1777            );
1778            assert_eq!(
1779                await_selection_async(budget, &mut signaled).await,
1780                Ok(Selection::Current(()))
1781            );
1782            let (evidence, _channel) = signaled.into_parts();
1783            assert_eq!(evidence.waits, 1);
1784        }
1785    }
1786}