shep-daemon 0.25.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
//! Buffering, flushing, reopen, rotation, line-tearing and stamp timing.

use core::pin::Pin;
use core::sync::atomic::{AtomicUsize, Ordering};
use core::task::{Context, Poll};
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;

use tokio::io::{AsyncWrite, BufWriter};
use tokio::sync::oneshot;
use tokio::time::{Instant, timeout};

use shep_core::logstamp::{LOG_STAMP_BYTES, Stamper, stamp_into};

use crate::runner::LogCtl;

use super::super::log_file::{LogFile, LogFiles, PipeFds, open_append, record_lock};
#[cfg(unix)]
use super::super::pump::LIVE_WAIT;
use super::super::{CHANNEL_CAPACITY, IDLE_FLUSH, LOG_BUFFER};
use super::*;

/// Lines longer than `LOG_BUFFER` force the multi-`poll_write` case
/// `O_APPEND` alone cannot make atomic, which is what this guards.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn narration_cannot_tear_a_line_the_pump_is_writing() {
    let dir = tempfile::tempdir().expect("a temp dir");
    let path = dir.path().join("torn.log");

    let long = "x".repeat(LOG_BUFFER + 512);
    let pump = {
        let path = path.clone();
        let long = long.clone();
        tokio::spawn(async move {
            let mut log = LogFile::open(path, true).await;
            for i in 0..1200 {
                log.append(&format!("{i} {long}")).await;
                if i % 4 == 0 {
                    tokio::task::yield_now().await;
                }
            }
            log.flush().await.expect("the pump's own flush");
        })
    };
    let narrator = {
        let path = path.clone();
        tokio::spawn(async move {
            for i in 0..600 {
                let mut written = String::new();
                stamp_into(&mut written);
                written.push_str(&format!("[shep] narration {i}\n"));
                let file = open_append(&path).await.expect("the narration handle");
                let _record = record_lock(&path).lock_owned().await;
                let mut file = file;
                use tokio::io::AsyncWriteExt as _;
                file.write_all(written.as_bytes()).await.expect("write");
                file.flush().await.expect("flush");
            }
        })
    };
    pump.await.expect("the pump task");
    narrator.await.expect("the narration task");

    let text = std::fs::read_to_string(&path).expect("the log is readable");
    let mut lines = 0_usize;
    for line in text.lines() {
        lines += 1;
        let once = shep_core::logstamp::strip(line);
        assert_ne!(
            once, line,
            "a line reached the file with no stamp: {line:.80}"
        );
        assert_eq!(
            shep_core::logstamp::strip(once),
            once,
            "a line carries two stamps, so a record was torn: {line:.80}"
        );
    }
    assert_eq!(lines, 1800, "every record reached the file exactly once");
}

/// The pump's `select!` has a branch for the sheep task dropping its `logs`
/// receiver, for a lamb that holds the pipe open past the child's own exit.
/// That branch competes with the read branches rather than following them, so
/// a child that writes and exits can leave both ready in the same poll.
///
/// No child here: the harness writes the line and then does what `run_sheep`
/// does on return, isolating the race from the process lifecycle.
#[tokio::test]
async fn a_last_line_written_before_the_sheep_task_lets_go_reaches_the_file() {
    for attempt in 0..RACE_ATTEMPTS {
        let mut pump = PumpHarness::start();
        pump.out_writer.write_all(b"last-words\n").await.unwrap();
        // The child exiting, then `run_sheep` breaking out of its loop:
        // the write end closes and the receiver goes, in that order,
        // with the line still unread in between.
        drop(pump.out_writer);
        drop(pump.err_writer);
        drop(pump.logs);

        let settled = wait_for_file_to_contain(&pump.out_path, "last-words").await;
        assert!(
            settled,
            "attempt {attempt}: the pump dropped the line it had not read yet, \
             leaving {:?}",
            fs::read_to_string(&pump.out_path)
        );
    }
}

/// The same line, lost the other way: through the control channel rather than
/// through `logs`.
///
/// `logs` and `log_ctl` do not close together ordinarily: the sheep task drops
/// `logs` when its child exits, while the slot's `log_ctl` sender outlives it.
/// Those are the cases where both close at once, giving the control arm's own
/// `None` a fourth way out of the loop that can win the same random pick.
#[tokio::test]
async fn a_last_line_survives_the_control_channel_closing_with_the_logs() {
    for attempt in 0..RACE_ATTEMPTS {
        let mut pump = PumpHarness::start();
        pump.out_writer.write_all(b"last-words\n").await.unwrap();
        drop(pump.out_writer);
        drop(pump.err_writer);
        // A delete: the slot's sender and the sheep task's receiver go
        // in the same moment, so all four exits are live at once.
        drop(pump.ctl);
        drop(pump.logs);

        let settled = wait_for_file_to_contain(&pump.out_path, "last-words").await;
        assert!(
            settled,
            "attempt {attempt}: the pump took an exit that skips the drain, \
             leaving {:?}",
            fs::read_to_string(&pump.out_path)
        );
    }
}

/// Whether `path` holds `needle` in its text before [`PUMP_DEADLINE`] runs
/// out, polling every 2ms.
///
/// The two `a_last_line_` races share the wait and keep their own verdict:
/// each names the exit it blames and its attempt number, which a shared
/// assertion could not say. Reads through `unwrap_or_default`, since a pump
/// that never opened the path is one of the outcomes worth reporting.
async fn wait_for_file_to_contain(path: &Path, needle: &str) -> bool {
    timeout(PUMP_DEADLINE, async {
        while !fs::read_to_string(path)
            .unwrap_or_default()
            .contains(needle)
        {
            tokio::time::sleep(Duration::from_millis(2)).await;
        }
    })
    .await
    .is_ok()
}

/// A sink standing in for the [`tokio::fs::File`] a real [`LogFile`]
/// holds, counting the writes that reach it.
///
/// Bytes on disk come out the same whether or not appends are batched;
/// only the write count shows it, so a counter is what pins
/// [`LOG_BUFFER`] against a future append that writes straight through.
#[derive(Clone, Debug, Default)]
struct WriteCounter {
    writes: Arc<AtomicUsize>,
    bytes: Arc<AtomicUsize>,
}

impl WriteCounter {
    fn writes(&self) -> usize {
        self.writes.load(Ordering::Relaxed)
    }

    fn bytes(&self) -> usize {
        self.bytes.load(Ordering::Relaxed)
    }
}

impl AsyncWrite for WriteCounter {
    fn poll_write(
        self: Pin<&mut Self>,
        _cx: &mut Context<'_>,
        buf: &[u8],
    ) -> Poll<io::Result<usize>> {
        self.writes.fetch_add(1, Ordering::Relaxed);
        self.bytes.fetch_add(buf.len(), Ordering::Relaxed);
        Poll::Ready(Ok(buf.len()))
    }

    fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
        Poll::Ready(Ok(()))
    }

    fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
        Poll::Ready(Ok(()))
    }
}

/// A [`LogFile`] writing through `sink`, for the cases that drive one directly
/// rather than through [`PumpHarness`]. The one place these tests name
/// [`LogFile`]'s fields, and generic over the sink for the same reason
/// [`LogFile`] is.
///
/// Neither goes through [`LogFile::open`], which is the one that swallows a
/// failed open. The buffer is the production [`LOG_BUFFER`] over `sink`.
fn buffered_log<W: AsyncWrite + Unpin>(path: &Path, sink: W) -> LogFile<W> {
    LogFile {
        record: record_lock(path),
        path: path.to_path_buf(),
        handle: Some(BufWriter::with_capacity(LOG_BUFFER, sink)),
        buffered_since: None,
        stamp: String::new(),
        stamper: Some(Stamper::default()),
    }
}

/// `tokio::fs::File` hands each write to the blocking pool: 32.8 us of daemon
/// CPU per line against 0.99 us for the `write(2)` underneath. A buffer turns
/// N writes into one per bufferful, and the write count is the only observable
/// that shows it.
///
/// Paused clock: no time passes, so [`IDLE_FLUSH`] never fires.
#[tokio::test(start_paused = true)]
async fn a_run_of_lines_costs_one_write_per_bufferful_not_one_per_line() {
    // 69 characters: `append`'s newline makes a 70-byte line, the size
    // `LOG_BUFFER` was measured against. The stamp is counted
    // separately so 70 stays that number.
    const LINE: &str = "012345678901234567890123456789012345678901234567890123456789012345678";
    const LINE_BYTES: usize = LOG_STAMP_BYTES + LINE.len() + 1;
    let lines = 3 * LOG_BUFFER / LINE_BYTES;

    let sink = WriteCounter::default();
    let mut log = buffered_log(Path::new("counted.log"), sink.clone());
    for _ in 0..lines {
        log.append(LINE).await;
    }
    log.flush().await.unwrap();

    let total = lines * LINE_BYTES;
    let ceiling = total.div_ceil(LOG_BUFFER) + 1; // + the closing flush's partial buffer
    assert_eq!(
        sink.bytes(),
        total,
        "buffering must not lose or repeat a byte"
    );
    assert!(
        sink.writes() <= ceiling,
        "{lines} lines cost {} writes; one per bufferful plus the closing flush is {ceiling}",
        sink.writes()
    );
    assert!(
        sink.writes() < lines,
        "a write per line is the regression this exists to catch: \
         {lines} lines, {} writes",
        sink.writes()
    );
}

/// Fails if a line only leaves the buffer when the buffer fills or when
/// something asks: that is, if [`IDLE_FLUSH`] bounds nothing. A sheep that logs
/// once and goes quiet, with no `Flush`, no reopen, and no second line.
///
/// Paused clock: the wait is on [`IDLE_FLUSH`], and a real 50ms would be a
/// claim about the machine. Counting sink: a file would only confirm the bytes
/// left once the blocking pool caught up.
#[tokio::test(start_paused = true)]
async fn a_line_from_a_sheep_that_then_goes_quiet_still_reaches_its_file() {
    const LINE: &str = "the-only-line";
    let sink = WriteCounter::default();
    let mut log = buffered_log(Path::new("quiet.log"), sink.clone());

    log.append(LINE).await;
    assert_eq!(
        sink.bytes(),
        0,
        "one short line must sit in the buffer, or there is nothing for the idle flush to do"
    );

    let armed = log
        .buffered_since
        .expect("an appended line must arm the flush deadline");
    tokio::time::sleep_until(armed + IDLE_FLUSH).await;
    log.flush().await.unwrap();

    assert_eq!(
        sink.bytes(),
        LOG_STAMP_BYTES + LINE.len() + 1,
        "the stamp, the line and its newline must reach the file once the window closes"
    );
    assert_eq!(
        log.buffered_since, None,
        "a flush must retire the deadline, or the pump re-arms it every window"
    );
}

/// Fails if a line reaches its file without the time it was written.
///
/// Both files, and past a reopen: the setting belongs to the sheep, not
/// to the handle a rotation replaces. Read raw, since `log_text` strips.
#[tokio::test]
async fn a_sheep_with_log_timestamps_off_gets_its_lines_as_written() {
    let mut pump = PumpHarness::start_stamped(false);
    pump.feed(false, "out before").await;
    pump.feed(true, "err before").await;
    pump.reopen().await;
    pump.feed(false, "out after").await;
    pump.flush().await;

    let raw = |path: &PathBuf| fs::read_to_string(path).unwrap();
    assert_eq!(raw(&pump.out_path), "out before\nout after\n");
    assert_eq!(raw(&pump.err_path), "err before\n");
}

/// `mtime` answers for the whole file and only until something touches it
/// again, so a per-line stamp is what an operator can trust after a rotation.
///
/// Asserts the shape and width, not a value: the value is the wall clock, and
/// pinning it would test the clock instead. The width is what every reader
/// stripping the prefix depends on, and what [`LOG_STAMP_BYTES`] claims.
#[tokio::test]
async fn a_line_carries_the_time_it_was_written() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("stamped.log");
    let mut log = LogFile::open(path.clone(), true).await;

    log.append("the-sheep-said-this").await;
    log.flush().await.expect("the file must be writable");

    let written = fs::read_to_string(&path).unwrap();
    let line = written.strip_suffix('\n').expect("one whole line");
    let (stamp, rest) = line.split_at(LOG_STAMP_BYTES);
    assert_eq!(
        rest, "the-sheep-said-this",
        "the sheep's own bytes must survive the prefix, whole and unaltered"
    );
    let stamp = stamp
        .strip_suffix(' ')
        .expect("one space must separate the stamp from the line");
    let parsed = chrono::DateTime::parse_from_rfc3339(stamp)
        .unwrap_or_else(|err| panic!("{stamp:?} must parse as RFC 3339: {err}"));
    // A minute of slack, against a line written a moment ago: wide
    // enough that a loaded runner cannot fail it, narrow enough that a
    // stamp reading the epoch, the wrong unit, or the wrong offset does.
    let drift = (chrono::Utc::now() - parsed.to_utc()).num_seconds().abs();
    assert!(
        drift < 60,
        "{stamp:?} is {drift}s from now; the stamp must be the moment the line was written"
    );
}

/// Fails if the idle-flush window is measured from the newest buffered
/// line rather than the oldest: a stream logging just inside the window
/// would then keep pushing the deadline out, and "buffered but not on
/// disk" would have no bound at all.
///
/// Paused clock: the question is arithmetic over `Instant`s, and an
/// append that fits in the buffer touches no file to wait on.
#[tokio::test(start_paused = true)]
async fn the_idle_flush_window_is_measured_from_the_oldest_buffered_line() {
    let dir = tempfile::tempdir().unwrap();
    let mut files = LogFiles {
        out: LogFile::open(dir.path().join("out.log"), true).await,
        err: LogFile::open(dir.path().join("err.log"), true).await,
        pipes: PipeFds::default(),
        #[cfg(unix)]
        parked: false,
        behind: false,
        live_waited: Duration::ZERO,
    };
    assert_eq!(
        files.flush_deadline(),
        None,
        "an untouched pair owes no flush"
    );

    let oldest = Instant::now();
    files.stream(false).append("out-first").await;
    assert_eq!(files.flush_deadline(), Some(oldest + IDLE_FLUSH));

    // Both streams, so the deadline is answering for the pair rather
    // than for whichever one happened to be written last.
    tokio::time::advance(IDLE_FLUSH / 2).await;
    files.stream(false).append("out-second").await;
    files.stream(true).append("err-first").await;
    assert_eq!(
        files.flush_deadline(),
        Some(oldest + IDLE_FLUSH),
        "a later line must not push the window out"
    );

    files.flush_idle().await;
    assert_eq!(
        files.flush_deadline(),
        None,
        "a flush must retire the deadline it satisfied"
    );
}

/// Fails if the pump waits for room on `logs` with a bare
/// `logs_tx.send(...).await` inside the `select!` handler: a handler is not
/// cancellable, so the control channel goes unpolled for as long as that wait
/// lasts, and with nothing draining `logs` it lasts forever.
///
/// One layer up this is the cycle `Actor::claim_manual` documents: the party
/// that drains `logs` is the sheep task.
#[tokio::test]
async fn a_reopen_is_answered_while_the_logs_channel_is_full() {
    let mut pump = PumpHarness::start();

    // One line more than `logs` can hold, and nothing here drains it:
    // the pump appends every line to the file, hands CHANNEL_CAPACITY of
    // them to the channel, and is left waiting for room for the last.
    let flooded = CHANNEL_CAPACITY + 1;
    let mut written = String::new();
    for n in 0..flooded {
        let line = format!("line-{n}\n");
        pump.out_writer.write_all(line.as_bytes()).await.unwrap();
        written.push_str(&line);
    }

    // The append comes before the send, so a file holding every line is
    // proof the pump has read the last one and is parked on its send,
    // so the reopen below lands on a pump already waiting, not one
    // that merely might be.
    assert_file_settles(&pump.out_path, &written).await;
    // Deleted rather than renamed, so the reopen under test is the only
    // thing that could put these paths back.
    fs::remove_file(&pump.out_path).unwrap();
    fs::remove_file(&pump.err_path).unwrap();

    pump.reopen().await;
    assert!(pump.out_path.exists(), "stdout's path must be back");
    assert!(pump.err_path.exists(), "stderr's path must be back");

    // The line the pump was holding is owed to the receiver, not
    // dropped: serving the reopen must not have cost the send its place.
    for n in 0..flooded {
        let observed = timeout(PUMP_DEADLINE, pump.logs.recv())
            .await
            .expect("the pump must resume once `logs` has room")
            .expect("the pump must not end while its streams are open");
        assert_eq!(observed.line, format!("line-{n}"));
    }
}

/// Fails if a pty pump stops reading its stream because `logs` is full: the
/// lines past the channel's capacity never reach the file.
#[cfg(unix)]
#[tokio::test]
async fn a_pty_pump_keeps_reading_past_a_full_logs_channel() {
    let mut pump = PumpHarness::start_as_a_pty_pump();
    let past = CHANNEL_CAPACITY + 10;
    let mut written = String::new();
    for n in 0..past {
        let line = format!("line-{n}\n");
        pump.out_writer.write_all(line.as_bytes()).await.unwrap();
        written.push_str(&line);
    }

    // Nothing here reads `logs`, so a pump that waits for room on it never
    // reaches the lines past the one that filled it.
    assert_file_settles(&pump.out_path, &written).await;
}

/// Fails if a pty pump skips a line the moment `logs` is full, rather than
/// spending [`LIVE_WAIT`] on a reader that keeps up.
#[cfg(unix)]
#[tokio::test]
async fn a_pty_pump_forwards_a_burst_to_a_reader_that_keeps_up() {
    const BURST_LINES: usize = CHANNEL_CAPACITY * 10;
    let mut pump = PumpHarness::start_as_a_pty_pump();
    for n in 0..BURST_LINES {
        pump.out_writer
            .write_all(format!("line-{n}\n").as_bytes())
            .await
            .unwrap();
    }

    // This is the reader that keeps up: every `recv` frees the slot the pump
    // is waiting on, so no wait of the pump's runs out.
    for n in 0..BURST_LINES {
        let observed = timeout(PUMP_DEADLINE, pump.logs.recv())
            .await
            .expect("the pump must forward a burst its reader keeps up with")
            .expect("the pump must not end while its streams are open");
        assert_eq!(
            observed,
            LogLine {
                err: false,
                line: format!("line-{n}")
            }
        );
    }
}

/// Fails if a pty pump that has given up on `logs` stays given up: the next
/// line after a drained channel must reach the live bus again.
#[cfg(unix)]
#[tokio::test]
async fn a_pty_pump_forwards_again_once_a_slot_opens() {
    let mut pump = PumpHarness::start_as_a_pty_pump();
    // Two lines past the channel: the first spends [`LIVE_WAIT`] and is then
    // skipped, the second is skipped at once. The file holding both is proof
    // the pump stopped waiting, since a parked pump never reads the second.
    let past = CHANNEL_CAPACITY + 2;
    let mut written = String::new();
    for n in 0..past {
        let line = format!("line-{n}\n");
        pump.out_writer.write_all(line.as_bytes()).await.unwrap();
        written.push_str(&line);
    }
    assert_file_settles(&pump.out_path, &written).await;

    // The skipped lines are gone from `logs`, so exactly its capacity lands.
    for n in 0..CHANNEL_CAPACITY {
        let observed = timeout(PUMP_DEADLINE, pump.logs.recv())
            .await
            .expect("the pump must forward what fits on `logs`")
            .expect("the pump must not end while its streams are open");
        assert_eq!(observed.line, format!("line-{n}"));
    }

    pump.feed(false, "after-a-drain").await;
}

/// Fails if a piped pump loses the backpressure a pipe gives it: it stays
/// parked on a full `logs` channel rather than skipping ahead.
#[cfg(unix)]
#[tokio::test]
async fn a_piped_pump_still_waits_for_room_on_a_full_logs_channel() {
    assert!(
        STILL_WINDOWS * IDLE_FLUSH > LIVE_WAIT,
        "the still window must outlast the wait a piped pump never spends"
    );
    let mut pump = PumpHarness::start();
    let parked = CHANNEL_CAPACITY + 1;
    let mut written = String::new();
    for n in 0..parked {
        let line = format!("line-{n}\n");
        pump.out_writer.write_all(line.as_bytes()).await.unwrap();
        written.push_str(&line);
    }
    assert_file_settles(&pump.out_path, &written).await;

    // The last line is owed to `logs` and nothing is draining it, so the pump
    // is parked on that send: what the sheep writes next stays unread, past
    // every window a pty pump would have spent before skipping.
    pump.out_writer.write_all(b"held-back\n").await.unwrap();
    assert_file_holds_still(&pump.out_path, &written).await;

    written.push_str("held-back\n");
    for n in 0..parked {
        let observed = timeout(PUMP_DEADLINE, pump.logs.recv())
            .await
            .expect("the pump must resume once `logs` has room")
            .expect("the pump must not end while its streams are open");
        assert_eq!(observed.line, format!("line-{n}"));
    }
    let held = timeout(PUMP_DEADLINE, pump.logs.recv())
        .await
        .expect("the line the pump held must reach a drained `logs`")
        .expect("the pump must not end while its streams are open");
    assert_eq!(held.line, "held-back");
    assert_file_settles(&pump.out_path, &written).await;
}

/// Fails if a pump that has ended leaves its control channel reachable.
/// The failed send is what `ProcIo::log_ctl` promises callers as the
/// signal that the pump is already gone, and it is what lets a reopen
/// aimed at a stopped sheep be a no-op rather than an error worth
/// reporting to whoever asked for it.
#[tokio::test]
async fn a_send_fails_once_the_pump_has_ended() {
    let mut pump = PumpHarness::start();
    pump.feed(false, "before-out").await;
    pump.feed(true, "before-err").await;

    // Both writers gone = both streams at EOF, which is what a child
    // exiting looks like from inside the pump.
    drop(pump.out_writer);
    drop(pump.err_writer);

    // The pump's `logs` sender drops with the task, so this `None` is a
    // bounded wait for the task to have finished rather than a guess
    // that it already has.
    let ended = timeout(PUMP_DEADLINE, pump.logs.recv())
        .await
        .expect("the pump must end once both streams reach EOF");
    assert!(ended.is_none(), "a pump that has ended sends nothing more");

    let (done, ack) = oneshot::channel();
    assert!(
        pump.ctl.send(LogCtl::Reopen { done }).await.is_err(),
        "a reopen aimed at an ended pump must fail to send"
    );
    // The rejected request takes its acknowledgement down with it, so a
    // caller that had already started awaiting one is told the same
    // thing rather than left pending.
    assert!(ack.await.is_err());
}

/// No polling: an answered flush means the files hold every line already
/// handed to them, and a `Flush` that dropped a handle would leave the pump
/// writing the stream nowhere from then on.
///
/// Does not catch a `LogFile::flush` that answers without asking its file: on
/// a write this small the content assertion would win that race anyway.
/// [`a_flush_reports_the_write_its_file_never_took`] pins that leg.
#[tokio::test]
async fn a_flush_lands_both_streams_and_keeps_writing_afterwards() {
    let mut pump = PumpHarness::start();
    pump.feed(false, "out-before").await;
    pump.feed(true, "err-before").await;

    pump.flush().await;

    // No polling: the acknowledgement is the barrier this half of `shep
    // flush` exists to provide, and a test that polled for the content
    // would pass against a pump that never provided it.
    assert_eq!(log_text(&pump.out_path), "out-before\n");
    assert_eq!(log_text(&pump.err_path), "err-before\n");

    // A flush keeps the handle; a reopen replaces it. The next line
    // appends to what is already there rather than starting a file.
    pump.feed(false, "out-after").await;
    assert_file_settles(&pump.out_path, "out-before\nout-after\n").await;
}

/// Fails if [`LogFile::flush`] stops asking the file anything: an early
/// `return Ok(())`, or a `map_err` traded for `.ok()`. `tokio::fs::File`
/// reports a failed write on the next operation rather than the one that
/// failed, so a flush that answered without asking would swallow the only
/// signal a sheep's log went unwritten.
///
/// A handle opened for reading alone makes the write fail on every platform.
#[tokio::test]
async fn a_flush_reports_the_write_its_file_never_took() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("out.log");
    fs::write(&path, "").unwrap();
    let file = tokio::fs::File::open(&path).await.unwrap();
    let mut log = buffered_log(&path, file);
    // Swallowed by design: the pump keeps draining a child whose log
    // it cannot write. The failure is owed at the flush instead.
    log.append("never-lands").await;

    let error = log
        .flush()
        .await
        .expect_err("a write that never reached the file must fail the flush");
    assert!(
        error.message.starts_with(&format!("{}: ", path.display())),
        "the failure must name the file it belongs to: {error}"
    );
}

/// `O_APPEND` is a file status flag on the open file description, so it
/// crosses an exec with the descriptor. A handle that lost it writes at its own
/// tracked offset instead, overwriting the first line here, or leaving a sparse
/// hole after a `copytruncate` rotation.
///
/// Reading both lines is what tells the two apart. `cfg(unix)` alongside the
/// constructor it drives: Windows has no `execve`.
#[cfg(unix)]
#[tokio::test]
async fn a_log_file_from_an_open_handle_still_appends() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("out.log");
    fs::write(&path, "first\n").unwrap();

    // Opened exactly as a predecessor's pump had it, and handed over
    // rather than reopened by path.
    let handle = open_append(&path).await.unwrap();
    let mut log = LogFile::from_file(path.clone(), handle, true);

    log.append("second").await;
    log.flush().await.expect("the carried handle must be live");

    assert_eq!(
        log_text(&path),
        "first\nsecond\n",
        "a carried handle must append, not write at its own offset"
    );
}

/// Fails if the pump can only notice a departed `logs` receiver from inside
/// `deliver_line`, that is, if it has no `select!` branch of its own for it.
///
/// A child that forked a lamb holding its pipes open satisfies none of the
/// other ways out: the lamb keeps both streams from EOF, and the supervisor
/// keeps a control sender while the sheep stays registered
/// (`SheepSlot::log_ctl`).
#[tokio::test]
async fn dropping_the_logs_receiver_ends_a_pump_with_no_line_to_carry_the_news() {
    let pump = PumpHarness::start();
    // A barrier proving the pump is up and serving control requests, so
    // the drop below cannot race its initial open.
    pump.reopen().await;

    drop(pump.logs); // the sheep task returning

    // Both writers held (no stream at EOF) and `pump.ctl` alive
    // (control channel open), so dropping `logs` is the only thing
    // that can end this pump. `closed()` resolves when the pump's own
    // `ctl_rx` drops with its task: a bounded wait, not a guess.
    timeout(PUMP_DEADLINE, pump.ctl.closed())
        .await
        .expect("a pump whose `logs` receiver is gone must end");
}

/// Fails if the pump treats a closed control channel as nothing to do:
/// `ProcIo::log_ctl` documents that dropping the sender ends the pump,
/// and an arm that ignored the `None` would spin on it forever, since a
/// closed `mpsc::Receiver` is ready on every poll.
#[tokio::test]
async fn dropping_the_control_sender_ends_the_pump() {
    let mut pump = PumpHarness::start();
    pump.feed(false, "still-here").await;

    drop(pump.ctl);

    let after = timeout(PUMP_DEADLINE, pump.logs.recv())
        .await
        .expect("the pump must end once nothing can control it");
    assert!(after.is_none(), "a pump that has ended sends nothing more");
}

/// Fails if a reader that frees a slot just in time on every line holds a pty
/// pump at its pace. The pump is the only reader its terminal has, and macOS
/// reads a master for about 0.6s after its last slave closes, so a pump that
/// pays a wait per line loses the sheep's tail outright rather than only its
/// live copies.
#[cfg(unix)]
#[tokio::test]
async fn a_pty_pump_does_not_pay_a_wait_per_line_to_a_reader_that_keeps_just_up() {
    const LINES: usize = CHANNEL_CAPACITY + 50;
    /// Just under [`LIVE_WAIT`], so a pump that pays a fresh wait per line
    /// finds a slot for every one of them and never skips a live copy.
    const READER_GAP: Duration = Duration::from_millis(90);
    /// A pump held at `READER_GAP` per line reaches its stream's EOF only
    /// after forwarding all `LINES`, which is 7.4s. A pump that gives up on
    /// live copies reads the whole stream in a fraction of this.
    const PUMP_END_BOUND: Duration = Duration::from_secs(2);

    let mut pump = PumpHarness::start_as_a_pty_pump();
    let mut written = String::new();
    for n in 0..LINES {
        let line = format!("line-{n}\n");
        pump.out_writer.write_all(line.as_bytes()).await.unwrap();
        written.push_str(&line);
    }
    drop(pump.out_writer);
    drop(pump.err_writer);

    // The reader that keeps just up: slower than this pump, but it hands back
    // a slot inside every `LIVE_WAIT`, which is what makes a per-line wait
    // look like progress.
    let seen = Arc::new(AtomicUsize::new(0));
    let counting = Arc::clone(&seen);
    let mut logs = pump.logs;
    let reader = tokio::spawn(async move {
        while logs.recv().await.is_some() {
            counting.fetch_add(1, Ordering::Relaxed);
            tokio::time::sleep(READER_GAP).await;
        }
    });

    // The pump's own end is the measurement, since the pump is what reads the
    // master: it drops its control receiver only once its streams are drained.
    let started = Instant::now();
    let ended = timeout(PUMP_END_BOUND, async {
        while !pump.ctl.is_closed() {
            tokio::time::sleep(Duration::from_millis(5)).await;
        }
    })
    .await;
    let elapsed = started.elapsed();
    let forwarded = seen.load(Ordering::Relaxed);
    reader.abort();

    assert!(
        ended.is_ok(),
        "a pump paying a fresh `LIVE_WAIT` per line read {forwarded} of {LINES} lines in \
         {elapsed:?} at its reader's {READER_GAP:?} pace, and never came back to its master"
    );
    assert_file_settles(&pump.out_path, &written).await;
}