shep 0.16.0

The shep binary: a process manager that keeps a flock of long-running processes alive on macOS, Linux and Windows, with logs, watch and cron restarts, and webhook alerts
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
//! The host-usage strip: one line, several self-labelled segments.
//!
//! Half is read from this machine ([`super::super::source::HostSample`])
//! and half is summed from [`super::super::app::App::all_rows`], the whole
//! flock, never the name-filtered set [`super::super::app::App::rows_len`]
//! counts: a name filter must not narrow what this strip claims. Every segment names its
//! half, so a truncated strip never leaves a bare `mem 12.4G` beside a bare
//! `mem 706.0M`.
//!
//! Segments join in the drop order (`up` last, load average first) and are
//! truncated from the right, so truncation is the drop order with no second
//! mechanism.

use ratatui::style::Style;
use ratatui::text::{Line, Span};
use shep_core::status::ProcStatus;

use super::super::app::App;
use super::super::theme::Palette;
use super::cell;
use super::flock::take_columns;
use crate::output::width::char_columns;
use crate::output::{human_bytes, human_duration};

/// The strip, fitted to `width`.
#[must_use]
pub fn strip_line(app: &App, width: u16) -> Line<'static> {
    if app.remote_host().is_some() {
        return Line::from(Span::raw(" ".repeat(usize::from(width))));
    }
    Line::from(Ink::fit(&runs(app), width, app.data_palette()))
}

// One run of the strip and the role it renders in.
struct Run {
    text: String,
    ink: Ink,
}

// Which colour a run wears. Most of the line is words and numbers with no
// status behind them, so it stays muted. The load and memory gauges are the
// exception: two bars on one line are unreadable without something to tell
// them apart by, which is meaning rather than the plain decoration the rest
// of the line has none of.
#[derive(Clone, Copy)]
enum Ink {
    Muted,
    Load,
    Mem,
}

impl Ink {
    fn style(self, palette: Palette) -> Style {
        match self {
            Ink::Muted => palette.muted(),
            Ink::Load => palette.attention(),
            Ink::Mem => palette.sky(),
        }
    }

    /// `runs` truncated to exactly `width` columns, one `Span` per surviving
    /// run plus the tail. Calls [`super::flock::take_columns`] once per run
    /// with the budget its predecessors left. A run whose first character
    /// does not fit contributes nothing: later runs still get what remains.
    fn fit(runs: &[Run], width: u16, palette: Palette) -> Vec<Span<'static>> {
        let width = usize::from(width);
        let total: usize = runs
            .iter()
            .flat_map(|run| run.text.chars())
            .map(char_columns)
            .sum();
        if total <= width {
            let mut spans: Vec<Span<'static>> = runs
                .iter()
                .map(|run| Span::styled(run.text.clone(), run.ink.style(palette)))
                .collect();
            spans.push(Span::styled(" ".repeat(width - total), palette.muted()));
            return spans;
        }
        if width == 0 {
            return vec![Span::raw(String::new())];
        }
        let budget = width - 1;
        let mut spans = Vec::new();
        let mut used = 0;
        for run in runs {
            if used >= budget {
                break;
            }
            let (kept, taken) = take_columns(&run.text, budget - used);
            used += taken;
            if !kept.is_empty() {
                spans.push(Span::styled(kept, run.ink.style(palette)));
            }
        }
        spans.push(Span::styled("…", palette.muted()));
        if budget > used {
            spans.push(Span::styled(" ".repeat(budget - used), palette.muted()));
        }
        spans
    }
}

// The load gauge: ten cells, filled by the one-minute average as a
// percentage of `cores`. `None`, and a reported zero cores, draw an empty
// gauge rather than a division by zero: a platform that cannot say how many
// cores it has cannot say what fraction of them is busy either.
fn load_gauge(load: f64, cores: Option<usize>) -> String {
    match cores.filter(|&count| count > 0) {
        Some(cores) => {
            let percent = (load / cores as f64 * 100.0).round() as u64;
            cell::gauge(percent, Some(100), 10)
        }
        None => cell::gauge(0, None, 10),
    }
}

// `N errored · N parked`, summed over the whole flock. Errored filters
// `ProcStatus::Errored`; parked counts a non-empty `pending`, the same
// field `crate::output::cfg_cell` reads for the `CFG` column.
fn summary(app: &App) -> String {
    let rows = app.all_rows();
    let errored = rows
        .iter()
        .filter(|row| row.info.status == ProcStatus::Errored)
        .count();
    let parked = rows
        .iter()
        .filter(|row| {
            row.info
                .pending
                .as_deref()
                .is_some_and(|fields| !fields.is_empty())
        })
        .count();
    format!("{errored} errored · {parked} parked")
}

// The runs, widest set first.
fn runs(app: &App) -> Vec<Run> {
    let mut out = Vec::with_capacity(9);
    let sep = || Run {
        text: "   ".to_string(),
        ink: Ink::Muted,
    };
    match app.host() {
        Some(host) => {
            let (one, five, fifteen) = host.load;
            // Gauge before numbers, per the design: a reader scanning for
            // "is this loaded" anchors on a fixed offset only if the gauge
            // sits right after its label, not after numbers of varying width.
            out.push(Run {
                text: "host  load  ".to_string(),
                ink: Ink::Muted,
            });
            out.push(Run {
                text: load_gauge(one, host.cores),
                ink: Ink::Load,
            });
            let load_suffix = match host.cores {
                Some(cores) => format!(" {one:.2} {five:.2} {fifteen:.2} / {cores} cores  "),
                // No denominator: the numbers alone are not readable, so they
                // are shown without a claim about how many cores they are
                // spread over rather than with a guessed one.
                None => format!(" {one:.2} {five:.2} {fifteen:.2}  "),
            };
            out.push(Run {
                text: load_suffix,
                ink: Ink::Muted,
            });
            out.push(sep());
            out.push(Run {
                text: "host mem  ".to_string(),
                ink: Ink::Muted,
            });
            out.push(Run {
                text: cell::gauge(host.memory_used_bytes, Some(host.memory_total_bytes), 10),
                ink: Ink::Mem,
            });
            out.push(Run {
                text: format!(
                    " {} / {}  ",
                    human_bytes(host.memory_used_bytes),
                    human_bytes(host.memory_total_bytes)
                ),
                ink: Ink::Muted,
            });
            // The two host readings sit together, then the flock's own
            // numbers: the line reads left to right as machine, then flock,
            // rather than interleaving them. The cost is that the summary
            // is no longer the segment most likely to survive a narrow
            // terminal; the test named for it pins where it now falls off.
            out.push(sep());
            out.push(Run {
                text: summary(app),
                ink: Ink::Muted,
            });
        }
        None if app.host_unsupported() => {
            out.push(Run {
                text: "host  usage is not available on this platform".to_string(),
                ink: Ink::Muted,
            });
            out.push(sep());
            out.push(Run {
                text: summary(app),
                ink: Ink::Muted,
            });
        }
        // Reachable for at most one redraw: `tokio::time::interval`'s first
        // tick is immediate, so the heartbeat samples before the second
        // frame.
        None => {
            out.push(Run {
                text: "host  not read yet".to_string(),
                ink: Ink::Muted,
            });
            out.push(sep());
            out.push(Run {
                text: summary(app),
                ink: Ink::Muted,
            });
        }
    }

    // Summed from the whole flock, `all_rows`, not the filtered `rows`, so
    // `-` here always means no reading, never "the filter matched nothing".
    //
    // `App::cpu_now`, the same per-sheep source `App::flock_cpu_history`
    // sums into its own newest sample, not `cpu_percent`: that keeps this
    // figure and the sparkline beside it answering the same question. Not
    // read from `flock_cpu_history` directly, though: that buffer sums a
    // zero for a sheep with no reading (so its window does not slide), and
    // reading its sum back as `0.0%` when literally nothing in the flock
    // has a reading yet would claim a measurement nobody made.
    let rows = app.all_rows();
    let cpu: Option<f32> = rows
        .iter()
        .filter_map(|row| app.cpu_now(row.info.id))
        .fold(None, |sum, value| Some(sum.unwrap_or(0.0) + value));
    let mem: Option<u64> = rows
        .iter()
        .filter_map(|row| row.info.memory_bytes)
        .fold(None, |sum, value| Some(sum.unwrap_or(0) + value));
    let cpu_text = cpu.map_or_else(
        || "flock cpu -".to_string(),
        |cpu| format!("flock cpu {cpu:.1}%"),
    );
    out.push(sep());
    out.push(Run {
        text: format!(
            "{cpu_text} {}",
            // The flock series is a SUM across sheep, so it does not share
            // the per-row ceiling: its own window peak is the only honest
            // denominator for a line whose scale is the whole flock.
            cell::sparkline(app.flock_cpu_history(), 8, flock_cpu_ceiling(app)),
        ),
        ink: Ink::Muted,
    });
    out.push(sep());
    out.push(Run {
        text: mem.map_or_else(
            || "flock mem -".to_string(),
            |mem| format!("flock mem {}", human_bytes(mem)),
        ),
        ink: Ink::Muted,
    });
    // Last, and therefore the first thing a narrow terminal loses: a host that
    // has been up six days explains nothing about right now.
    if let Some(host) = app.host() {
        out.push(sep());
        out.push(Run {
            text: format!("up {}", human_duration(host.uptime_seconds * 1_000)),
            ink: Ink::Muted,
        });
    }
    out
}

/// The ceiling the flock-wide CPU sparkline scales against.
///
/// Its own window's peak, floored the same way [`App::cpu_ceiling`] floors
/// the per-row one. The flock line plots a sum rather than one sheep, so it
/// has no reason to share the table's ceiling and every reason not to: a
/// single busy sheep would flatten the whole-flock line against it.
fn flock_cpu_ceiling(app: &App) -> f32 {
    app.flock_cpu_history().iter().copied().fold(2.0, f32::max)
}

#[cfg(test)]
mod tests {
    use super::super::MIN_TERM_WIDTH;
    use super::super::fixtures::{
        app_with, flock_of, plain, rendered, sample, with_host, with_host_none,
    };
    use super::*;
    use shep_core::protocol::ProcessInfo;
    use shep_core::status::ProcStatus;
    use std::ffi::OsStr;

    /// `fixture_with` in the brief's own words: an app over one flock and no
    /// host reading, since these tests exercise `summary` alone.
    fn fixture_with(flock: Vec<ProcessInfo>) -> App {
        app_with(flock, plain())
    }

    #[test]
    fn the_load_gauge_measures_against_the_core_count() {
        // 5.12 of 14 cores is 37%, four of ten cells.
        assert_eq!(load_gauge(5.12, Some(14)), "████░░░░░░");
    }

    #[test]
    fn a_host_that_cannot_report_cores_draws_no_load_gauge() {
        assert_eq!(load_gauge(5.12, None), "░░░░░░░░░░");
    }

    #[test]
    fn the_summary_counts_errored_and_parked_sheep() {
        let app = fixture_with(vec![
            ProcessInfo::builder(1, "flaky", ProcStatus::Errored).build(),
            ProcessInfo::builder(2, "catcher", ProcStatus::Online)
                .pending(Some(vec!["max_memory".to_string(), "err_file".to_string()]))
                .build(),
        ]);
        assert_eq!(summary(&app), "1 errored · 1 parked");
    }

    /// A truncated strip must never leave a bare `mem 12.4G` beside a bare
    /// `mem 706.0M`, one the host's and one the flock's.
    #[test]
    fn every_segment_names_whose_number_it_is() {
        let app = with_host(sample(), flock_of(4, 1));
        let line = rendered(&strip_line(&app, 200));
        for segment in ["host  load", "host mem", "flock cpu", "flock mem", "up "] {
            assert!(line.contains(segment), "missing {segment:?} in {line:?}");
        }
    }

    /// There is no drop loop and no width table: `Ink::fit` truncates from
    /// the right, so truncating is the drop order.
    #[test]
    fn a_narrow_strip_truncates_visibly_and_keeps_the_load_average() {
        let app = with_host(sample(), flock_of(4, 1));

        let narrow = rendered(&strip_line(&app, 40));
        assert!(narrow.starts_with("host  load"), "got {narrow:?}");
        assert!(
            narrow.ends_with('…'),
            "a truncation the operator can see: {narrow:?}"
        );
        assert!(
            !narrow.contains("up "),
            "`up` is the first thing off the end"
        );

        // At the floor the strip still says whose number it is quoting, which
        // is the reason every segment carries its own label.
        let floor = rendered(&strip_line(&app, MIN_TERM_WIDTH));
        assert!(floor.starts_with("host  load"), "got {floor:?}");

        // And where it fits, nothing is cut.
        let full = rendered(&strip_line(&app, 200));
        assert!(!full.contains('…'));
        assert!(
            full.contains("up 6d"),
            "the last segment is there: {full:?}"
        );
    }

    /// `Ink::fit` cuts each run against the budget the runs before it left,
    /// not the whole line as one string: a run too narrow for its own first
    /// character contributes nothing and a later run still draws.
    #[test]
    fn a_run_too_narrow_for_its_first_character_contributes_nothing() {
        let narrow_runs = [
            Run {
                text: "ab".to_owned(),
                ink: Ink::Muted,
            },
            Run {
                text: "羊".to_owned(),
                ink: Ink::Muted,
            },
            Run {
                text: "cd".to_owned(),
                ink: Ink::Muted,
            },
        ];
        // Budget 3: `ab` spends two columns, `羊` needs two of the one left,
        // and `c` takes the last.
        let line = Line::from(Ink::fit(
            &narrow_runs,
            4,
            Palette::detect(Some(OsStr::new("1")), None, None),
        ));
        assert_eq!(rendered(&line), "abc…");

        // The strip's own runs never hit that case: each starts one column
        // wide, so a run goes off the end only once the budget is spent.
        let built = runs(&with_host(sample(), flock_of(4, 1)));
        assert!(built.len() > 1, "an empty walk would pass this vacuously");
        for run in &built {
            let first = run.text.chars().next().expect("every run has text");
            assert_eq!(char_columns(first), 1, "{:?} starts a run", run.text);
        }
    }

    /// `ProcessInfo`'s `None` covers three cases (not running, under one
    /// sampling window, or a shepherd predating the field), and a reader
    /// renders all three as unknown, never as zero.
    #[test]
    fn a_flock_with_no_readings_shows_a_dash_and_not_a_zero() {
        let app = with_host(
            sample(),
            vec![ProcessInfo::builder(1, "web", ProcStatus::Errored).build()],
        );
        let line = rendered(&strip_line(&app, 200));
        assert!(line.contains("flock cpu -"), "got {line:?}");
        assert!(line.contains("flock mem -"), "got {line:?}");
        assert!(!line.contains("0.0%"));
    }

    /// A silently dropped host half would look identical to numbers that
    /// have not arrived yet.
    #[test]
    fn an_unread_host_says_which_of_the_two_reasons_it_is() {
        let unsupported = with_host_none(flock_of(4, 1), true);
        assert!(
            rendered(&strip_line(&unsupported, 200))
                .contains("host  usage is not available on this platform")
        );

        let not_yet = with_host_none(flock_of(4, 1), false);
        assert!(rendered(&strip_line(&not_yet, 200)).contains("host  not read yet"));

        // Both keep the flock half, which lookout can always compute.
        assert!(rendered(&strip_line(&unsupported, 200)).contains("flock cpu"));
    }

    /// A filter matching nothing would otherwise make a running flock's
    /// strip print `-`, the same cell reserved for "no reading arrived yet".
    ///
    /// `with_host`/`flock_of` fire one poll, which never has anything
    /// differenced to sum, so this test drives two by hand: a baseline
    /// `cpu_ms` reading, then one two seconds later that has something to
    /// difference.
    #[test]
    fn the_flock_totals_ignore_the_filter() {
        use std::time::{Duration, Instant};

        use super::super::super::app::{Control, Msg};

        let row = |cpu_ms: u64| {
            ProcessInfo::builder(1, "web", ProcStatus::Online)
                .cpu_ms(Some(cpu_ms))
                .build()
        };
        let t0 = Instant::now();
        let mut app = App::new(
            plain(),
            Control::ReadOnly,
            "/home/ada/.shep".to_string(),
            t0,
        );
        app.update(Msg::Snapshot {
            rows: vec![row(0)],
            at: t0,
        });
        let t1 = t0 + Duration::from_secs(2);
        app.update(Msg::Snapshot {
            rows: vec![row(2_000)],
            at: t1,
        });
        app.update(Msg::Host {
            sample: Some(sample()),
        });

        let full = rendered(&strip_line(&app, 200));
        assert!(full.contains("flock cpu 100.0%"), "sanity: got {full:?}");

        // A filter matching nothing empties the table (`rows()`) without
        // touching the flock itself (`all_rows()`, what the strip reads).
        app.set_filter_for_tests("zzz");
        assert!(app.rows().is_empty(), "sanity: the filter matched nothing");
        let filtered = rendered(&strip_line(&app, 200));

        assert_eq!(
            full, filtered,
            "the strip must not change when the table's filter does: {filtered:?}"
        );
        assert!(
            !filtered.contains("flock cpu -"),
            "a filter matching nothing is not the same as no reading arriving: {filtered:?}"
        );
    }

    /// The flock cpu figure is a sum of `App::cpu_now` across the same
    /// sheep `App::flock_cpu_history`'s own sum walks, so the text figure
    /// and the sparkline's newest cell must agree, one sheep unsampled and
    /// all.
    #[test]
    fn the_flock_figure_sums_the_same_sheep_the_flock_series_does() {
        use std::time::{Duration, Instant};

        use super::super::super::app::{Control, Msg};

        let rows = |cpu_ms: [Option<u64>; 3]| {
            vec![
                ProcessInfo::builder(1, "web", ProcStatus::Online)
                    .cpu_ms(cpu_ms[0])
                    .build(),
                ProcessInfo::builder(2, "api", ProcStatus::Online)
                    .cpu_ms(cpu_ms[1])
                    .build(),
                // Never samples at all: an unsampled sheep among two live
                // ones is the case that most easily lets a figure and a
                // sparkline disagree.
                ProcessInfo::builder(3, "cron", ProcStatus::Stopped)
                    .cpu_ms(cpu_ms[2])
                    .build(),
            ]
        };
        let t0 = Instant::now();
        let mut app = App::new(
            plain(),
            Control::ReadOnly,
            "/home/ada/.shep".to_string(),
            t0,
        );
        app.update(Msg::Snapshot {
            rows: rows([Some(0), Some(0), None]),
            at: t0,
        });
        let t1 = t0 + Duration::from_secs(2);
        app.update(Msg::Snapshot {
            rows: rows([Some(2_000), Some(4_000), None]),
            at: t1,
        });

        let line = rendered(&strip_line(&app, 200));
        // 100.0% (web) + 200.0% (api) + nothing from cron, the same sum
        // `App::flock_cpu_history`'s own newest entry holds.
        assert_eq!(app.flock_cpu_history().last().copied(), Some(300.0));
        assert!(
            line.contains("flock cpu 300.0%"),
            "the figure must sum the same sheep the series does: {line:?}"
        );
    }

    /// The strip reads machine, then flock: load and host memory together,
    /// then `N errored · N parked` and the flock's own numbers.
    ///
    /// The summary used to sit second, ahead of host memory, so that it was
    /// the segment most likely to survive a cut. Grouping the two host
    /// readings costs it that, and how much it costs is not a property of
    /// this code: the strip's length depends on the machine's own numbers,
    /// so a host with fourteen cores and a 48G total pushes the summary
    /// right by a dozen columns against one with ten and 32G. Only the order
    /// is invariant, so only the order is asserted here.
    #[test]
    fn host_memory_comes_before_the_flock_summary() {
        let app = with_host(sample(), flock_of(4, 1));
        let line = rendered(&strip_line(&app, 200));

        assert!(
            line.contains("0 errored · 0 parked"),
            "summary dropped even at 200 columns: {line:?}"
        );
        assert!(
            line.find("host mem").unwrap() < line.find("errored").unwrap(),
            "the line reads machine, then flock: {line:?}"
        );
        assert!(
            line.find("errored").unwrap() < line.find("flock cpu").unwrap(),
            "the summary still leads the flock's own numbers: {line:?}"
        );
    }
}