shep-daemon 0.15.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
//! Tests for reopening log files after a rotation.
//!
//! A reopen has to reach the pump a restart spawned rather than the one it
//! replaced, and must not stall the actor when a pump has stopped reading or
//! has already gone.

use super::*;

/// Fails if the actor awaits a reopen's acknowledgement inside its own
/// loop. An actor parked on one stops draining its mailbox, so its sheep
/// tasks block sending into it, so nothing drains their `logs`.
///
/// `list` is the probe: it is answered from the actor loop and nowhere
/// else, and the request must reach the pump first for it to mean anything.
#[tokio::test(start_paused = true)]
async fn the_actor_keeps_answering_while_a_reopen_waits_on_a_silent_pump() {
    let (events, _rx) = crate::bus::test_bus(64);
    let (runner, mut requests) = SilentPumpRunner::new(vec![ProcScript::never_exits()]);
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(runner, test_paths(&dir), events);
    handle
        .start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
        .await
        .unwrap();

    let reopening = tokio::spawn({
        let handle = handle.clone();
        async move { handle.reopen(ProcessSelector::All).await }
    });

    tokio::time::timeout(Duration::from_secs(5), requests.wait_for(|seen| *seen == 1))
        .await
        .expect("the reopen must reach the pump")
        .expect("the runner outlives this wait, so its sender cannot have closed");

    let listed = tokio::time::timeout(Duration::from_secs(5), handle.list())
        .await
        .expect("the actor must keep answering while a reopen is outstanding");
    assert_eq!(listed.len(), 1);
    assert!(
        !reopening.is_finished(),
        "sanity: nothing can acknowledge this reopen, so `list` answering \
         above is not just the reopen having finished first"
    );
    reopening.abort();
}

/// Fails if a reopen skips a matched sheep's pump, reaches a sheep the
/// selector never named, or answers with the wrong set.
///
/// The counts are what make this more than a smoke test: three sheep and a
/// selector naming two of them catches both too narrow and too wide.
#[tokio::test(start_paused = true)]
async fn a_reopen_reaches_every_matched_sheep_and_no_others() {
    let (events, _rx) = crate::bus::test_bus(64);
    // Three scripts for three instances: a fourth spawn would land that
    // sheep `Errored` with no pump, which reads like the skip under test.
    let runner = Arc::new(ScriptedRunner::new(vec![
        ProcScript::never_exits(),
        ProcScript::never_exits(),
        ProcScript::never_exits(),
    ]));
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);

    let mut web = AppConfig::minimal("web", "./srv");
    web.instances = 2;
    handle
        .start(vec![
            normalize(web).unwrap(),
            normalize(AppConfig::minimal("api", "./api")).unwrap(),
        ])
        .await
        .unwrap();

    let reopened = handle
        .reopen(ProcessSelector::Name("web".to_string()))
        .await
        .unwrap();

    assert_eq!(
        reopened.iter().map(|info| info.id).collect::<Vec<_>>(),
        vec![0, 1],
        "the reply must carry both `web` instances, id-sorted, and no `api`"
    );
    assert_eq!(runner.reopens(0), 1, "web's first instance");
    assert_eq!(runner.reopens(1), 1, "web's second instance");
    assert_eq!(runner.reopens(2), 0, "api was never named");
}

/// Fails if a respawn leaves [`SheepSlot::log_ctl`] pointing at the pump of
/// the process it replaced: `slot.log_ctl = Some(log_ctl);` dropped from
/// [`Actor::respawn`]. The send then fails, a failed send is the documented
/// no-op success, and `shep reopen` exits 0 having reached nothing.
#[tokio::test(start_paused = true)]
async fn a_reopen_after_a_restart_reaches_the_pump_the_restart_spawned() {
    let (events, _rx) = crate::bus::test_bus(64);
    let runner = Arc::new(ScriptedRunner::new(vec![
        ProcScript::never_exits(),
        ProcScript::never_exits(),
    ]));
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
    handle
        .start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
        .await
        .unwrap();

    let restarted = handle
        .restart(ProcessSelector::Name("web".to_string()))
        .await
        .unwrap();
    // The premise, stated rather than assumed: a reopen aimed at a sheep
    // that never restarted would reach the first pump and prove nothing.
    assert_eq!(
        (restarted[0].restarts, restarted[0].status),
        (1, ProcStatus::Online),
        "the sheep must be back up on a second process before the reopen"
    );

    let reopened =
        tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
            .await
            .expect("a live pump must answer rather than leave the reopen waiting")
            .expect("a running sheep's reopen must succeed");

    assert_eq!(reopened.len(), 1);
    assert_eq!(
        runner.reopens(1),
        1,
        "the reopen must reach the pump the restart spawned"
    );
    assert_eq!(
        runner.reopens(0),
        0,
        "the pre-restart pump belongs to a process that is gone; a reopen \
         sent there reaches nothing and is reported as a success"
    );
}

/// Fails if a reopen aimed at a sheep whose respawn could not start waits
/// for an acknowledgement nobody is left to send, or answers with a flock
/// other than that sheep.
///
/// The runner has one script, so the restart's spawn is the one that runs
/// out: the sheep is `Errored`, and its entry still holds the `log_ctl` of
/// the pump that served the process which exited.
#[tokio::test(start_paused = true)]
async fn a_reopen_after_a_failed_respawn_is_a_prompt_no_op_success() {
    let (events, _rx) = crate::bus::test_bus(64);
    let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits()]));
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
    handle
        .start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
        .await
        .unwrap();

    let restarted = handle
        .restart(ProcessSelector::Name("web".to_string()))
        .await
        .unwrap();
    // The premise, stated rather than assumed: the first process is gone and
    // the respawn put nothing in its place.
    assert_eq!(
        restarted[0].status,
        ProcStatus::Errored,
        "a restart whose spawn runs out of scripts must land `Errored`: {restarted:?}"
    );

    let reopened =
        tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
            .await
            .expect("a reopen aimed at a pump that is gone must not wait for an acknowledgement")
            .expect("a sheep with no process has nothing to reopen, which is a success");

    assert_eq!(reopened.len(), 1);
    assert_eq!(
        reopened[0].status,
        ProcStatus::Errored,
        "the reply's table row is the sheep the operator asked about: {reopened:?}"
    );
    assert_eq!(
        runner.reopens(0),
        0,
        "the sender this sheep still holds belongs to the pump that served \
         the process that exited, and it answered nothing"
    );
}

/// Fails if a reopen aimed at a sheep waiting out a backoff after a refused
/// spawn waits for an acknowledgement nobody is left to send, or answers
/// with a flock other than that sheep.
///
/// The route to a refused respawn is an operator's own: `set_sheep_env`
/// parks a config the next spawn cannot assemble, and `restart` promotes
/// it. The refusal is the retriable half, so the sheep waits rather than
/// errors, and its entry still holds the `log_ctl` of the pump that served
/// the process which exited.
#[tokio::test(start_paused = true)]
async fn a_reopen_after_a_refused_spawn_is_a_prompt_no_op_success() {
    let (events, _rx) = crate::bus::test_bus(64);
    let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits()]));
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events);
    handle
        .start(vec![normalize(AppConfig::minimal("web", "./srv")).unwrap()])
        .await
        .unwrap();

    handle
        .set_sheep_env(
            "web".to_string(),
            "PW".to_string(),
            Some(EnvValue::from("{{secret:vault/K}}".to_string())),
        )
        .await
        .unwrap()
        .expect("web is registered");

    let restarted = handle
        .restart(ProcessSelector::Name("web".to_string()))
        .await
        .unwrap();
    assert_eq!(
        restarted[0].status,
        ProcStatus::WaitingRestart,
        "a namespace no provider dog has pushed to is a retriable refusal: {restarted:?}"
    );
    assert_eq!(
        runner.spawn_count(),
        1,
        "the refusal is decided while the parked config is assembled, so no \
         second process ever reached the runner"
    );

    let reopened =
        tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
            .await
            .expect("a reopen aimed at a pump that is gone must not wait for an acknowledgement")
            .expect("a sheep with no process has nothing to reopen, which is a success");

    assert_eq!(reopened.len(), 1);
    assert_eq!(
        reopened[0].status,
        ProcStatus::WaitingRestart,
        "the reply's table row is the sheep the operator asked about: {reopened:?}"
    );
    assert_eq!(
        runner.reopens(0),
        0,
        "the sender this sheep still holds belongs to the pump that served \
         the process that exited, and it answered nothing"
    );
}

/// Fails if a selector that matches nothing is answered as a success.
/// `reopen` is the one selector verb with a default, so silence would look
/// like a rotation that worked.
#[tokio::test(start_paused = true)]
async fn a_reopen_matching_nothing_is_not_found() {
    let (events, _rx) = crate::bus::test_bus(64);
    let runner = ScriptedRunner::new(vec![]);
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(runner, test_paths(&dir), events);

    assert_eq!(
        handle.reopen(ProcessSelector::All).await,
        Err(SupervisorError::NotFound)
    );
}

/// Fails if a stopped sheep makes a reopen error out, or hang waiting for
/// an acknowledgement that cannot come. The fake's control task ends with
/// its proc, so this sheep's pump is gone by the time the reopen is issued.
#[tokio::test(start_paused = true)]
async fn a_stopped_sheep_is_a_no_op_success() {
    let (events, _rx) = crate::bus::test_bus(64);
    let runner = ScriptedRunner::new(vec![ProcScript::never_exits()]);
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(runner, test_paths(&dir), events);
    let mut app = AppConfig::minimal("web", "./srv");
    app.autorestart = false;
    handle.start(vec![normalize(app).unwrap()]).await.unwrap();
    handle
        .stop(ProcessSelector::Name("web".to_string()))
        .await
        .unwrap();
    assert_eq!(handle.list().await[0].status, ProcStatus::Stopped);

    let reopened =
        tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
            .await
            .expect("a reopen aimed at a stopped sheep must not wait for an acknowledgement")
            .expect("a stopped sheep has nothing to reopen, which is a success");
    assert_eq!(reopened.len(), 1);
    assert_eq!(reopened[0].status, ProcStatus::Stopped);
}

/// Fails if [`reopen_logs`] waits on an acknowledgement nobody is left to
/// send: the leg where the pump is gone and the send itself fails.
#[tokio::test(start_paused = true)]
async fn a_reopen_whose_pump_is_already_gone_returns_at_once() {
    let (tx, rx) = mpsc::channel(1);
    drop(rx); // the pump ended before the request was made

    let outcome = tokio::time::timeout(Duration::from_secs(5), reopen_logs(&tx))
        .await
        .expect("a failed send must end the reopen, not leave it waiting");
    assert_eq!(
        outcome,
        Ok(()),
        "a pump that was never reached reopened nothing, which is a no-op \
         success rather than a reopen that failed"
    );
}

/// Fails if [`reopen_logs`] keeps waiting on a dropped acknowledgement: the
/// leg where the pump ends between accepting the request and answering it.
#[tokio::test(start_paused = true)]
async fn a_pump_that_ends_mid_request_still_ends_the_reopen() {
    let (tx, mut rx) = mpsc::channel(1);
    tokio::spawn(async move {
        let request = rx.recv().await.expect("the reopen must reach the pump");
        drop(request); // ends without answering, exactly as a closing pump does
    });

    let outcome = tokio::time::timeout(Duration::from_secs(5), reopen_logs(&tx))
        .await
        .expect("a dropped acknowledgement must end the reopen, not leave it waiting");
    assert_eq!(
        outcome,
        Ok(()),
        "a pump that ended mid-request reopened nothing, which is the same \
         no-op success a failed send is"
    );
}

/// The flush half of
/// [`a_pump_that_ends_mid_request_still_ends_the_reopen`]. A pump that
/// ended mid-request owes no bytes, but the truncate still has to run.
#[tokio::test(start_paused = true)]
async fn a_pump_that_ends_mid_request_still_ends_the_flush() {
    let (tx, mut rx) = mpsc::channel(1);
    tokio::spawn(async move {
        let request = rx.recv().await.expect("the flush must reach the pump");
        drop(request); // ends without answering, exactly as a closing pump does
    });

    let outcome = tokio::time::timeout(Duration::from_secs(5), flush_logs(&tx))
        .await
        .expect("a dropped acknowledgement must end the flush, not leave it waiting");
    assert_eq!(
        outcome,
        Ok(()),
        "a pump that ended mid-request owes no bytes, which is the same \
         no-op success a failed send is"
    );
}

/// Fails if a pump that could not reopen its files is reported as a
/// success. That sheep writes a stream nowhere while `shep reopen` exits 0.
/// The healthy sheep is the second half: a failure must name its own sheep
/// and must not stop the rest of the flock being reopened.
#[tokio::test(start_paused = true)]
async fn a_pump_that_could_not_reopen_fails_the_request_and_names_its_sheep() {
    let (events, _rx) = crate::bus::test_bus(64);
    // Two scripts for two instances: a third spawn would land that sheep
    // `Errored` with no pump at all.
    let scripted = Arc::new(ScriptedRunner::new(vec![
        ProcScript::never_exits(),
        ProcScript::never_exits(),
    ]));
    let dir = tempfile::tempdir().unwrap();
    let handle = spawn_supervisor(
        FailingPumpRunner::new(Arc::clone(&scripted)),
        test_paths(&dir),
        events,
    );
    handle
        .start(vec![
            normalize(AppConfig::minimal("web", "./srv")).unwrap(),
            normalize(AppConfig::minimal("api", "./api")).unwrap(),
        ])
        .await
        .unwrap();

    let error = tokio::time::timeout(Duration::from_secs(5), handle.reopen(ProcessSelector::All))
        .await
        .expect("a pump that answers must not leave the reopen waiting")
        .expect_err("a reopen a pump could not carry out must not answer Ok");

    assert_eq!(
        error,
        SupervisorError::ReopenFailed(format!("web (id 0): could not reopen {PUMP_REFUSAL}")),
        "the failure must carry the sheep and the path, and only the sheep that failed"
    );
    assert_eq!(
        scripted.reopens(1),
        1,
        "the healthy sheep must still have been reopened"
    );
}