shep-daemon 0.25.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
//! Staged, dependency-ordered restart and reload.
//!
//! [`ordered_walk`] groups a selector's matches into stages by
//! dependency depth; [`restart_in_stages`] and [`reload_in_stages`]
//! run each stage in turn and collect every refusal rather than
//! stopping at the first.

use core::time::Duration;

use std::collections::{BTreeMap, BTreeSet};

use shep_core::protocol::{ProcessInfo, Reply, Response, RpcError, SelectorSpec, SheepRefusal};
use shep_core::selector::ProcessSelector;
use shep_core::status::ProcStatus;

use crate::supervisor::SupervisorError;

use super::context::{Outcome, RpcContext};
use super::error::rpc_error;
use super::selector_verbs::selector_of;

/// A restart or reload the selector matched more than one sheep for, grouped
/// into the stages it walks.
pub(super) struct OrderedWalk {
    /// The matched names, dependencies first, one `Vec` per stage.
    pub(super) stages: Vec<Vec<String>>,
    /// The matched names another matched name waits for.
    ///
    /// The only ones a stage has to be held for: a sheep nothing in this
    /// request depends on can settle after the walk has moved on, and waiting
    /// for it would let one slow app that nobody waits for cost every later
    /// stage its bound. The same gate `start_in_stages` builds.
    pub(super) depended_on: BTreeSet<String>,
}

/// How `selector` is walked, or `None` when it names at most one sheep and
/// there is nothing to order.
///
/// A single-target `shep restart web` therefore goes out as the one
/// supervisor call it has always been, deadlines and refusal and all.
///
/// The match is computed the way `Actor::matching_ids` computes it, dogs and all.
pub(super) fn ordered_walk(
    ctx: &RpcContext,
    selector: &ProcessSelector,
    flock: &[ProcessInfo],
) -> Option<OrderedWalk> {
    let exact = selector.is_exact();
    let matched: BTreeSet<String> = flock
        .iter()
        .filter(|info| exact || info.dog.is_none())
        // A sweep passes a disabled app by. Walked, it would reach the
        // supervisor as an exact name, which refuses it.
        .filter(|info| exact || info.disabled != Some(true))
        .filter(|info| selector.matches(&info.name, info.id, info.fold.as_deref(), info.instance))
        .map(|info| info.name.clone())
        .collect();
    if matched.len() < 2 {
        return None;
    }

    let edges = ctx.registry.depends_on_by_name();
    let plan = crate::boot_order::plan_for_names(&edges);
    let mut stages: Vec<Vec<String>> = plan
        .stages
        .iter()
        .map(|stage| {
            stage
                .iter()
                .filter(|name| matched.contains(name.as_str()))
                .cloned()
                .collect::<Vec<String>>()
        })
        .filter(|stage| !stage.is_empty())
        .collect();
    // A matched name the registry does not hold has no node in the plan, and
    // dropping it here would leave a sheep the operator named unrestarted
    // while the reply says otherwise. It goes last, since nothing here can
    // say what waits for it.
    let placed: BTreeSet<&str> = stages.iter().flatten().map(String::as_str).collect();
    let unplaced: Vec<String> = matched
        .iter()
        .filter(|name| !placed.contains(name.as_str()))
        .cloned()
        .collect();
    if !unplaced.is_empty() {
        stages.push(unplaced);
    }

    let depended_on = matched
        .iter()
        .flat_map(|name| matched_dependencies(&edges, name, &matched))
        .collect();
    Some(OrderedWalk {
        stages,
        depended_on,
    })
}

/// Every name in `matched` that `start` waits for, however many hops away.
///
/// The transitive walk is the point. A direct-edge intersection loses a
/// dependency whose intermediate the selector missed: with `web -> mid ->
/// db` matched at the ends only, `web` reads as waiting for nothing and its
/// stage restarts against a `db` no stage held. `start` itself is never
/// returned, so a name in a knot does not wait on its own stage.
fn matched_dependencies(
    edges: &BTreeMap<String, Vec<String>>,
    start: &str,
    matched: &BTreeSet<String>,
) -> BTreeSet<String> {
    let mut found = BTreeSet::new();
    let mut seen: BTreeSet<&str> = [start].into_iter().collect();
    let mut frontier: Vec<&str> = vec![start];
    while let Some(name) = frontier.pop() {
        for dependency in edges.get(name).into_iter().flatten() {
            if !seen.insert(dependency.as_str()) {
                continue;
            }
            if matched.contains(dependency.as_str()) {
                found.insert(dependency.clone());
            }
            frontier.push(dependency.as_str());
        }
    }
    found
}

/// How long ONE INSTANCE gets to settle, before the stage's slack.
///
/// A restart waits out `listen_timeout`; a reload waits out
/// `graceful_timeout` as well, which is the pair its own swap is already
/// bounded by (`Actor::arm_reload_deadline`).
///
/// A name the registry does not hold falls back to nothing at all.
fn settle_bound(ctx: &RpcContext, name: &str, reloading: bool) -> Duration {
    let (readiness, drain) = ctx.registry.timeouts_of(name).unwrap_or_default();
    let drain = if reloading { drain } else { Duration::ZERO };
    readiness + drain
}

/// The longest bound any of a restart stage's members asks for.
///
/// One instance's worth, unlike [`reload_stage_bound`]: a restart's
/// instances go down and come back together, so the stage costs the
/// slowest of them rather than the sum.
fn stage_bound(ctx: &RpcContext, stage: &[String]) -> Duration {
    stage
        .iter()
        .map(|name| settle_bound(ctx, name, false))
        .max()
        .unwrap_or_default()
        + crate::boot_order::STAGE_SLACK
}

/// [`stage_bound`] for a reload, sized by the swaps each member still owes.
///
/// `advance_reload` replaces one instance at a time, so a three-instance app
/// costs three drains and three readiness waits, not one of each. A per-app
/// bound is under a third of that at the defaults, so the stage times out
/// and lets a dependant reload against a dependency still half swapped. The
/// counts come from `waiting`, so this bound can only be short, never long.
pub(super) fn reload_stage_bound(ctx: &RpcContext, waiting: &BTreeMap<String, usize>) -> Duration {
    waiting
        .iter()
        .map(|(name, swaps)| {
            settle_bound(ctx, name, true).saturating_mul(u32::try_from(*swaps).unwrap_or(u32::MAX))
        })
        .max()
        .unwrap_or_default()
        + crate::boot_order::STAGE_SLACK
}

/// `Restart`'s arm: ordered when the selector matches several sheep, the
/// plain supervisor call when it matches one.
///
/// The refused half of the reply is a walk's alone: the supervisor refuses a
/// selector matching one app whole, so that arm answers `Err` and has
/// nothing to name.
pub(super) async fn restart_request(id: u64, spec: SelectorSpec, ctx: &RpcContext) -> Outcome {
    let result = match selector_of(spec) {
        Err(err) => Err(err),
        Ok(selector) => match walk_for(ctx, &selector).await {
            None => ctx
                .supervisor
                .restart(selector)
                .await
                .map(|accepted| Response::Restarted {
                    accepted,
                    refused: Vec::new(),
                })
                .map_err(|err| rpc_error(&err)),
            Some(walk) => restart_in_stages(ctx, &walk)
                .await
                .map(|(accepted, refused)| Response::Restarted { accepted, refused }),
        },
    };
    Outcome::Reply(Reply { id, result })
}

/// `Reload`'s arm, mirroring [`restart_request`] down to the refused half of
/// its reply.
pub(super) async fn reload_request(id: u64, spec: SelectorSpec, ctx: &RpcContext) -> Outcome {
    let result = match selector_of(spec) {
        Err(err) => Err(err),
        Ok(selector) => match walk_for(ctx, &selector).await {
            None => ctx
                .supervisor
                .reload(selector)
                .await
                .map(|accepted| Response::Reloading {
                    accepted,
                    refused: Vec::new(),
                })
                .map_err(|err| rpc_error(&err)),
            Some(walk) => reload_in_stages(ctx, &walk)
                .await
                .map(|(accepted, refused)| Response::Reloading { accepted, refused }),
        },
    };
    Outcome::Reply(Reply { id, result })
}

/// [`ordered_walk`] over a fresh listing, or `None` when there is nothing to
/// order, which includes an actor that could not answer: the supervisor call
/// the caller falls back to reports that failure itself, in its own words.
pub(super) async fn walk_for(ctx: &RpcContext, selector: &ProcessSelector) -> Option<OrderedWalk> {
    let flock = ctx.supervisor.list_checked().await.ok()?;
    ordered_walk(ctx, selector, &flock)
}

/// Restarts each stage's members at once, then holds the walk until the ones
/// a later stage waits for are back.
///
/// A member that fails is warned about and the walk continues, the rule
/// `start_in_stages` takes under `BatchPolicy::PerApp` and for its reason.
///
/// Every member that was refused is named in the second half of the answer:
/// a walk that restarted something still answers `Ok`, and exit 0 with a
/// row silently missing is what carrying the names replaces.
///
/// # Errors
/// - The first member's refusal, when no member restarted at all.
pub(super) async fn restart_in_stages(
    ctx: &RpcContext,
    walk: &OrderedWalk,
) -> Result<(Vec<ProcessInfo>, Vec<SheepRefusal>), RpcError> {
    let mut restarted = Vec::new();
    let mut refused: Vec<SheepRefusal> = Vec::new();
    let mut refusal = None;
    for stage in &walk.stages {
        // Subscribed before the calls for the reason `start_in_stages` gives:
        // a receiver taken afterwards starts past a fast sheep's `Online`.
        let rx = ctx.events.subscribe();
        // Concurrently inside a stage, serially across them. Awaiting members
        // in turn would make the walk cost the SUM of their kill ladders and
        // readiness deadlines where an unordered restart costs the longest
        // one; `stop_in_reverse` carries the same note.
        let outcomes = futures_util::future::join_all(stage.iter().map(|name| async move {
            (
                name,
                ctx.supervisor
                    .restart(ProcessSelector::Name(name.clone()))
                    .await,
            )
        }))
        .await;
        for (name, outcome) in outcomes {
            match outcome {
                Ok(infos) => restarted.extend(infos),
                // A `NotFound` here is not necessarily a defect: the walk was
                // planned from a listing the actor has since moved on from, so
                // a sheep that exited in between is one this names and the
                // supervisor no longer matches.
                Err(err) => {
                    tracing::warn!(
                        sheep = %name,
                        %err,
                        "a sheep did not restart in its stage; it may have left the flock since \
                         the walk was planned"
                    );
                    // Every one, where `refusal` below keeps the first:
                    // that one is only ever read when nothing restarted.
                    refused.push(SheepRefusal::new(name.clone(), err.to_string()));
                    refusal.get_or_insert(err);
                }
            }
        }

        let waiting: BTreeSet<String> = stage
            .iter()
            .filter(|name| walk.depended_on.contains(name.as_str()))
            .cloned()
            .collect();
        if waiting.is_empty() {
            continue;
        }
        let bound = stage_bound(ctx, stage);
        let unsettled = crate::boot_order::await_stage(rx, waiting, bound, &ctx.supervisor).await;
        warn_about_unsettled(&unsettled);
    }
    finished(restarted, refusal).map(|rows| (rows, refused))
}

/// [`restart_in_stages`] for a reload: same walk, a different call and a
/// different definition of done.
///
/// A stage is done when every member has emitted a `Reloaded` for each
/// instance the reload accepted, or a `ReloadAbandoned` for the app. Only
/// the instances answered as `Online` are counted, so a stage holding an
/// instance a failed earlier reload left up can advance while its last swap
/// is still running.
///
/// Every member that was refused is named in the second half of the answer.
///
/// # Errors
/// - The first member's refusal, when no member was accepted.
async fn reload_in_stages(
    ctx: &RpcContext,
    walk: &OrderedWalk,
) -> Result<(Vec<ProcessInfo>, Vec<SheepRefusal>), RpcError> {
    let mut accepted = Vec::new();
    let mut refused: Vec<SheepRefusal> = Vec::new();
    let mut refusal = None;
    for stage in &walk.stages {
        let rx = ctx.events.subscribe();
        let outcomes = futures_util::future::join_all(stage.iter().map(|name| async move {
            (
                name,
                ctx.supervisor
                    .reload(ProcessSelector::Name(name.clone()))
                    .await,
            )
        }))
        .await;
        let mut waiting: BTreeMap<String, usize> = BTreeMap::new();
        for (name, outcome) in outcomes {
            match outcome {
                Ok(infos) => {
                    if walk.depended_on.contains(name.as_str()) {
                        let swaps = infos
                            .iter()
                            .filter(|info| info.status == ProcStatus::Online)
                            .count();
                        if swaps > 0 {
                            waiting.insert(name.clone(), swaps);
                        }
                    }
                    accepted.extend(infos);
                }
                // `restart_in_stages`' note about a `NotFound` applies here
                // too.
                Err(err) => {
                    // An app already reloading is refused per app now, so it
                    // contributes no swaps of its own and the stage would
                    // return at once. The reload in flight is still one its
                    // dependants have to wait out, so it is waited for as one.
                    if matches!(err, SupervisorError::ReloadInFlight(_))
                        && walk.depended_on.contains(name.as_str())
                    {
                        waiting.insert(name.clone(), 1);
                    }
                    tracing::warn!(
                        sheep = %name,
                        %err,
                        "a sheep did not reload in its stage; it may have left the flock since \
                         the walk was planned"
                    );
                    // The daemon log is not somewhere a deploy script looks,
                    // so the name rides back on the reply as well.
                    refused.push(SheepRefusal::new(name.clone(), err.to_string()));
                    refusal.get_or_insert(err);
                }
            }
        }

        if waiting.is_empty() {
            continue;
        }
        let bound = reload_stage_bound(ctx, &waiting);
        let unsettled = crate::boot_order::await_reloads(rx, waiting, bound).await;
        warn_about_unsettled(&unsettled);
    }
    finished(accepted, refusal).map(|rows| (rows, refused))
}

/// A stage that ran out of time, named. Nothing is retried and the walk goes
/// on: a stage held to its bound has already cost the operator that wait, and
/// stopping here would leave the fold half done.
fn warn_about_unsettled(unsettled: &BTreeSet<String>) {
    if unsettled.is_empty() {
        return;
    }
    let names: Vec<&str> = unsettled.iter().map(String::as_str).collect();
    tracing::warn!(
        unsettled = ?names,
        "a stage did not settle inside its bound; advancing anyway"
    );
}

/// The walk's answer: the rows it moved, sorted as every operator-facing
/// listing is, or the refusal it kept when it moved nothing.
fn finished(
    mut rows: Vec<ProcessInfo>,
    refusal: Option<SupervisorError>,
) -> Result<Vec<ProcessInfo>, RpcError> {
    match refusal {
        Some(err) if rows.is_empty() => Err(rpc_error(&err)),
        _ => {
            // Sorted here rather than left in stage order: each supervisor
            // call sorts its own answer, and a table stitched from several is
            // otherwise ordered by the graph, which is not what an operator
            // reading a flock listing expects.
            shep_core::protocol::sort_flock(&mut rows);
            Ok(rows)
        }
    }
}