1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
//! Talking to a running sheep, and the bookkeeping around it.
//!
//! Actions, signals and stdin writes all go out to a child that may take its
//! time answering, so each is armed and resolved across turns rather than
//! awaited. The rest are the small operations the other handlers lean on:
//! arming extras, setting a status, emitting an event.
use super::*;
impl<R: ProcessRunner> Actor<R> {
/// Resolves `selector`, puts one action on the shepherd channel of every
/// matched sheep that can take one, and answers with a row per match.
///
/// The rows are joined in [`spawn_trigger_task`], not in `pending`, so
/// nothing is awaited here; see [`Self::handle_reopen`]. Both refusals are
/// decided before the wait: a sheep refused after one was armed would
/// leave a wait nothing drives home.
pub(super) fn begin_action(
&mut self,
selector: &ProcessSelector,
action: String,
params: Option<String>,
caller: Option<String>,
reply: oneshot::Sender<Result<Vec<ActionReply>, SupervisorError>>,
) {
// `matching_ids` answers in id order, so delivery is not arbitrary;
// the answer's own order rests on `spawn_trigger_task`'s final sort.
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut refused = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_action: `matched` holds ids read off this map a moment ago");
let config = slot.entry.spec.config();
let name = config.name.clone();
// Each sheep's own `action_timeout` bounds its own wait.
let action_timeout = config.action_timeout.as_duration();
if matches!(slot.entry.reload, ReloadState::Drainee { .. }) {
refused.push(ActionReply {
id,
name,
outcome: ActionOutcome::Skipped,
});
continue;
}
let Some(to_child) = slot.open_channel().cloned() else {
// A dog has no Flockfile key to set, so its row names the dog
// as the one that has to ask.
let outcome = if slot.entry.dog.is_some() && !config.channel {
ActionOutcome::DogNoChannel
} else {
ActionOutcome::NoChannel
};
refused.push(ActionReply { id, name, outcome });
continue;
};
let answer = self.arm_action(
id,
to_child,
(action.clone(), params.clone(), caller.clone()),
action_timeout,
);
waits.push((id, name, answer));
}
if waits.is_empty() {
// Not a silent success. The rows say what happened to each sheep;
// this says it once in the daemon's own log, where a flock with
// `channel` unset everywhere is one cause rather than a table of
// refusals.
let skipped = refused
.iter()
.filter(|row| row.outcome == ActionOutcome::Skipped)
.count();
tracing::warn!(
action,
matched = refused.len(),
skipped,
"no matched sheep could take this action; nothing was delivered"
);
let _ = reply.send(Ok(refused));
return;
}
spawn_trigger_task(refused, waits, reply);
}
/// Delivers one signal to every matched sheep's own process.
///
/// Off the actor loop, like [`Self::begin_action`]: each delivery is a
/// round trip through a sheep task, bounded by the syscall rather than a
/// timeout. A sheep with no live task answers
/// [`SignalOutcome::NotRunning`] without one. A drainee is signalled like
/// any other live sheep: a signal expects nothing back.
pub(super) fn begin_signal(
&mut self,
selector: &ProcessSelector,
sig: OperatorSignal,
reply: oneshot::Sender<Result<Vec<SignalReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut settled = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_signal: `matched` holds ids read off this map a moment ago");
let name = slot.entry.spec.config().name.clone();
let Some(signals) = slot.signals.clone() else {
settled.push(SignalReply {
id,
name,
outcome: SignalOutcome::NotRunning,
});
continue;
};
let (done, answer) = oneshot::channel();
if signals.try_send(SignalRequest { sig, done }).is_err() {
// A full queue means this sheep's task has not drained several
// signals, which for a syscall-fast handler means it is busy
// dying; a closed one means it already has.
settled.push(SignalReply {
id,
name,
outcome: SignalOutcome::NotRunning,
});
continue;
}
waits.push((id, name, answer));
}
spawn_signal_task(settled, waits, reply);
}
/// Queues [`ShepherdMessage::LambSignalled`] on sheep `id`'s channel, and
/// says whether it was queued.
///
/// `try_send`, so a sheep that is not reading cannot park the actor loop.
/// A sheep with no open channel is told nothing, and neither is one whose
/// process is no longer `root`: the lamb belonged to its predecessor.
pub(super) fn tell_lamb_signalled(
&self,
id: u32,
root: u32,
pid: u32,
sig: OperatorSignal,
) -> bool {
self.sheep
.get(&id)
.filter(|slot| slot.entry.pid == Some(root))
.and_then(SheepSlot::open_channel)
.is_some_and(|to_child| {
to_child
.try_send(ShepherdMessage::LambSignalled {
pid,
signal: sig.as_str().to_string(),
})
.is_ok()
})
}
/// Writes one line to every matched sheep's stdin, off the actor loop like
/// [`Self::begin_signal`].
///
/// A sheep with no live task, or one running without `stdin = true`,
/// answers [`LineOutcome::NoStdin`] off [`SheepSlot::open_stdin`]. A
/// drainee gets the line. The enqueue is `try_send`: awaiting into a full
/// queue would park the actor loop on a wedged sheep's pipe.
pub(super) fn begin_send_line(
&mut self,
selector: &ProcessSelector,
line: String,
reply: oneshot::Sender<Result<Vec<LineReply>, SupervisorError>>,
) {
let matched = self.matching_ids(selector);
if matched.is_empty() {
let _ = reply.send(Err(SupervisorError::NotFound));
return;
}
let mut settled = Vec::new();
let mut waits = Vec::new();
for id in matched {
let slot = self
.sheep
.get(&id)
.expect("begin_send_line: `matched` holds ids read off this map a moment ago");
let name = slot.entry.spec.config().name.clone();
let Some(to_stdin) = slot.open_stdin().cloned() else {
settled.push(LineReply {
id,
name,
outcome: LineOutcome::NoStdin,
});
continue;
};
let (done, answer) = oneshot::channel();
match to_stdin.try_send(StdinWrite {
line: line.clone(),
done,
}) {
Ok(()) => waits.push((id, name, answer)),
// The queue is full: the writer is blocked on a pipe the app
// is not draining. The reason names no duration, since
// `try_send` looked at the queue once and an operator would
// read an elapsed time as the timeout path's bound.
Err(mpsc::error::TrySendError::Full(_)) => settled.push(LineReply {
id,
name,
outcome: LineOutcome::NotWritten {
reason: "the app is not reading its stdin (its queue was \
already full when this line arrived)"
.to_string(),
},
}),
// Closed: the writer task is gone, so the process is too.
Err(mpsc::error::TrySendError::Closed(_)) => settled.push(LineReply {
id,
name,
outcome: LineOutcome::NoStdin,
}),
}
}
spawn_send_line_task(settled, waits, reply);
}
/// Puts one action on `id`'s shepherd channel and arms the wait for its
/// reply, handing back the receiver its outcome will arrive on.
///
/// Infallible: every question an action can be refused over is answered in
/// [`Self::begin_action`]'s selector pass, before this is called.
pub(super) fn arm_action(
&mut self,
id: u32,
to_child: mpsc::Sender<ShepherdMessage>,
(action, params, caller): (String, Option<String>, Option<String>),
timeout: Duration,
) -> oneshot::Receiver<ActionOutcome> {
let (reply, answer) = oneshot::channel();
let stamp = self.next_action_stamp;
self.next_action_stamp += 1;
let waiter = spawn_action_task(
id,
stamp,
ShepherdMessage::Action {
name: action.clone(),
params,
caller,
id: stamp,
},
to_child,
timeout,
self.tx.clone(),
);
// After the task is spawned, and safely: its first act is a send that
// must reach a child and come back through `run_sheep`, none of which
// can happen before this handler returns.
self.sheep
.get_mut(&id)
.expect("arm_action: the slot was read a moment ago")
.actions
.arm(PendingAction {
stamp,
action,
waiter: Some(waiter),
reply,
});
answer
}
/// Forwards one shepherd-channel reply to the action wait it belongs to,
/// if it belongs to one. A reply with nowhere to go is dropped silently;
/// see [`ActionWaits::answer`].
pub(super) fn handle_action_reply(
&mut self,
id: u32,
action: &str,
body: String,
stamp: Option<u64>,
) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(waiter) = slot.actions.answer(action, stamp) {
let _ = waiter.send(body);
}
}
/// An action wait resolved: answer its caller.
///
/// Guarded on the stamp alone, where [`Self::handle_ready_result`] guards
/// on four things: an action's result changes no flock state, and no
/// respawn can make the stamp ambiguous.
pub(super) fn handle_action_result(&mut self, id: u32, stamp: u64, outcome: ActionOutcome) {
let Some(slot) = self.sheep.get_mut(&id) else {
return;
};
if let Some(reply) = slot.actions.resolve(stamp) {
let _ = reply.send(outcome);
}
}
/// Spawns the backoff timer for a scheduled restart. `None` still hops
/// through a task and a mailbox send, so an immediate restart stays
/// observable as a scheduling step.
pub(super) fn schedule_restart(&self, id: u32, epoch: u64, delay: Option<Duration>) {
let tx = self.tx.clone();
tokio::spawn(async move {
if let Some(delay) = delay {
tokio::time::sleep(delay).await;
}
let _ = tx.send(Msg::RestartDue { id, epoch }).await;
});
}
/// Removes `id` from every pending reply's `remaining` set, appending
/// `info` to each match; fulfills (and drops) any pending reply this
/// empties. Returns `true` iff a `Shutdown` reply was just fulfilled.
pub(super) fn resolve_pending(&mut self, id: u32, info: ProcessInfo) -> bool {
let mut shutdown_completed = false;
let mut i = 0;
while i < self.pending.len() {
if self.pending[i].remaining.remove(&id) {
self.pending[i].results.push(info.clone());
}
if self.pending[i].remaining.is_empty() {
let mut pending = self.pending.remove(i);
sort_flock(&mut pending.results);
if matches!(pending.reply, ReplyKind::Shutdown(_)) {
shutdown_completed = true;
}
send_reply(pending.reply, Ok(pending.results));
} else {
i += 1;
}
}
shutdown_completed
}
/// One sheep's transition to `Online`: emits the event, then arms every
/// lifecycle extra its configuration asks for.
///
/// Arming happens at the transition, not the spawn: a liveness probe armed
/// against an app that has not finished starting fails its threshold and
/// restarts the app before it comes up. An abandoned reload's replacement
/// is armed without going through here, since it never becomes `Online`.
pub(super) fn went_online(&mut self, id: u32, info: ProcessInfo, manually: bool) {
// Whatever an earlier reload concluded about this instance, it is
// serving now. See `SheepSlot::ready_failed`.
if let Some(slot) = self.sheep.get_mut(&id) {
slot.ready_failed = false;
}
self.emit(ProcessEventKind::Online, info, manually);
self.arm_extras(id);
}
/// Arms `id`'s lifecycle extras, rebuilding the spec the running process
/// was spawned from.
///
/// Rebuilding is what makes one arming body possible: a caller holds an id
/// and nothing else, and `describe` is pure over a `spec`, `instance` and
/// `credentials` that never change. The store it reads can move under a
/// running sheep, hence [`describe`] and not [`assemble`].
pub(super) fn arm_extras(&mut self, id: u32) {
let Some(extras) = self.extras.as_ref() else {
return;
};
let Some(slot) = self.sheep.get(&id) else {
return;
};
let supervisor = SupervisorHandle {
tx: self.tx.clone(),
};
// Rebuilt for the prober's sake alone, as `spawn_verify_task` does:
// nothing spawns this sheep's own program from it. Extras are armed
// only for a process already running, which resolved its identity to
// start.
let credentials = slot.entry.credentials.credentials();
let described = describe(
&slot.entry.spec,
slot.entry.instance,
&self.paths,
credentials,
&self.secret_view(&slot.entry.spec),
);
self.registry
.arm(&slot.entry, spec_prober(&described), extras, &supervisor);
}
/// Disarms `id`'s lifecycle extras, and its name-group's cron worker and
/// watch when `id` was the last armed instance of `name`.
///
/// Called from every terminal transition. A sheep on its way to
/// `WaitingRestart` keeps its arming, the respawn replacing its liveness
/// loop. Re-disarming is a no-op, so a duplicate site costs nothing and a
/// missing one leaks a task.
pub(super) fn disarm_extras(&mut self, id: u32, name: &str) {
self.registry.disarm(id, name);
}
/// Sets `id`'s status and returns its refreshed snapshot.
pub(super) fn set_status(&mut self, id: u32, status: ProcStatus) -> ProcessInfo {
let slot = self.sheep.get_mut(&id).expect("set_status: unknown id");
slot.entry.status = status;
to_info(&slot.entry, &self.smits)
}
/// Paints or clears one sheep's smit and answers with that sheep's
/// instances as they now stand.
///
/// Every instance of the name, not one row: the map is keyed by name, so
/// every instance carries the mark.
pub(super) fn handle_set_smit(
&mut self,
conn: ConnId,
sheep: &str,
smit: Option<Smit>,
) -> Result<Vec<ProcessInfo>, SupervisorError> {
if !self
.sheep
.values()
.any(|slot| slot.entry.spec.config().name == sheep)
{
// Refused rather than stored against a name nothing holds: an
// orphan entry would show in no listing.
return Err(SupervisorError::NotFound);
}
match smit {
Some(smit) => {
self.smits
.insert(sheep.to_string(), (conn, smit.to_string()));
}
// Only this connection's own; see `Command::SetSmit`'s doc.
None => {
if self
.smits
.get(sheep)
.is_some_and(|(painter, _)| *painter == conn)
{
self.smits.remove(sheep);
}
}
}
Ok(self
.snapshot_all()
.into_iter()
.filter(|info| info.name == sheep)
.collect())
}
/// The working directory each sheep was configured with, by id.
///
/// A sheep with no `cwd` has no entry: it would run in the shepherd's own
/// directory, which is not a statement about that sheep.
#[cfg(feature = "vcs")]
pub(super) fn working_dirs(&self) -> HashMap<u32, PathBuf> {
self.sheep
.values()
.filter_map(|slot| {
let cwd = slot.entry.spec.config().cwd.as_ref()?;
Some((slot.entry.id, PathBuf::from(cwd)))
})
.collect()
}
/// Full flock listing, grouped by app name.
///
/// Sorted by [`sort_flock`], the rule every operator-facing listing takes:
/// name, then instance slot, then id. The slot is in the key because a
/// replacement gets a fresh id at the drainee's slot number. A listing
/// whose rows all report `None` for [`ProcessInfo::instance`] collapses to
/// `(name, id)` order.
pub(super) fn snapshot_all(&self) -> Vec<ProcessInfo> {
let mut listing: Vec<ProcessInfo> = self
.sheep
.values()
.map(|slot| self.with_mark(to_info(&slot.entry, &self.smits)))
.collect();
sort_flock(&mut listing);
listing
}
/// Broadcasts one lifecycle transition. Send failures (no receivers)
/// are not an error: the bus is fire-and-forget from the actor's side.
pub(super) fn emit(&self, event: ProcessEventKind, info: ProcessInfo, manually: bool) {
let _ = self.events.send(SharedEvent::new(BusEvent::Process {
event,
info,
manually,
at_ms: crate::now_ms(),
}));
}
}