Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62#[cfg(unix)]
63#[path = "supervise/admission.rs"]
64mod admission;
65#[cfg(all(test, windows))]
66mod carried_job_tests;
67pub(crate) mod containment;
68#[cfg(all(test, unix))]
69mod containment_tests;
70mod slow_retry;
71use slow_retry::{FailedRetry, FailureContext};
72
73#[cfg(all(test, unix))]
74mod slow_retry_tests;
75
76/// Command-line flag used by supervised modules to find subc.
77///
78/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
79/// The path points at the TCP+key connection file; it is not an ambient signal and
80/// is never inherited by standalone children.
81pub const SUBC_ARG: &str = "--subc";
82
83const DEFAULT_MAX_RESTARTS: u32 = 3;
84const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
85const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
86/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
87/// to contain a real crash loop (which respawns in seconds) and short enough
88/// that unrelated crashes hours apart never accumulate into a permanent stop.
89const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
90/// How long a drain waits for already-dispatched requests to finalize before
91/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
92/// not RPC-scale: the original 2s value silently cut nearly every real tool
93/// call at the fence, making the wait-for-finalize design decorative for the
94/// workloads it existed for. Quiescence short-circuits, so an idle module
95/// restarts immediately regardless of this value; the budget is spent only
96/// when a genuine in-flight request is worth finishing. Per-module override:
97/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
98/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
99/// where a stuck request will never settle).
100pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
101pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
102const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
103const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
104/// How long a drain waits after one busy-gauge probe completes before it asks
105/// again.
106///
107/// A drain re-probes on this short interval of its own rather than on the
108/// module's health cadence. The default cadence (30 s) equals the default
109/// drain ceiling, so a cadence-paced drain that found the module busy at its
110/// first probe never asked a second time: it waited out the whole ceiling and
111/// forced GOODBYE even after the gauges had read zero. The interval is counted
112/// from the moment the previous probe finished, and probes run one at a time,
113/// so a slow module is never asked more often than it can answer. Periodic
114/// health checks outside a drain keep using the configured cadence.
115const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
116/// How long a restart waits for the exited process's output readers.
117///
118/// The restart does not depend on either output reader finishing. A stderr
119/// reader still running at this bound is left running, and its later output goes
120/// into the exited process's own section of the stderr ring (see
121/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
122/// bound no longer decides whether a crash's last lines are kept: under load
123/// the reader may simply not have been scheduled yet, and cutting it there
124/// discarded exactly the lines that explained the crash.
125///
126/// What the bound still decides is when the tail starts reporting
127/// `Incomplete`: a pipe open past it usually means a descendant of the exited
128/// process still holds it, and the tail cannot claim to be whole until that
129/// pipe closes. That is also the only case in which the wait costs the restart
130/// anything, because a pipe with no other holder reaches EOF when the process
131/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
132/// `Captured` at EOF with nothing lost.
133///
134/// The stdout reader carries no ring, only the capture file. It also continues
135/// to EOF after this bound: child exit closes the writer but does not mean the
136/// reader has consumed all buffered bytes. Late output from either pipe can
137/// therefore reach the shared capture file after the successor starts.
138const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
139/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
140pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
141const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
142/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
143/// dropped because the subscriber stopped draining its frames. The detail's
144/// `first_undelivered_cursor` names the first event it did not receive; the
145/// client resubscribes from the last cursor it did receive.
146pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
147
148#[cfg(target_os = "macos")]
149struct PrivacyExec {
150    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
151    deadline: tokio::time::Instant,
152    expected: Option<subc_os::FileIdentity>,
153    trampoline: Option<subc_os::FileIdentity>,
154    script: bool,
155    module_id: String,
156    #[cfg(test)]
157    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
158}
159
160#[cfg(all(test, target_os = "macos"))]
161pub(crate) fn test_privacy_trampoline() -> PathBuf {
162    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
163    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
164    let path = std::env::current_exe()
165        .unwrap()
166        .parent()
167        .unwrap()
168        .parent()
169        .unwrap()
170        .join("privacy-trampoline-fixture");
171    // Without the fixture every macOS spawn is refused, and the tests that
172    // spawn fail later as a module in state Failed, which names the wrong
173    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
174    assert!(
175        path.exists(),
176        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
177         --bins --features test-support` or `cargo test -p subc-daemon` first",
178        path.display()
179    );
180    path
181}
182
183#[cfg(target_os = "macos")]
184fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
185    use std::io::Read;
186    let mut probe = std::process::Command::new(path)
187        .args(["__disclaim-exec", "--probe"])
188        .stdin(Stdio::null())
189        .stdout(Stdio::piped())
190        .stderr(Stdio::piped())
191        .spawn()
192        .map_err(|error| {
193            format!(
194                "privacy trampoline probe failed for {}: {error}",
195                path.display()
196            )
197        })?;
198    let deadline = std::time::Instant::now() + Duration::from_secs(5);
199    let status = loop {
200        match probe.try_wait() {
201            Ok(Some(status)) => break status,
202            Ok(None) if std::time::Instant::now() < deadline => {
203                std::thread::sleep(Duration::from_millis(5))
204            }
205            result => {
206                let _ = probe.kill();
207                let _ = probe.wait();
208                return Err(format!(
209                    "privacy trampoline probe failed or timed out for {}: {result:?}",
210                    path.display()
211                ));
212            }
213        }
214    };
215    let mut answer = String::new();
216    if let Some(stdout) = probe.stdout.take() {
217        let _ = stdout.take(256).read_to_string(&mut answer);
218    }
219    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
220        return Ok(());
221    }
222    let mut diagnostic = String::new();
223    if let Some(stderr) = probe.stderr.take() {
224        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
225    }
226    let cause = diagnostic
227        .trim()
228        .strip_prefix("ck-subc: own privacy identity refused: ")
229        .unwrap_or("binary does not implement the privacy trampoline protocol");
230    Err(format!(
231        "{cause}: probe of {} exited {status}",
232        path.display()
233    ))
234}
235
236#[cfg(target_os = "macos")]
237fn privacy_command(
238    spec: &ModuleSpec,
239    roster: &ChildRoster,
240) -> Result<
241    (
242        Command,
243        Option<PrivacyExec>,
244        subc_os::privacy_identity::ExecAcknowledgement,
245    ),
246    SuperviseError,
247> {
248    let failure = |cause: String| {
249        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
250        SuperviseError::Spawn {
251            program: spec.program.clone(),
252            source: io::Error::other(cause),
253            cgroup_path: None,
254        }
255    };
256    let trampoline = roster.privacy_trampoline().map_err(failure)?;
257    // Resolve PATH with the same environment the Command will receive. For
258    // scripts retain the existing orphan-identity rule: the kernel chooses
259    // the interpreter, and its observed image is the one recorded. Do not
260    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
261    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
262        let path = spec
263            .env
264            .iter()
265            .find(|(key, _)| key == "PATH")
266            .map(|(_, value)| std::ffi::OsString::from(value))
267            .or_else(|| std::env::var_os("PATH"))
268            .unwrap_or_else(|| "/usr/bin:/bin".into());
269        std::env::split_paths(&path)
270            .map(|dir| dir.join(&spec.program))
271            .find(|path| path.is_file())
272            .unwrap_or_else(|| spec.program.clone())
273    } else {
274        spec.program.clone()
275    };
276    let expected = subc_os::file_identity(&program);
277    let trampoline_image = subc_os::file_identity(&trampoline);
278    let script = {
279        use std::io::Read;
280        let mut prefix = [0u8; 2];
281        std::fs::File::open(&program)
282            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
283    };
284    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
285        return Err(failure(
286            "privacy identity module executable is missing or is the trampoline itself".to_string(),
287        ));
288    }
289    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
290        .map_err(|error| failure(error.to_string()))?;
291    let reader =
292        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
293    let mut command = Command::new(&trampoline);
294    command
295        .arg("__disclaim-exec")
296        .arg(ack.fd().to_string())
297        .arg(&program);
298    Ok((
299        command,
300        Some(PrivacyExec {
301            reader,
302            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
303            expected,
304            trampoline: trampoline_image,
305            script,
306            module_id: spec.module_id.clone(),
307            #[cfg(test)]
308            confirmation_pause: spec
309                .env
310                .iter()
311                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
312                .map(|(_, address)| (address.clone(), None)),
313        }),
314        ack,
315    ))
316}
317
318struct SupervisedChild {
319    subtree_target: crate::subtree_resources::Target,
320    child: containment::ProcessChild,
321    exec: crate::activity::ExecOwner,
322    #[cfg(windows)]
323    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
324    #[cfg(target_os = "macos")]
325    privacy_exec: Option<PrivacyExec>,
326    /// Set once this launch's exec acknowledgement confirms the module image.
327    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
328    /// `subc_os::privacy_identity`), which then replaces itself with the
329    /// module. The supervisor owns and can kill that pid from spawn, but
330    /// status readers report it only after this latch is set, so nothing
331    /// reports the trampoline's image as the module's.
332    #[cfg(target_os = "macos")]
333    report_ready: Arc<OnceLock<()>>,
334    /// Supervisor failure detail retained for the terminal record, including
335    /// image refusals and first-HELLO timeouts after a successful spawn.
336    spawn_failure: Option<String>,
337    /// The protocol this process was launched with. A reload can store a new
338    /// launch spec with a different protocol, but that takes effect only at the
339    /// next spawn, so this process keeps being handled by the protocol it
340    /// actually speaks.
341    protocol: ModuleProtocol,
342    /// This container's cgroup name: a bounded module/slot label followed by a
343    /// unique suffix (when cgroup placement is on). Carried replacements reuse
344    /// the name; ordinary launches each get a fresh container. A
345    /// retired process in a slot may still be draining when a later one is
346    /// spawned into that slot, so the suffix keeps the later process out of
347    /// the retired one's cgroup, which is the domain a kill applies to.
348    #[cfg(target_os = "linux")]
349    module_id: String,
350    #[cfg(target_os = "linux")]
351    cgroup_placement: Option<subc_cgroup::Placement>,
352    /// The job for an ordinary, non-carried launch. For carried
353    /// launches the actor owns it and the child holds only a weak container link.
354    ///
355    /// Dropping this handle is what reaps a surviving tree when no supervisor
356    /// code runs — a daemon crash — because the job carries
357    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
358    ///
359    /// That limit is not crash-only, and the difference is worth knowing: a
360    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
361    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
362    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
363    /// module at once. Before this change they survived that, saw EOF on the
364    /// control socket, and ran their own teardown; Unix keeps that path
365    /// deliberately, so a module can seal a WAL or close a capture rather than
366    /// be killed mid-write. So this trades graceful teardown on every Windows
367    /// daemon stop for containment on a crash, which is the right way round
368    /// today: orphaned GPU workers are a reported, recurring problem, and the
369    /// modules that write most heavily do not run on Windows.
370    ///
371    /// The fix is a real Windows stop path — the daemon draining before it
372    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
373    /// reaches only what the drain left behind, which is what it should reach.
374    #[cfg(windows)]
375    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
376    stdout_pump: Option<JoinHandle<()>>,
377    stderr_pump: Option<StderrPump>,
378    stderr_ring: Arc<Mutex<StderrRing>>,
379    /// Conservative monotonic bound sampled before `command.spawn()`, not when
380    /// identity recording finishes or the spawn generation is published.
381    spawned_at: Instant,
382    /// Ordinary spawns must send their first HELLO within this budget. Once
383    /// accepted, a later disconnect must not restart the registration clock.
384    registration_deadline: Option<Instant>,
385    registration_watch: Option<RegistrationWatch>,
386    spawned_at_ms: u64,
387    spawned_from: PathBuf,
388    launch_spec_digest: LaunchSpecDigest,
389    spawned_file_identity: Option<SpawnedFileIdentity>,
390    process_start_time: Option<u64>,
391    process_identity: Option<ProcessIdentity>,
392    pid: u32,
393    /// This process's entry in the daemon's child roster, released when the
394    /// process is reaped or this handle is dropped.
395    roster_guard: Option<crate::child_roster::RosterGuard>,
396}
397
398impl SupervisedChild {
399    fn registered_since_spawn(&self) -> bool {
400        self.registration_watch
401            .as_ref()
402            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
403    }
404
405    fn id(&self) -> Option<u32> {
406        Some(self.pid)
407    }
408
409    fn process_identity(&self) -> Option<ProcessIdentity> {
410        self.process_identity
411    }
412
413    async fn wait(&mut self) -> io::Result<ExitStatus> {
414        #[cfg(target_os = "macos")]
415        self.confirm_privacy_exec().await;
416        // The roster entry is NOT released here. A daemon shutdown waits for the
417        // roster to empty and then exits the process, so releasing at the reap
418        // let it exit before the exit handler wrote this child's terminal record
419        // (the stderr drain and snapshot update sit in between), and the
420        // shutdown's own `daemon_shutdown` record was intermittently lost. The
421        // caller releases it after recording the exit (`release_roster`), and
422        // dropping the handle releases it too.
423        let result = self.child.wait().await;
424        #[cfg(target_os = "linux")]
425        if result.is_ok() && self.child.container().is_none() {
426            if let Some(placement) = self.cgroup_placement.as_ref() {
427                // Ordinary children still own a per-spawn cgroup: returning
428                // from wait must leave it empty and removed, as before carry.
429                // Keep cleanup ownership across cancellation until it finishes.
430                cleanup_reaped_cgroup(placement, &self.module_id).await;
431                self.cgroup_placement = None;
432            }
433        }
434        result
435    }
436
437    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
438        if self.child.container().is_some() {
439            if !carry {
440                self.child.terminate_container().await;
441            }
442        } else {
443            #[cfg(target_os = "linux")]
444            if let Some(placement) = self.cgroup_placement.as_ref() {
445                cleanup_reaped_cgroup(placement, &self.module_id).await;
446                self.cgroup_placement = None;
447            }
448        }
449        self.child.reap().await?;
450        self.release_roster();
451        Ok(())
452    }
453
454    #[cfg(target_os = "macos")]
455    async fn confirm_privacy_exec(&mut self) {
456        let Some(pending) = &mut self.privacy_exec else {
457            return;
458        };
459        let result = tokio::time::timeout_at(pending.deadline, async {
460            let mut record = Vec::new();
461            loop {
462                let mut ready = pending.reader.readable().await?;
463                let read = ready.try_io(|reader| {
464                    use std::io::Read;
465                    let mut reader = reader.get_ref();
466                    let mut buffer = [0u8; 256];
467                    reader.read(&mut buffer).map(|count| (count, buffer))
468                });
469                match read {
470                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
471                    Ok(Ok((count, buffer))) => {
472                        if record.len() + count > 1024 {
473                            return Err(io::Error::other(
474                                "privacy exec refusal record is too long",
475                            ));
476                        }
477                        record.extend_from_slice(&buffer[..count]);
478                    }
479                    Ok(Err(error)) => return Err(error),
480                    Err(_) => continue,
481                }
482            }
483        })
484        .await;
485        #[cfg(test)]
486        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
487            use tokio::io::{AsyncReadExt, AsyncWriteExt};
488            if stream.is_none() {
489                *stream = Some(
490                    tokio::net::TcpStream::connect(address.as_str())
491                        .await
492                        .unwrap(),
493                );
494                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
495                stream
496                    .as_mut()
497                    .unwrap()
498                    .write_all(&self.pid.to_ne_bytes())
499                    .await
500                    .unwrap();
501            }
502            let mut reply = [0];
503            stream
504                .as_mut()
505                .unwrap()
506                .read_exact(&mut reply)
507                .await
508                .unwrap();
509            pending.confirmation_pause = None;
510            if &reply == b"F" {
511                Ok(Err(io::Error::other("injected late exec refusal")))
512            } else {
513                assert_eq!(&reply, b"X");
514                result
515            }
516        } else {
517            result
518        };
519        // Keep the reader in self across await: select cancellation must not
520        // discard the handshake or reset its original five-second deadline.
521        let pending = self.privacy_exec.as_ref().expect("pending exec");
522        let cause = match result {
523            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
524            Ok(Err(error)) => Some(format!(
525                "privacy identity exec acknowledgement failed: {error}"
526            )),
527            Ok(Ok(record)) if !record.is_empty() => Some(
528                std::str::from_utf8(&record)
529                    .ok()
530                    .and_then(|record| {
531                        record
532                            .trim()
533                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
534                    })
535                    .filter(|cause| !cause.is_empty())
536                    .unwrap_or("invalid privacy exec refusal record")
537                    .to_string(),
538            ),
539            Ok(Ok(_)) => match self.child.try_wait() {
540                // Empty EOF is the exec acknowledgement. A real module may exit
541                // immediately, including with a reserved trampoline status; no
542                // image is admitted, and its ordinary exit contract stays intact.
543                Ok(Some(_status)) => None,
544                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
545                Ok(None) => {
546                    let image = observe_spawned_image(self.pid);
547                    if let Some(image) = image.filter(|image| {
548                        image.executable.is_some()
549                            && image.executable != pending.trampoline
550                            && (image.executable == pending.expected || pending.script)
551                    }) {
552                        if let Err(error) = self
553                            .roster_guard
554                            .as_ref()
555                            .map(|guard| guard.confirm_executable(image))
556                            .transpose()
557                        {
558                            Some(format!(
559                                "could not durably record confirmed module image: {error}"
560                            ))
561                        } else {
562                            let _ = self.report_ready.set(());
563                            info!(module_id = %pending.module_id, pid = self.pid,
564                                "module spawned with own privacy identity (responsibility disclaimed)");
565                            None
566                        }
567                    } else if image.is_none()
568                        || image.is_some_and(|image| image.executable.is_none())
569                    {
570                        // A process can exit between try_wait and the kernel
571                        // image read. Empty EOF already acknowledged exec, so
572                        // preserve that module's ordinary exit rather than
573                        // mislabel a disappearing image as trampoline refusal.
574                        // Keep pending in self across await so cancellation does
575                        // not discard validation or reset its original deadline.
576                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
577                            Ok(Ok(_status)) => None,
578                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
579                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
580                        }
581                    } else {
582                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
583                    }
584                }
585            },
586        };
587        let pending = self.privacy_exec.take().expect("pending exec");
588        // Exec confirmation describes launch success, not roster retirement.
589        // Keep even a reaped child recorded until its terminal history exists.
590        self.exec.0.finish(if cause.is_some() {
591            crate::activity::ExecState::Refused
592        } else {
593            crate::activity::ExecState::CommittedExec
594        });
595        if let Some(cause) = cause {
596            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
597            self.spawn_failure = Some(cause);
598            // No image is admitted on failure. Reach the entire fresh process
599            // group, including a module which spawned a helper before refusal.
600            if let Some(container) = self.child.container() {
601                container.hard_kill();
602            } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
603                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
604            }
605            let _ = self.child.start_kill();
606        }
607    }
608
609    /// Releases the leader's roster entry after recording and finalizing its
610    /// exit. A carried container retains its separate roster entry and, for a
611    /// Unix process group, its unreaped first leader that reserves the group ID.
612    fn release_roster(&mut self) {
613        self.roster_guard = None;
614    }
615
616    /// Kill the child and, where containment is available, its process tree.
617    ///
618    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
619    /// helper process leaked the helper — the Synapse embedding module's CUDA
620    /// worker holds the GPU allocation, so the leak cost VRAM until the next
621    /// restart of something else. Terminating the job reaches grandchildren that
622    /// a tree walk cannot, including one whose parent has already exited and
623    /// been reparented away.
624    ///
625    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
626    /// direct-child kill still decides the outcome, so containment can never
627    /// change whether a module is reported as stopped.
628    fn start_kill(&mut self) -> io::Result<()> {
629        if let Some(container) = self.child.container() {
630            container.hard_kill();
631            return self.kill_leader();
632        }
633        #[cfg(windows)]
634        if let Some(job) = &self.job {
635            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
636                debug!(
637                    error = %error,
638                    "job termination failed; the direct-child kill still owns the outcome"
639                );
640            }
641        }
642        #[cfg(target_os = "linux")]
643        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
644        self.child.start_kill()
645    }
646
647    fn kill_leader(&mut self) -> io::Result<()> {
648        self.child.start_kill()
649    }
650
651    async fn drain_stderr(&mut self, module_id: &str) {
652        if let Some(pump) = self.stdout_pump.take() {
653            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
654        }
655
656        let Some(pump) = self.stderr_pump.take() else {
657            return;
658        };
659        settle_stderr_pump(
660            module_id,
661            &self.stderr_ring,
662            pump,
663            STDERR_PUMP_DRAIN_TIMEOUT,
664        )
665        .await;
666    }
667}
668
669/// Bound the restart's wait, not the lifetime of the exited child's stdout
670/// reader. Slow pipe reads or capture writes must not turn the bound into lost
671/// output; dropping the handle lets the reader finish at EOF, like stderr.
672async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
673    match timeout(bound, &mut pump).await {
674        Ok(Ok(())) => {}
675        Ok(Err(error)) => {
676            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
677        }
678        Err(_) => {
679            drop(pump);
680            warn!(
681                module_id,
682                waited = ?bound,
683                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
684            );
685        }
686    }
687}
688
689/// The reader task for one process's stderr, with the ring generation its
690/// lines are attributed to.
691struct StderrPump {
692    task: JoinHandle<()>,
693    generation: u64,
694}
695
696/// Retire an exited process's stderr reader and wait up to `bound` for it to
697/// reach EOF. A reader still running at the bound is detached, not stopped: it
698/// keeps filling the exited process's section of the ring until its pipe
699/// closes, and the tail reads `Incomplete` until then. See
700/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
701async fn settle_stderr_pump(
702    module_id: &str,
703    ring: &Arc<Mutex<StderrRing>>,
704    pump: StderrPump,
705    bound: Duration,
706) {
707    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
708    let StderrPump {
709        mut task,
710        generation,
711    } = pump;
712    lock().retire_pump(generation);
713    match timeout(bound, &mut task).await {
714        Ok(Ok(())) => {}
715        Ok(Err(err)) => {
716            let mut ring = lock();
717            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
718            ring.finish_pump(generation);
719            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
720        }
721        Err(_) => {
722            // Dropping the handle detaches the task; it ends at EOF on its pipe.
723            drop(task);
724            lock().mark_pump_late(
725                generation,
726                format!(
727                    "stderr of the exited process had not reached EOF {bound:?} after it was \
728                     retired (a descendant may still hold the pipe open); lines it still \
729                     writes are kept in that process's section"
730                ),
731            );
732            warn!(
733                module_id,
734                waited = ?bound,
735                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
736            );
737        }
738    }
739}
740
741fn registration_release_events() -> &'static watch::Sender<u64> {
742    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
743    EVENTS.get_or_init(|| {
744        let (sender, _receiver) = watch::channel(0);
745        sender
746    })
747}
748
749pub(crate) fn notify_registration_release() {
750    let events = registration_release_events();
751    let next_generation = (*events.borrow()).wrapping_add(1);
752    events.send_replace(next_generation);
753}
754
755/// How to launch one singleton module process.
756#[derive(Debug, Clone, PartialEq, Eq)]
757pub struct ModuleSpec {
758    pub module_id: String,
759    pub program: PathBuf,
760    pub args: Vec<String>,
761    pub env: Vec<(String, String)>,
762    /// When true this is a reserved module: each spawn gets a fresh one-time launch
763    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
764    /// process can register this module_id (a security-boundary module like the
765    /// credential vault must not be impersonable while it is down/restarting).
766    pub reserved: bool,
767    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
768    /// Prefixes come from daemon config and must end in `:` before they reach the
769    /// supervisor; the owner module's current spawn nonce authorizes claims under
770    /// each prefix.
771    pub reserved_prefixes: Vec<String>,
772    /// The wire protocol this module speaks, as DECLARED in daemon config.
773    ///
774    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
775    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
776    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
777    /// and NO launch nonce, and a clean exit the daemon did not request is
778    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
779    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
780    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
781    /// because a process ignores an environment variable it does not read.
782    ///
783    /// The argument is the part that cannot be "harmless to a process that
784    /// ignores it": a stock binary exits on an unknown flag before it listens
785    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
786    /// first conformance run against this mode found it. The nonce is withheld
787    /// because a process that will never present it gains nothing from holding
788    /// it, and a secret in the environment of a process that does not need it is
789    /// a leak surface for no benefit.
790    pub protocol: ModuleProtocol,
791    /// Whether two processes of this module may run at once, which is what a
792    /// blue/green swap does for the length of its overlap. Declared in daemon
793    /// config because the daemon must be able to answer it while the module is
794    /// down, and so a module cannot talk itself into it after registering.
795    pub overlap: ModuleOverlap,
796    /// Keep module-started helpers in the same container across replacements.
797    /// This is a daemon-config policy, fixed for the lifetime of a container;
798    /// stopping the module or losing its supervisor still destroys the container.
799    pub carry_containment: bool,
800}
801
802/// Only the configured launch inputs enter this value; spawned connection paths,
803/// nonces and inherited environment never do. Its bytes remain daemon-private.
804#[derive(Clone, PartialEq, Eq)]
805pub(crate) struct LaunchSpecDigest([u8; 32]);
806
807impl fmt::Debug for LaunchSpecDigest {
808    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
809        f.write_str("LaunchSpecDigest([redacted])")
810    }
811}
812
813impl LaunchSpecDigest {
814    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
815        fn field(hash: &mut Sha256, bytes: &[u8]) {
816            hash.update((bytes.len() as u64).to_le_bytes());
817            hash.update(bytes);
818        }
819        let mut hash = Sha256::new();
820        if let Some(program) = spec.program.to_str() {
821            field(&mut hash, b"utf8-program");
822            field(&mut hash, program.as_bytes());
823        } else {
824            // The daemon config is UTF-8, but in-process callers may provide
825            // native paths: lossy conversion would collapse distinct programs.
826            #[cfg(unix)]
827            {
828                use std::os::unix::ffi::OsStrExt;
829                field(&mut hash, b"unix-program");
830                field(&mut hash, spec.program.as_os_str().as_bytes());
831            }
832            #[cfg(windows)]
833            {
834                use std::os::windows::ffi::OsStrExt;
835                field(&mut hash, b"windows-program");
836                let bytes = spec
837                    .program
838                    .as_os_str()
839                    .encode_wide()
840                    .flat_map(u16::to_le_bytes)
841                    .collect::<Vec<_>>();
842                field(&mut hash, &bytes);
843            }
844            #[cfg(not(any(unix, windows)))]
845            field(&mut hash, spec.program.to_string_lossy().as_bytes());
846        }
847        hash.update((spec.args.len() as u64).to_le_bytes());
848        for arg in &spec.args {
849            field(&mut hash, arg.as_bytes());
850        }
851        let mut env = spec.env.iter().collect::<Vec<_>>();
852        env.sort_unstable();
853        hash.update((env.len() as u64).to_le_bytes());
854        for (key, value) in env {
855            field(&mut hash, key.as_bytes());
856            field(&mut hash, value.as_bytes());
857        }
858        field(
859            &mut hash,
860            match spec.protocol {
861                ModuleProtocol::Subc => b"subc",
862                ModuleProtocol::None => b"none",
863            },
864        );
865        Self(hash.finalize().into())
866    }
867}
868
869/// Whether a module tolerates a second process of itself running alongside.
870///
871/// Most modules are single-writer on their store (a WAL, a capture log, a
872/// resident index behind a writer barrier), and two processes on one store
873/// corrupt it. So a swap, which overlaps the old and new process by design,
874/// is refused unless the module's config opts in with `overlap: "safe"`.
875#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
876pub enum ModuleOverlap {
877    /// Never run two processes of this module at once. The default.
878    #[default]
879    Exclusive,
880    /// The module has said a second process of itself is harmless for the
881    /// length of a swap.
882    ///
883    /// Declare it only if a second instance can run for a few seconds without
884    /// touching ANY single-writer store: every database, WAL, index, projector
885    /// and scheduled job the module owns. A lease on part of that state is not
886    /// enough. broca's session lease guards WAL appends while its run index, its
887    /// store projector and its archive fold timer (which unlinks live WAL files)
888    /// stay single-writer, so broca is exclusive despite holding a lease. The
889    /// refusal only fires after this has been decided, so the decision is the
890    /// check.
891    Safe,
892}
893
894impl ModuleOverlap {
895    pub fn as_str(self) -> &'static str {
896        match self {
897            Self::Exclusive => "exclusive",
898            Self::Safe => "safe",
899        }
900    }
901}
902
903/// Environment variable telling a spawned module which case it was started
904/// for, before it sends HELLO. Only a swap candidate carries it, as
905/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
906///
907/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
908/// longer because nobody waits on it, while a plain restart must flip ready
909/// quickly because callers see `module_warming` until it does. Absence means
910/// plain restart, the safe reading. The daemon trusts nothing about it; the
911/// candidate is proven by its launch nonce at HELLO.
912pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
913/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
914pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
915/// How long a swap waits for its candidate to register and declare itself
916/// ready when the operator does not say. A module warming as a swap candidate
917/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
918/// daemon allows that plus time to start the process and send HELLO.
919pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
920
921/// Bounded restart policy for crash exits.
922///
923/// `max_restarts` is the number of replacement processes allowed after the
924/// initial spawn WITHIN `window`. After that many crash restarts inside one
925/// window the module enters [`ModuleState::Failed`] and the supervisor stops
926/// the crash loop.
927///
928/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
929/// and that only survived because crashes were rare: a module that crashed
930/// three times across a week was disabled forever by crashes that had nothing
931/// to do with each other. That stopped being survivable once modules began
932/// exiting non-zero whenever the daemon's connection to them drops, because
933/// then every daemon-side connection drop spends a unit of the same budget and
934/// one flappy hour permanently stops a healthy module. Restarts older than
935/// `window` release their slot, so a module that crashed twice yesterday has a
936/// full budget today, while a genuine crash loop -- which is fast by
937/// definition -- still reaches the cap and stops.
938#[derive(Debug, Clone, Copy, PartialEq, Eq)]
939pub struct RestartPolicy {
940    pub max_restarts: u32,
941    /// Base delay before a crash replacement. The actual delay escalates with
942    /// the number of recent crash replacements and is capped by `max_backoff`.
943    pub backoff: Duration,
944    /// Maximum delay before a crash replacement.
945    pub max_backoff: Duration,
946    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
947    /// budget effectively infinite (nothing is ever in-window), which is why
948    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
949    pub window: Duration,
950}
951
952impl RestartPolicy {
953    /// A policy with the default crash window. Callers that care about the
954    /// window say so with [`Self::with_window`]; the ones that do not are
955    /// asking for the standard rate limit, not for no limit.
956    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
957        Self {
958            max_restarts,
959            backoff,
960            max_backoff: DEFAULT_MAX_BACKOFF,
961            window: DEFAULT_RESTART_WINDOW,
962        }
963    }
964
965    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
966        self.max_backoff = max_backoff;
967        self
968    }
969
970    pub fn with_window(mut self, window: Duration) -> Self {
971        self.window = window;
972        self
973    }
974
975    /// Calculate the capped exponential delay for the next crash replacement.
976    /// `restart_in_window` is zero for the first replacement after an operator
977    /// action (restart, reload, re-enable) cleared the crash ring, or after all
978    /// older crash replacements have aged out of the window.
979    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
980        if self.backoff.is_zero() || self.max_backoff.is_zero() {
981            return Duration::ZERO;
982        }
983
984        let mut delay = self.backoff;
985        for _ in 0..restart_in_window {
986            if delay >= self.max_backoff {
987                return self.max_backoff;
988            }
989            delay = delay
990                .checked_mul(10)
991                .unwrap_or(self.max_backoff)
992                .min(self.max_backoff);
993        }
994        delay.min(self.max_backoff)
995    }
996
997    /// The one sentence that explains a budget-exhausted stop, used for both the
998    /// log line and the terminal record so the two cannot drift. It names the
999    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1000    /// exactly what this budget is not.
1001    fn budget_exhausted_detail(&self) -> String {
1002        format!(
1003            "crash budget exhausted: max_restarts={} within window_secs={}",
1004            self.max_restarts,
1005            self.window.as_secs()
1006        )
1007    }
1008}
1009
1010impl Default for RestartPolicy {
1011    fn default() -> Self {
1012        Self {
1013            max_restarts: DEFAULT_MAX_RESTARTS,
1014            backoff: DEFAULT_BACKOFF,
1015            max_backoff: DEFAULT_MAX_BACKOFF,
1016            window: DEFAULT_RESTART_WINDOW,
1017        }
1018    }
1019}
1020
1021#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1022struct CrashRestartSchedule {
1023    restart_in_window: u32,
1024    delay: Duration,
1025}
1026
1027/// Whether the daemon itself will bring this module back after the exit being
1028/// handled: it is enabled AND its in-window crash restarts are below the cap.
1029///
1030/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1031/// the window are dropped here rather than by a timer, so the count is right
1032/// the moment somebody asks and no bookkeeping runs for idle modules.
1033fn daemon_will_restart(
1034    state: &mut SupervisorSnapshot,
1035    policy: &RestartPolicy,
1036    now: Instant,
1037) -> bool {
1038    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1039}
1040
1041const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1042const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1043const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1044const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1045
1046#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1047pub enum HealthAction {
1048    Report,
1049    Restart,
1050    Alert,
1051}
1052
1053impl fmt::Display for HealthAction {
1054    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1055        f.write_str(match self {
1056            Self::Report => "report",
1057            Self::Restart => "restart",
1058            Self::Alert => "alert",
1059        })
1060    }
1061}
1062
1063#[derive(Debug, Clone, PartialEq, Eq)]
1064pub struct HealthConfig {
1065    /// Optional loopback HTTP endpoint for a managed non-wire process.
1066    /// Changing it applies live on rescan; the process protocol changes only
1067    /// at its next spawn.
1068    pub http: Option<String>,
1069    pub cadence: Duration,
1070    pub deadline: Duration,
1071    pub failure_threshold: u32,
1072    pub on_degraded: HealthAction,
1073    pub on_failing: HealthAction,
1074    pub critical: bool,
1075}
1076
1077impl Default for HealthConfig {
1078    fn default() -> Self {
1079        Self {
1080            http: None,
1081            cadence: DEFAULT_HEALTH_CADENCE,
1082            deadline: DEFAULT_HEALTH_DEADLINE,
1083            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1084            on_degraded: HealthAction::Report,
1085            on_failing: HealthAction::Report,
1086            critical: false,
1087        }
1088    }
1089}
1090
1091/// The supervisor's view of one module's health, relayed to clients over
1092/// channel-0 and rendered by `ck health`.
1093///
1094/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1095/// stated here rather than only at the wire type a consumer reads. A reader can
1096/// look up what `None` means; only a writer can silently change it, and the
1097/// writer has no reason to go looking at a downstream contract before editing.
1098///
1099/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1100/// back to `None` on re-registration precisely so a respawned module does not
1101/// carry its predecessor's timestamp — so an old value and an absent one call for
1102/// opposite readings, and anything that defaulted this to a number would make a
1103/// never-probed module indistinguishable from one probed at the epoch.
1104///
1105/// `detail` and `metrics` are `None` when the module published none on this
1106/// probe, which does not mean it reported nothing wrong — it is also the shape
1107/// when the probe never reached it. `last_probe_ms` is what separates those.
1108#[derive(Debug, Clone, PartialEq)]
1109pub struct ModuleHealthStatus {
1110    pub status: SupervisorHealthStatus,
1111    pub last_probe_ms: Option<u64>,
1112    pub detail: Option<String>,
1113    pub metrics: Option<Value>,
1114    pub consecutive_failures: u32,
1115    /// Number of replies received after a recurring health probe's deadline.
1116    /// Unlike a timeout, every increment proves the module was alive.
1117    pub late_answer_count: u64,
1118    /// End-to-end latency of the newest late reply, measured from probe start.
1119    pub last_late_answer_latency_ms: Option<u64>,
1120    pub last_action: Option<String>,
1121    /// Set together with `last_action`; the pair moves as one, and both being
1122    /// absent means no escalation has ever been taken rather than that the last
1123    /// one succeeded.
1124    pub last_action_ms: Option<u64>,
1125}
1126
1127impl Default for ModuleHealthStatus {
1128    fn default() -> Self {
1129        Self {
1130            status: SupervisorHealthStatus::Unknown,
1131            last_probe_ms: None,
1132            detail: None,
1133            metrics: None,
1134            consecutive_failures: 0,
1135            late_answer_count: 0,
1136            last_late_answer_latency_ms: None,
1137            last_action: None,
1138            last_action_ms: None,
1139        }
1140    }
1141}
1142
1143/// Typed lifecycle state for a supervised module.
1144#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1145pub enum ModuleState {
1146    Starting,
1147    Running,
1148    Unresponsive,
1149    Restarting,
1150    Draining,
1151    Stopped,
1152    Failed,
1153    Disabled,
1154}
1155
1156impl fmt::Display for ModuleState {
1157    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1158        f.write_str(match self {
1159            Self::Starting => "starting",
1160            Self::Running => "running",
1161            Self::Unresponsive => "unresponsive",
1162            Self::Restarting => "restarting",
1163            Self::Draining => "draining",
1164            Self::Stopped => "stopped",
1165            Self::Failed => "failed",
1166            Self::Disabled => "disabled",
1167        })
1168    }
1169}
1170
1171/// Supervisor classification of a child-process exit.
1172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1173pub enum ExitKind {
1174    Clean,
1175    Crash,
1176    DeliberateSeverance,
1177}
1178
1179impl From<ExitKind> for TerminalExitKind {
1180    fn from(kind: ExitKind) -> Self {
1181        match kind {
1182            ExitKind::Clean => Self::Clean,
1183            ExitKind::Crash => Self::Crash,
1184            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1185        }
1186    }
1187}
1188
1189/// Exact process identity retained when a supervised module registers its
1190/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1191#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1192pub(crate) struct ProcessIdentity {
1193    pub(crate) pid: u32,
1194    pub(crate) start_time: u64,
1195}
1196
1197/// Last observed child exit, if any.
1198#[derive(Debug, Clone, PartialEq, Eq)]
1199pub struct ExitReport {
1200    pub kind: ExitKind,
1201    pub code: Option<i32>,
1202    pub signal: Option<i32>,
1203    pub at_ms: u64,
1204}
1205
1206/// Point-in-time module status answerable by subc without forwarding to the
1207/// module process.
1208#[derive(Debug, Clone, PartialEq)]
1209pub struct ModuleStatus {
1210    pub module_id: String,
1211    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1212    pub waiting_for_previous_pid: Option<u32>,
1213    /// Why the previous instance still prevents a new start.
1214    pub waiting_for_previous_detail: Option<String>,
1215    /// The daemon's Windows per-spawn record: `pipe` after the process it
1216    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1217    /// List/status reports use this record, not the source the module sends
1218    /// in its registration message's manifest.provenance.launch_nonce_source.
1219    pub launch_nonce_source: Option<String>,
1220    pub state: ModuleState,
1221    pub enabled: bool,
1222    pub process_alive: bool,
1223    pub registration_active: bool,
1224    /// The module's declared wire protocol, carried beside `live` because it is
1225    /// what makes `live` readable: the two fields answer one question together.
1226    /// While a process is alive this is its launch declaration, not a later
1227    /// pending-reload edit. When down it is the configured next launch protocol.
1228    pub protocol: ModuleProtocol,
1229    /// Whether the module is serving, under the strongest definition the daemon
1230    /// can assert for its protocol.
1231    ///
1232    /// A subc module must also be REGISTERED: its process being alive says
1233    /// nothing about whether it can take a request. A `protocol: "none"` module
1234    /// never registers, so that term is dropped and this falls back to "enabled,
1235    /// running, and the process the daemon launched is alive" -- which is all
1236    /// the daemon observes about a process that speaks no subc wire. It stays a
1237    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1238    /// rather than printing it bare.
1239    pub live: bool,
1240    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1241    /// restarts have already released their slot, so this count can go down
1242    /// without anybody touching the module.
1243    pub restart_count: u32,
1244    /// Replacement processes spawned over this module's entire supervisor lifetime;
1245    /// unlike `restart_count`, this value is never reset by an operator action
1246    /// and never falls out of a window.
1247    pub lifetime_restarts: u32,
1248    pub spawn_generation: u64,
1249    /// The budget `restart_count` is spent against. Carried alongside the count
1250    /// because the count alone does not say how close the module is to being
1251    /// disabled, and reporting one without the other is what makes an
1252    /// about-to-be-retired module look ordinary.
1253    pub max_restarts: u32,
1254    /// The span `restart_count` is counted over. Carried with the pair above for
1255    /// the same reason they are carried together: "2 of 3" means one thing for a
1256    /// ten-minute window and something else entirely for a lifetime.
1257    pub restart_window: Duration,
1258    /// Effective drain and restart timing policy used by this running module.
1259    /// These values are carried together with the restart budget so status
1260    /// readers can compare configured intent with what the supervisor applied.
1261    pub drain_timeout: Duration,
1262    pub restart_backoff: Duration,
1263    pub restart_max_backoff: Duration,
1264    /// The module's process. On macOS this stays absent while the `ck-subc`
1265    /// launch trampoline is still running in that pid, and appears once the
1266    /// exec acknowledgement confirms the module image has replaced it. Launch
1267    /// time and the supervisor's own hold on the process are unaffected.
1268    pub pid: Option<u32>,
1269    pub spawned_at_ms: Option<u64>,
1270    pub spawned_from: Option<PathBuf>,
1271    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1272    pub process_start_time: Option<u64>,
1273    pub last_exit: Option<ExitReport>,
1274    pub health: ModuleHealthStatus,
1275}
1276
1277#[cfg(test)]
1278#[derive(Debug, Clone, Copy, PartialEq)]
1279struct ActorSelectCheckpoint {
1280    generation: u64,
1281    turn: u64,
1282    registered_connection: Option<ConnectionId>,
1283    next_probe_at: Option<Instant>,
1284    wake_after: Option<Duration>,
1285}
1286
1287#[derive(Debug, Clone, PartialEq)]
1288struct SupervisorSnapshot {
1289    subtree_target: crate::subtree_resources::Target,
1290    failed_retry: FailedRetry,
1291    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1292    #[cfg(windows)]
1293    pipe_consumed: Option<Arc<OnceLock<()>>>,
1294    /// Counts running-child loop turns, not executor polls of a parked wait.
1295    #[cfg(test)]
1296    actor_turns: u64,
1297    /// Running select after exec confirmation and with no unconsumed registry
1298    /// event. Recording a spawn or starting a loop turn is not this barrier.
1299    #[cfg(test)]
1300    actor_select: Option<ActorSelectCheckpoint>,
1301    state: ModuleState,
1302    enabled: bool,
1303    process_alive: bool,
1304    spawned_protocol: Option<ModuleProtocol>,
1305    spawn_failure: Option<String>,
1306    waiting_for_previous_pid: Option<u32>,
1307    waiting_for_previous_detail: Option<String>,
1308    /// When each crash restart was spent, oldest first. This IS the crash
1309    /// budget: its in-window length is the count an operator sees and the count
1310    /// the restart decision is made against, so there is no second counter that
1311    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1312    /// operator actions that used to zero the old lifetime counter.
1313    crash_restarts: VecDeque<Instant>,
1314    lifetime_restarts: u32,
1315    /// Successful child spawns in this daemon incarnation.
1316    ///
1317    /// `lifetime_restarts` was considered and rejected: it starts at zero
1318    /// (line 640), successful initial/operator spawns in `set_running` do not
1319    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1320    /// increments before a successful replacement exists (lines 604, 3846,
1321    /// and 3921), so a failed spawn can consume it. This counter moves only
1322    /// when a live PID is accepted below.
1323    spawn_generation: u64,
1324    pid: Option<u32>,
1325    #[cfg(target_os = "macos")]
1326    report_ready: Option<Arc<OnceLock<()>>>,
1327    /// Last reaped child, retained after current process facts are cleared.
1328    reaped_pid: Option<u32>,
1329    /// Whether a replacement has already been scheduled for the current process.
1330    respawn_pending: bool,
1331    /// A second restart is waiting for the replacement already scheduled.
1332    coalesced_restart_pending: bool,
1333    spawned_at_ms: Option<u64>,
1334    spawned_from: Option<PathBuf>,
1335    launch_spec_digest: Option<LaunchSpecDigest>,
1336    spawned_file_identity: Option<SpawnedFileIdentity>,
1337    process_start_time: Option<u64>,
1338    deliberate_severance: Option<ProcessIdentity>,
1339    last_exit: Option<ExitReport>,
1340    /// Diagnostic attached to the next drain's terminal record, if any.
1341    drain_disposition_detail: Option<String>,
1342    health: ModuleHealthStatus,
1343    /// Whether the current process was started as a swap candidate and so
1344    /// lives in the module's alternate cgroup. The next swap's candidate takes
1345    /// the other one, so the two processes of a swap never share a cgroup. A
1346    /// plain spawn always uses the primary cgroup.
1347    in_alternate_slot: bool,
1348    /// Whether the current `Draining` state ends in a replacement process
1349    /// (restart, reload, health restart) rather than a stop. Only meaningful
1350    /// while `state` is `Draining`; every entry into that state rewrites it.
1351    /// It is what lets route.open answer the retryable `module_reloading` to a
1352    /// consumer that reaches a still-registered process mid-restart, instead of
1353    /// the `supervisor_not_live` a stop or disable deserves.
1354    draining_to_replace: bool,
1355    /// Whether a configuration update has been applied since the current
1356    /// process was spawned, so that process runs an older spec than the one
1357    /// the supervisor now holds. A queued restart is only coalesced into a
1358    /// fresher process when this is false: a restart requested to pick up a
1359    /// new configuration must not be satisfied by a process that predates it.
1360    configuration_updated_since_spawn: bool,
1361}
1362
1363impl SupervisorSnapshot {
1364    /// The pid that status, provenance and resource readings may report. While
1365    /// the launch trampoline still runs in the pid, reading its executable or
1366    /// resource use would describe `ck-subc`, not the module, so none is
1367    /// reported until the exec acknowledgement confirms the module image.
1368    fn reported_pid(&self) -> Option<u32> {
1369        #[cfg(target_os = "macos")]
1370        if self
1371            .report_ready
1372            .as_ref()
1373            .is_some_and(|ready| ready.get().is_none())
1374        {
1375            return None;
1376        }
1377        self.pid
1378    }
1379
1380    fn starting() -> Self {
1381        Self::new(ModuleState::Starting, true)
1382    }
1383
1384    fn disabled() -> Self {
1385        Self::new(ModuleState::Disabled, false)
1386    }
1387
1388    /// Crash restarts still inside `window`, having dropped the ones that are
1389    /// not. Pruning on read is what makes the budget a rate: an instant older
1390    /// than the window stops holding a slot the moment anybody counts.
1391    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1392        while let Some(oldest) = self.crash_restarts.front() {
1393            if now.duration_since(*oldest) > window {
1394                self.crash_restarts.pop_front();
1395            } else {
1396                break;
1397            }
1398        }
1399        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1400    }
1401
1402    /// Spend one unit of the crash budget and record the restart in the ledger.
1403    ///
1404    /// The ring is bounded by the cap because more than `max_restarts` in-window
1405    /// instants can never be reached (the caller refuses the restart first), so
1406    /// anything beyond that is an unbounded queue waiting to happen.
1407    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1408        self.crash_restarts.push_back(now);
1409        while self.crash_restarts.len() > policy.max_restarts as usize {
1410            self.crash_restarts.pop_front();
1411        }
1412        self.lifetime_restarts += 1;
1413    }
1414
1415    /// Reserve one crash-restart slot and calculate the delay before respawning.
1416    /// The count is captured before recording this restart, so the first retry
1417    /// uses the base delay and each later in-window retry escalates once.
1418    fn next_crash_restart(
1419        &mut self,
1420        policy: &RestartPolicy,
1421        now: Instant,
1422    ) -> Option<CrashRestartSchedule> {
1423        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1424        if restart_in_window >= policy.max_restarts {
1425            return None;
1426        }
1427        self.record_crash_restart(policy, now);
1428        Some(CrashRestartSchedule {
1429            restart_in_window,
1430            delay: policy.delay_for_restart(restart_in_window),
1431        })
1432    }
1433
1434    /// Give the module its full budget back, as an operator restart, reload, or
1435    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1436    /// ledger of what actually happened, and an operator action does not unmake
1437    /// the crashes.
1438    fn clear_crash_restarts(&mut self) {
1439        self.crash_restarts.clear();
1440    }
1441
1442    fn new(state: ModuleState, enabled: bool) -> Self {
1443        Self {
1444            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1445            failed_retry: FailedRetry::default(),
1446            exec_outcome: None,
1447            #[cfg(test)]
1448            actor_turns: 0,
1449            #[cfg(test)]
1450            actor_select: None,
1451            state,
1452            enabled,
1453            process_alive: false,
1454            #[cfg(windows)]
1455            pipe_consumed: None,
1456            spawned_protocol: None,
1457            spawn_failure: None,
1458            waiting_for_previous_pid: None,
1459            waiting_for_previous_detail: None,
1460            crash_restarts: VecDeque::new(),
1461            lifetime_restarts: 0,
1462            spawn_generation: 0,
1463            pid: None,
1464            #[cfg(target_os = "macos")]
1465            report_ready: None,
1466            reaped_pid: None,
1467            respawn_pending: false,
1468            coalesced_restart_pending: false,
1469            spawned_at_ms: None,
1470            spawned_from: None,
1471            launch_spec_digest: None,
1472            spawned_file_identity: None,
1473            process_start_time: None,
1474            deliberate_severance: None,
1475            last_exit: None,
1476            drain_disposition_detail: None,
1477            health: ModuleHealthStatus::default(),
1478            in_alternate_slot: false,
1479            draining_to_replace: false,
1480            configuration_updated_since_spawn: false,
1481        }
1482    }
1483}
1484
1485type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1486
1487type SpawnSubscriberKey = (ConnectionId, u64);
1488
1489#[derive(Debug)]
1490struct SpawnSubscriber {
1491    version: u8,
1492    frames: mpsc::Sender<Frame>,
1493    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1494    /// from which event. The full frame channel cannot carry that news, so it
1495    /// travels beside it; see `SpawnEventFeed::subscribe`.
1496    lagged: Option<oneshot::Sender<SpawnCursor>>,
1497}
1498
1499#[derive(Debug)]
1500struct SpawnEventState {
1501    daemon_incarnation: String,
1502    seq: u64,
1503    capacity: usize,
1504    live: HashMap<String, LiveSpawn>,
1505    generations: HashMap<String, u64>,
1506    events: VecDeque<SpawnEvent>,
1507    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1508}
1509
1510impl Default for SpawnEventState {
1511    fn default() -> Self {
1512        Self {
1513            daemon_incarnation: "unconfigured".to_string(),
1514            seq: 0,
1515            capacity: SPAWN_EVENT_RING_CAPACITY,
1516            live: HashMap::new(),
1517            generations: HashMap::new(),
1518            events: VecDeque::new(),
1519            subscribers: HashMap::new(),
1520        }
1521    }
1522}
1523
1524#[derive(Debug, Clone, Default)]
1525struct SpawnEventFeed(
1526    Arc<Mutex<SpawnEventState>>,
1527    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1528);
1529
1530#[derive(Debug, Clone, PartialEq, Eq)]
1531pub(crate) enum SpawnSubscribeRefusal {
1532    ForeignIncarnation { current: String },
1533    TooOld { oldest: SpawnCursor },
1534    Frame(String),
1535}
1536
1537impl SpawnEventFeed {
1538    fn configure_incarnation(&self, daemon_incarnation: String) {
1539        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1540        state.daemon_incarnation = daemon_incarnation;
1541        state.seq = 0;
1542        state.live.clear();
1543        state.generations.clear();
1544        state.events.clear();
1545        state.subscribers.clear();
1546    }
1547
1548    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1549        SpawnCursor {
1550            daemon_incarnation: state.daemon_incarnation.clone(),
1551            seq: state.seq,
1552        }
1553    }
1554
1555    fn snapshot(&self) -> SpawnSnapshot {
1556        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1557        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1558        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1559        SpawnSnapshot {
1560            cursor: Self::cursor(&state),
1561            ring_bound: state.capacity as u64,
1562            live,
1563        }
1564    }
1565
1566    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1567        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1568        let generation = state
1569            .generations
1570            .get(module_id)
1571            .copied()
1572            .unwrap_or(0)
1573            .checked_add(1)
1574            .expect("spawn generation exhausted");
1575        state.generations.insert(module_id.to_string(), generation);
1576        let live = LiveSpawn {
1577            module_id: module_id.to_string(),
1578            spawn_generation: generation,
1579            pid,
1580            spawned_at_ms,
1581        };
1582        state.live.insert(module_id.to_string(), live);
1583        Self::emit_locked(
1584            &mut state,
1585            SpawnEventKind::Spawned,
1586            module_id.to_string(),
1587            generation,
1588            pid,
1589            None,
1590            None,
1591        );
1592        generation
1593    }
1594
1595    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1596        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1597        let Some(live) = state.live.remove(module_id) else {
1598            warn!(
1599                module_id,
1600                "terminal record had no live spawn event identity"
1601            );
1602            return;
1603        };
1604        Self::emit_locked(
1605            &mut state,
1606            SpawnEventKind::Exited,
1607            module_id.to_string(),
1608            live.spawn_generation,
1609            live.pid,
1610            exit_code,
1611            exit_signal,
1612        );
1613    }
1614
1615    /// Report the exit of a process that a swap has already replaced.
1616    ///
1617    /// `emit_exited` removes the module's live entry, which after a swap's
1618    /// cutover describes the promoted candidate, not the old process now
1619    /// exiting. This emits the old generation's exit and leaves the live entry
1620    /// alone unless it still names that generation.
1621    fn emit_superseded_exited(
1622        &self,
1623        module_id: &str,
1624        spawn_generation: u64,
1625        pid: u32,
1626        exit_code: Option<i32>,
1627        exit_signal: Option<i32>,
1628    ) {
1629        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1630        if state
1631            .live
1632            .get(module_id)
1633            .is_some_and(|live| live.spawn_generation == spawn_generation)
1634        {
1635            state.live.remove(module_id);
1636        }
1637        Self::emit_locked(
1638            &mut state,
1639            SpawnEventKind::Exited,
1640            module_id.to_string(),
1641            spawn_generation,
1642            pid,
1643            exit_code,
1644            exit_signal,
1645        );
1646    }
1647
1648    #[allow(clippy::too_many_arguments)]
1649    fn emit_locked(
1650        state: &mut SpawnEventState,
1651        kind: SpawnEventKind,
1652        module_id: String,
1653        spawn_generation: u64,
1654        pid: u32,
1655        exit_code: Option<i32>,
1656        exit_signal: Option<i32>,
1657    ) {
1658        state.seq = state
1659            .seq
1660            .checked_add(1)
1661            .expect("spawn event sequence exhausted");
1662        let event = SpawnEvent {
1663            cursor: Self::cursor(state),
1664            kind,
1665            module_id,
1666            spawn_generation,
1667            pid,
1668            exit_code,
1669            exit_signal,
1670        };
1671        state.events.push_back(event.clone());
1672        while state.events.len() > state.capacity {
1673            state.events.pop_front();
1674        }
1675        let body = match serde_json::to_vec(&event) {
1676            Ok(body) => body,
1677            Err(error) => {
1678                error!(%error, "failed to serialize supervisor spawn event");
1679                return;
1680            }
1681        };
1682        state.subscribers.retain(|(connection_id, corr), subscriber| {
1683            let frame = Frame::build_with_version(
1684                subscriber.version,
1685                FrameType::StreamData,
1686                control_flags(),
1687                0,
1688                0,
1689                *corr,
1690                body.clone(),
1691            );
1692            match frame {
1693                Ok(frame) => {
1694                    if subscriber.frames.try_send(frame).is_ok() {
1695                        true
1696                    } else {
1697                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1698                        if let Some(lagged) = subscriber.lagged.take() {
1699                            let _ = lagged.send(event.cursor.clone());
1700                        }
1701                        false
1702                    }
1703                }
1704                Err(error) => {
1705                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1706                    false
1707                }
1708            }
1709        });
1710    }
1711
1712    fn subscribe(
1713        &self,
1714        connection_id: ConnectionId,
1715        corr: u64,
1716        version: u8,
1717        since: Option<SpawnCursor>,
1718        sink: FrameSink,
1719    ) -> Result<(), SpawnSubscribeRefusal> {
1720        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1721        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1722        {
1723            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1724            let replay = if let Some(since) = since {
1725                if since.daemon_incarnation != state.daemon_incarnation {
1726                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1727                        current: state.daemon_incarnation.clone(),
1728                    });
1729                }
1730                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1731                    if since.seq < oldest.seq.saturating_sub(1) {
1732                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1733                    }
1734                }
1735                state
1736                    .events
1737                    .iter()
1738                    .filter(|event| event.cursor.seq > since.seq)
1739                    .cloned()
1740                    .collect::<Vec<_>>()
1741            } else {
1742                Vec::new()
1743            };
1744            for event in replay {
1745                let body = serde_json::to_vec(&event)
1746                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1747                let frame = Frame::build_with_version(
1748                    version,
1749                    FrameType::StreamData,
1750                    control_flags(),
1751                    0,
1752                    0,
1753                    corr,
1754                    body,
1755                )
1756                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1757                frames
1758                    .try_send(frame)
1759                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1760            }
1761            state.subscribers.insert(
1762                (connection_id, corr),
1763                SpawnSubscriber {
1764                    version,
1765                    frames,
1766                    lagged: Some(lagged),
1767                },
1768            );
1769        }
1770        // The lagged terminal is sent here, by the forwarder, rather than by
1771        // the emitter: at the moment of the drop the subscriber's own channel
1772        // is full, and writing to the connection sink directly from the emitter
1773        // would put the Error AHEAD of the events still queued in that channel
1774        // (and the emitter holds the feed lock, so it cannot await the sink).
1775        // Dropping the subscriber drops the only sender, so `recv` drains every
1776        // queued event and then returns `None`; only then is the Error sent, so
1777        // the client sees each event it can keep, then the reason it was cut.
1778        // Cancel and connection removal drop the oneshot unsent, so they end
1779        // the stream with no Error.
1780        tokio::spawn(async move {
1781            while let Some(frame) = receiver.recv().await {
1782                if sink.send(frame).await.is_err() {
1783                    return;
1784                }
1785            }
1786            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1787                return;
1788            };
1789            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1790                Ok(frame) => {
1791                    let _ = sink.send(frame).await;
1792                }
1793                Err(error) => {
1794                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1795                }
1796            }
1797        });
1798        Ok(())
1799    }
1800
1801    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1802        let Some(subscriber) = self
1803            .0
1804            .lock()
1805            .unwrap_or_else(|p| p.into_inner())
1806            .subscribers
1807            .remove(&(connection_id, corr))
1808        else {
1809            return false;
1810        };
1811        if let Ok(frame) = Frame::build_with_version(
1812            subscriber.version,
1813            FrameType::StreamEnd,
1814            control_flags(),
1815            0,
1816            0,
1817            corr,
1818            Vec::new(),
1819        ) {
1820            tokio::spawn(async move {
1821                let _ = subscriber.frames.send(frame).await;
1822            });
1823        }
1824        true
1825    }
1826
1827    fn remove_connection(&self, connection_id: ConnectionId) {
1828        self.0
1829            .lock()
1830            .unwrap_or_else(|p| p.into_inner())
1831            .subscribers
1832            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1833    }
1834
1835    #[cfg(any(test, feature = "test-support"))]
1836    fn set_capacity(&self, capacity: usize) {
1837        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1838    }
1839
1840    #[cfg(any(test, feature = "test-support"))]
1841    fn subscriber_count(&self) -> usize {
1842        self.0
1843            .lock()
1844            .unwrap_or_else(|p| p.into_inner())
1845            .subscribers
1846            .len()
1847    }
1848}
1849
1850/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1851/// The terminal Error a lagged spawn subscriber receives after its queued events.
1852fn spawn_subscriber_lagged_frame(
1853    version: u8,
1854    corr: u64,
1855    first_undelivered: SpawnCursor,
1856) -> Result<Frame, String> {
1857    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1858    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1859        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1860        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1861            .to_string(),
1862        detail: Some(serde_json::json!({
1863            "first_undelivered_cursor": first_undelivered
1864        })),
1865    })
1866    .map_err(|error| error.to_string())?;
1867    Frame::build_with_version(
1868        version,
1869        FrameType::Error,
1870        crate::router::daemon_error_flags(),
1871        0,
1872        0,
1873        corr,
1874        body,
1875    )
1876    .map_err(|error| error.to_string())
1877}
1878
1879pub trait ModuleProcessLiveness: Send + Sync {
1880    fn process_live(&self, module_id: &str) -> Option<bool>;
1881
1882    /// Whether the supervisor is replacing this module's process right now: an
1883    /// operator restart or reload, a health restart, or a crash respawn whose
1884    /// backoff is running. A module in that state is not live, but a consumer
1885    /// refused now should retry shortly rather than treat the target as gone.
1886    /// Stopped, failed, and disabled modules are not replacing.
1887    fn process_replacing(&self, _module_id: &str) -> bool {
1888        false
1889    }
1890}
1891
1892/// Shared process-liveness registry keyed by supervised `module_id`.
1893#[derive(Debug, Clone, Default)]
1894pub struct SupervisorProcessLiveness {
1895    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1896}
1897
1898impl SupervisorProcessLiveness {
1899    pub fn new() -> Self {
1900        Self::default()
1901    }
1902
1903    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1904        let mut snapshots = self
1905            .snapshots
1906            .lock()
1907            .unwrap_or_else(|poisoned| poisoned.into_inner());
1908        snapshots.insert(module_id, snapshot);
1909    }
1910
1911    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1912        let mut snapshots = self
1913            .snapshots
1914            .lock()
1915            .unwrap_or_else(|poisoned| poisoned.into_inner());
1916        let is_current = snapshots
1917            .get(module_id)
1918            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1919            .unwrap_or(false);
1920        if is_current {
1921            snapshots.remove(module_id);
1922        }
1923    }
1924}
1925
1926impl ModuleProcessLiveness for SupervisorProcessLiveness {
1927    fn process_live(&self, module_id: &str) -> Option<bool> {
1928        let snapshot = {
1929            let snapshots = self
1930                .snapshots
1931                .lock()
1932                .unwrap_or_else(|poisoned| poisoned.into_inner());
1933            snapshots.get(module_id).cloned()
1934        }?;
1935        let snapshot = snapshot
1936            .lock()
1937            .unwrap_or_else(|poisoned| poisoned.into_inner());
1938        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1939    }
1940
1941    fn process_replacing(&self, module_id: &str) -> bool {
1942        let Some(snapshot) = self
1943            .snapshots
1944            .lock()
1945            .unwrap_or_else(|poisoned| poisoned.into_inner())
1946            .get(module_id)
1947            .cloned()
1948        else {
1949            return false;
1950        };
1951        let snapshot = snapshot
1952            .lock()
1953            .unwrap_or_else(|poisoned| poisoned.into_inner());
1954        snapshot.enabled
1955            && match snapshot.state {
1956                ModuleState::Restarting => true,
1957                ModuleState::Draining => snapshot.draining_to_replace,
1958                ModuleState::Starting
1959                | ModuleState::Running
1960                | ModuleState::Unresponsive
1961                | ModuleState::Stopped
1962                | ModuleState::Failed
1963                | ModuleState::Disabled => false,
1964            }
1965    }
1966}
1967
1968#[cfg(test)]
1969#[derive(Debug, Default)]
1970struct ReloadExitRecordGate {
1971    reached: tokio::sync::Notify,
1972    resume: tokio::sync::Notify,
1973}
1974
1975#[derive(Debug, Clone, Copy)]
1976enum RespawnKind {
1977    Spawn,
1978    Reload,
1979}
1980
1981#[derive(Debug, Clone, Copy)]
1982struct PendingRespawn {
1983    deadline: Instant,
1984    kind: RespawnKind,
1985    daemon_initiated: bool,
1986}
1987
1988type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1989
1990#[derive(Debug, Clone)]
1991struct SupervisorRuntimeConfig {
1992    registry: Arc<Registry>,
1993    /// Retained across an operation's deferred backoff, until its child starts.
1994    recovery_is_daemon: bool,
1995    start_requester: Option<Principal>,
1996    pending_activity_restart: bool,
1997    provenance_probe: ExecutableIdentityProbe,
1998    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1999    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2000    /// A reload acknowledges completion only after its replacement registers.
2001    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2002    restart_policy: RestartPolicy,
2003    register_timeout: Duration,
2004    /// This module's RESOLVED drain budget: per-module config when present,
2005    /// else `default_drain_timeout`.
2006    drain_timeout: Duration,
2007    /// Shared with the status handle so the attested value changes atomically
2008    /// when a rescan updates the running drain policy.
2009    effective_drain_timeout: Arc<Mutex<Duration>>,
2010    /// The supervisor-wide fallback, kept so a configuration update that
2011    /// REMOVES the per-module override can re-resolve to it.
2012    default_drain_timeout: Duration,
2013    health: HealthConfig,
2014    connection_file_path: Option<PathBuf>,
2015    capture_logs_dir: Option<PathBuf>,
2016    forwarding: Option<Arc<ForwardingTable>>,
2017    /// The shared handle, so every spawn path (initial, restart, reload) records the
2018    /// reserved-module launch nonce the HELLO verifier checks against.
2019    supervisor_handle: Option<SupervisorHandle>,
2020    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2021    /// status queries.
2022    ///
2023    /// One ring per module, held across every respawn. The lines explaining an exit
2024    /// are written BEFORE that exit, so a ring recreated per process would be empty
2025    /// exactly when it is asked for.
2026    stderr_ring: Arc<Mutex<StderrRing>>,
2027    terminal_ring: Arc<Mutex<TerminalRing>>,
2028    spawn_events: SpawnEventFeed,
2029    child_roster: ChildRoster,
2030    #[cfg(target_os = "linux")]
2031    cgroup_placement: Option<subc_cgroup::Placement>,
2032    #[cfg(test)]
2033    test_seed_stale_facts_before_enable_spawn: bool,
2034    #[cfg(test)]
2035    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2036}
2037
2038impl SupervisorRuntimeConfig {
2039    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2040        FailureContext {
2041            daemon_initiated,
2042            shutdown: self.child_roster.shutdown_flag(),
2043        }
2044    }
2045}
2046
2047#[derive(Debug, Clone, PartialEq, Eq)]
2048struct SupervisedConfiguration {
2049    spec: ModuleSpec,
2050    health: HealthConfig,
2051}
2052
2053/// Shared daemon lookup table for supervised module handles.
2054///
2055/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2056/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2057/// launch nonces recorded at spawn are checked by the same daemon instance.
2058#[derive(Debug, Clone, Default)]
2059pub struct SupervisorHandle {
2060    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2061    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2062    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2063    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2064    /// Module ids the supervisor has taken on. An id is added BEFORE the
2065    /// module's first process is spawned and removed only when the module
2066    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2067    /// the keys of `modules`.
2068    ///
2069    /// `modules` cannot answer "is this module configured?" on its own: a
2070    /// [`SupervisedModule`] only exists once its process has been spawned, and
2071    /// a fast child can connect, register, sync its scopes and ask about them
2072    /// before the supervisor has inserted it. Answering "not configured" in that
2073    /// gap makes scope admission refuse with the terminal "will never sync"
2074    /// instead of the retryable "has not synced yet".
2075    configured_ids: Arc<Mutex<HashSet<String>>>,
2076    spawn_events: SpawnEventFeed,
2077    /// The current expected launch nonce for each reserved module_id. Set when the
2078    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2079    /// non-reserved module never has an entry here and is never nonce-checked.
2080    /// Reserved module ids and the nonce that authorizes their next HELLO.
2081    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2082    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2083    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2084    /// had NO entry and admitted anyone: the reservation protected the nonce
2085    /// holder, not the NAME (found live by CKCRED's canary probe registering
2086    /// against a reserved scratch id).
2087    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2088    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2089    ///
2090    /// This is deliberately in-memory only: subc is state-free across daemon
2091    /// restarts, and the tombstone only explains the hours-after-removal window
2092    /// while this executing daemon is still alive. Do not persist it in a store.
2093    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2094    /// The current launch nonce for every supervised spawn. This is separate from
2095    /// reserved_nonces because consumer route.open attestation applies to all spawned
2096    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2097    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2098    /// Reserved namespace prefixes mapped to the supervised owner module whose
2099    /// current spawn nonce authorizes HELLO claims below the prefix.
2100    ///
2101    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2102    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2103    /// accidental collisions and lower-trust processes from squatting protected
2104    /// namespaces.
2105    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2106    /// Blue/green swaps in progress, by module id. An entry exists from just
2107    /// before the candidate process is spawned until the swap has failed, or
2108    /// has cut over and the old process is gone. While it exists, HELLO for the
2109    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2110    /// consumer attestation accepts both processes' nonces.
2111    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2112    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2113    promotion_observer: PromotionObserverSlot,
2114    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2115    /// this daemon-wide ordering, a rescan could retire or update a module while a
2116    /// concurrent reload still held its old handle and launch specification.
2117    operation_lock: Arc<AsyncMutex<()>>,
2118}
2119
2120/// Told when a swap has promoted its candidate to be the module's active
2121/// registration.
2122///
2123/// An ordinary HELLO runs the control plane's registration side effects (the
2124/// capability cache, the deny census, the requirement recompute) as it
2125/// registers. A swap candidate's HELLO does not, because it is not routable;
2126/// promotion is when those must run instead, and promotion happens in the
2127/// supervisor, which has no other way into the control handler.
2128pub(crate) trait SwapPromotionObserver: Send + Sync {
2129    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2130}
2131
2132/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2133/// control handler) owns this handle, so a strong reference back would be a
2134/// cycle that keeps both alive.
2135#[derive(Clone, Default)]
2136struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2137
2138impl fmt::Debug for PromotionObserverSlot {
2139    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2140        f.write_str("PromotionObserverSlot")
2141    }
2142}
2143
2144/// The nonces of one open swap.
2145#[derive(Debug, Clone)]
2146struct OpenSwap {
2147    /// The launch nonce minted for the candidate process. It is the swap
2148    /// token: the only thing that admits a HELLO into the candidate slot.
2149    candidate_nonce: String,
2150    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2151    /// here because cutover moves the module's recorded spawn nonce to the
2152    /// candidate while the incumbent is still draining and its consumers are
2153    /// still attesting with this one.
2154    incumbent_nonce: Option<String>,
2155    /// Set once a HELLO has been admitted with the swap token, so the token
2156    /// admits one registration and cannot be replayed after cutover empties
2157    /// the candidate slot.
2158    candidate_admitted: bool,
2159}
2160
2161/// What the swap gate says about a HELLO. See
2162/// [`SupervisorHandle::swap_hello_admission`].
2163#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2164pub(crate) enum SwapHelloAdmission {
2165    /// No swap is open for the id (or the HELLO carries the incumbent's own
2166    /// nonce); the ordinary gates decide.
2167    NotSwapping,
2168    /// The HELLO carries the swap token: register it into the candidate slot.
2169    Candidate,
2170    /// A swap is open and the HELLO carries a nonce the supervisor did not
2171    /// mint for this id, no nonce, or a token already used.
2172    Refused,
2173}
2174
2175#[derive(Debug, Clone, PartialEq, Eq)]
2176pub(crate) enum ReservedHelloRejection {
2177    Exact {
2178        module_id: String,
2179    },
2180    Prefix {
2181        prefix: String,
2182        owner_module_id: String,
2183    },
2184}
2185
2186#[derive(Debug)]
2187struct DaemonRestartIntent {
2188    expires: Option<Duration>,
2189    window: Duration,
2190}
2191
2192impl Default for DaemonRestartIntent {
2193    fn default() -> Self {
2194        Self {
2195            expires: None,
2196            window: Duration::from_secs(120),
2197        }
2198    }
2199}
2200
2201impl SupervisorHandle {
2202    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2203        let mut intent = self
2204            .restart_intent
2205            .lock()
2206            .unwrap_or_else(|p| p.into_inner());
2207        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2208        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2209    }
2210
2211    #[cfg(feature = "test-support")]
2212    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2213        self.restart_intent
2214            .lock()
2215            .unwrap_or_else(|p| p.into_inner())
2216            .window = window;
2217    }
2218
2219    #[cfg(unix)]
2220    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2221        let expiry = self
2222            .restart_intent
2223            .lock()
2224            .unwrap_or_else(|p| p.into_inner())
2225            .expires
2226            .take();
2227        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2228            RouteCloseReason::Restart
2229        } else {
2230            RouteCloseReason::Disable
2231        }
2232    }
2233
2234    pub fn new() -> Self {
2235        Self::default()
2236    }
2237
2238    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2239        self.spawn_events.snapshot()
2240    }
2241
2242    pub(crate) fn subscribe_spawns(
2243        &self,
2244        connection_id: ConnectionId,
2245        corr: u64,
2246        version: u8,
2247        since: Option<SpawnCursor>,
2248        sink: FrameSink,
2249    ) -> Result<(), SpawnSubscribeRefusal> {
2250        self.spawn_events
2251            .subscribe(connection_id, corr, version, since, sink)
2252    }
2253
2254    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2255        self.spawn_events.cancel(connection_id, corr)
2256    }
2257
2258    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2259        self.spawn_events.remove_connection(connection_id);
2260    }
2261
2262    #[cfg(any(test, feature = "test-support"))]
2263    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2264        assert!(capacity > 0, "spawn event capacity must be non-zero");
2265        self.spawn_events.set_capacity(capacity);
2266    }
2267
2268    #[cfg(any(test, feature = "test-support"))]
2269    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2270        self.spawn_events.subscriber_count()
2271    }
2272
2273    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2274    /// a respawn invalidates stale consumer identities.
2275    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2276        self.spawn_nonces
2277            .lock()
2278            .unwrap_or_else(|poisoned| poisoned.into_inner())
2279            .insert(module_id.to_string(), nonce);
2280    }
2281
2282    /// Record the launch nonce expected from the next HELLO for a reserved module,
2283    /// replacing any prior nonce (a respawn invalidates the previous one).
2284    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2285        self.reserved_nonces
2286            .lock()
2287            .unwrap_or_else(|poisoned| poisoned.into_inner())
2288            .insert(module_id.to_string(), Some(nonce));
2289    }
2290
2291    /// Record namespace prefixes owned by a supervised module.
2292    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2293        let mut owners = self
2294            .reserved_prefix_owners
2295            .lock()
2296            .unwrap_or_else(|poisoned| poisoned.into_inner());
2297        owners.retain(|_, owner| owner != owner_module_id);
2298        for prefix in prefixes {
2299            owners.insert(prefix.clone(), owner_module_id.to_string());
2300        }
2301    }
2302
2303    /// The launch nonce most recently minted for a module's spawn, if any.
2304    #[cfg(test)]
2305    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2306        self.spawn_nonces
2307            .lock()
2308            .unwrap_or_else(|poisoned| poisoned.into_inner())
2309            .get(module_id)
2310            .cloned()
2311    }
2312
2313    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2314        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2315        let spawn_nonce = self
2316            .spawn_nonces
2317            .lock()
2318            .unwrap_or_else(|poisoned| poisoned.into_inner())
2319            .get(&spec.module_id)
2320            .cloned();
2321        let mut reserved_nonces = self
2322            .reserved_nonces
2323            .lock()
2324            .unwrap_or_else(|poisoned| poisoned.into_inner());
2325        if spec.reserved {
2326            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2327            // reserved name whose module has never spawned has no legitimate
2328            // holder, and the entry's absence is what used to leave the name
2329            // open to the first claimant.
2330            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2331        }
2332        drop(reserved_nonces);
2333        // A later unreserved declaration must not silently unreserve an id that
2334        // was retained after its reserved configuration was removed. The explicit
2335        // release ceremony is the only operation that retires that gate.
2336        self.removal_tombstones
2337            .lock()
2338            .unwrap_or_else(|poisoned| poisoned.into_inner())
2339            .remove(&spec.module_id);
2340    }
2341
2342    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2343    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2344    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2345    /// with no matching prefix are always authorized.
2346    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2347        self.reserved_hello_verification(module_id, presented)
2348            .is_ok()
2349    }
2350
2351    /// Whether HELLO passed a nonce check that was actually required by reserved
2352    /// module policy, or the refusal explaining why a required check failed.
2353    pub(crate) fn reserved_hello_verification(
2354        &self,
2355        module_id: &str,
2356        presented: Option<&str>,
2357    ) -> Result<bool, ReservedHelloRejection> {
2358        let nonces = self
2359            .reserved_nonces
2360            .lock()
2361            .unwrap_or_else(|poisoned| poisoned.into_inner());
2362        if let Some(expected) = nonces.get(module_id) {
2363            // `None` = reserved with no legitimate holder: refuse every
2364            // presentation, because no process can hold a nonce that was never
2365            // minted. Only a real minted nonce admits, in constant time.
2366            let authorized = match expected {
2367                Some(expected) => {
2368                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2369                }
2370                None => false,
2371            };
2372            if authorized {
2373                return Ok(true);
2374            }
2375            return Err(ReservedHelloRejection::Exact {
2376                module_id: module_id.to_string(),
2377            });
2378        }
2379        drop(nonces);
2380
2381        let matched_prefix = self
2382            .reserved_prefix_owners
2383            .lock()
2384            .unwrap_or_else(|poisoned| poisoned.into_inner())
2385            .iter()
2386            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2387            .max_by_key(|(prefix, _)| prefix.len())
2388            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2389        let Some((prefix, owner_module_id)) = matched_prefix else {
2390            return Ok(false);
2391        };
2392
2393        let authorized = presented.is_some_and(|presented| {
2394            self.spawn_nonces
2395                .lock()
2396                .unwrap_or_else(|poisoned| poisoned.into_inner())
2397                .get(&owner_module_id)
2398                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2399                // While the owner is being swapped, children started by
2400                // either of its two processes hold that process's nonce.
2401                || self.swap_nonce_matches(&owner_module_id, presented)
2402        });
2403        if authorized {
2404            Ok(true)
2405        } else {
2406            Err(ReservedHelloRejection::Prefix {
2407                prefix,
2408                owner_module_id,
2409            })
2410        }
2411    }
2412
2413    /// Whether a consumer connection proved it came from a daemon-spawned module.
2414    ///
2415    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2416    /// accepted only for module ids the supervisor has spawned.
2417    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2418        if presented.is_empty() {
2419            return false;
2420        }
2421        let nonces = self
2422            .spawn_nonces
2423            .lock()
2424            .unwrap_or_else(|poisoned| poisoned.into_inner());
2425        let current = nonces
2426            .get(module_id)
2427            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2428        drop(nonces);
2429        // During a swap two processes of the module are alive, and a consumer
2430        // started by either one presents that process's nonce. Accepting only
2431        // the recorded one would fail the incumbent's consumers for the whole
2432        // overlap once cutover moves the record to the candidate.
2433        current || self.swap_nonce_matches(module_id, presented)
2434    }
2435
2436    /// Whether `presented` is either nonce of an open swap for `module_id`.
2437    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2438        let swaps = self
2439            .swaps
2440            .lock()
2441            .unwrap_or_else(|poisoned| poisoned.into_inner());
2442        swaps.get(module_id).is_some_and(|swap| {
2443            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2444                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2445                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2446                })
2447        })
2448    }
2449
2450    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2451    /// Called before the candidate process exists.
2452    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2453        let incumbent_nonce = self
2454            .spawn_nonces
2455            .lock()
2456            .unwrap_or_else(|poisoned| poisoned.into_inner())
2457            .get(module_id)
2458            .cloned();
2459        self.swaps
2460            .lock()
2461            .unwrap_or_else(|poisoned| poisoned.into_inner())
2462            .insert(
2463                module_id.to_string(),
2464                OpenSwap {
2465                    candidate_nonce,
2466                    incumbent_nonce,
2467                    candidate_admitted: false,
2468                },
2469            );
2470    }
2471
2472    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2473    /// the module's recorded one.
2474    pub(crate) fn close_swap(&self, module_id: &str) {
2475        self.swaps
2476            .lock()
2477            .unwrap_or_else(|poisoned| poisoned.into_inner())
2478            .remove(module_id);
2479    }
2480
2481    /// Install the observer told about swap promotions, replacing any earlier
2482    /// one.
2483    pub(crate) fn set_swap_promotion_observer(
2484        &self,
2485        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2486    ) {
2487        *self
2488            .promotion_observer
2489            .0
2490            .lock()
2491            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2492    }
2493
2494    /// Tell the installed observer, if it is still alive, that a swap promoted
2495    /// `registration`.
2496    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2497        let observer = self
2498            .promotion_observer
2499            .0
2500            .lock()
2501            .unwrap_or_else(|poisoned| poisoned.into_inner())
2502            .as_ref()
2503            .and_then(std::sync::Weak::upgrade);
2504        if let Some(observer) = observer {
2505            observer.swap_promoted(registration);
2506        }
2507    }
2508
2509    /// Whether a swap is open for `module_id`.
2510    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2511        self.swaps
2512            .lock()
2513            .unwrap_or_else(|poisoned| poisoned.into_inner())
2514            .contains_key(module_id)
2515    }
2516
2517    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2518    /// respawn would, once cutover has made the candidate the module's process.
2519    /// The swap stays open so the incumbent's nonce keeps attesting until the
2520    /// incumbent has drained and exited.
2521    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2522        let candidate_nonce = self
2523            .swaps
2524            .lock()
2525            .unwrap_or_else(|poisoned| poisoned.into_inner())
2526            .get(module_id)
2527            .map(|swap| swap.candidate_nonce.clone());
2528        let Some(nonce) = candidate_nonce else {
2529            return;
2530        };
2531        self.set_spawn_nonce(module_id, nonce.clone());
2532        if reserved {
2533            self.set_reserved_nonce(module_id, nonce);
2534        }
2535    }
2536
2537    /// The swap gate for a HELLO claiming `module_id`.
2538    ///
2539    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2540    /// presents the candidate nonce, which the reserved gate (holding the
2541    /// incumbent's nonce) would refuse as `reserved_module` before swap
2542    /// admission was ever reached. And it applies to unreserved ids too: for an
2543    /// unreserved id the only thing that ever stopped a second process claiming
2544    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2545    /// refusal a swap lifts for its candidate.
2546    ///
2547    /// The incumbent's own nonce falls through to the ordinary gates, which
2548    /// treat it as they always have (a live incumbent is refused as a
2549    /// duplicate). Anything else while a swap is open is refused, including an
2550    /// absent nonce.
2551    pub(crate) fn swap_hello_admission(
2552        &self,
2553        module_id: &str,
2554        presented: Option<&str>,
2555    ) -> SwapHelloAdmission {
2556        let swaps = self
2557            .swaps
2558            .lock()
2559            .unwrap_or_else(|poisoned| poisoned.into_inner());
2560        let Some(swap) = swaps.get(module_id) else {
2561            return SwapHelloAdmission::NotSwapping;
2562        };
2563        let Some(presented) = presented else {
2564            return SwapHelloAdmission::Refused;
2565        };
2566        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2567            return if swap.candidate_admitted {
2568                SwapHelloAdmission::Refused
2569            } else {
2570                SwapHelloAdmission::Candidate
2571            };
2572        }
2573        if swap
2574            .incumbent_nonce
2575            .as_deref()
2576            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2577        {
2578            return SwapHelloAdmission::NotSwapping;
2579        }
2580        SwapHelloAdmission::Refused
2581    }
2582
2583    /// Record that the swap token has registered a candidate, so it admits no
2584    /// second HELLO.
2585    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2586        if let Some(swap) = self
2587            .swaps
2588            .lock()
2589            .unwrap_or_else(|poisoned| poisoned.into_inner())
2590            .get_mut(module_id)
2591        {
2592            swap.candidate_admitted = true;
2593        }
2594    }
2595
2596    /// Test/support lookup for the current launch nonce of a supervised spawn.
2597    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2598        self.spawn_nonces
2599            .lock()
2600            .unwrap_or_else(|poisoned| poisoned.into_inner())
2601            .get(module_id)
2602            .cloned()
2603    }
2604
2605    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2606    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2607        self.reserved_nonces
2608            .lock()
2609            .unwrap_or_else(|poisoned| poisoned.into_inner())
2610            .get(module_id)
2611            .cloned()
2612            .flatten()
2613    }
2614
2615    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2616        // Normally already marked before the process was spawned; marking here
2617        // too keeps `configured_ids` a superset of the roster for any caller
2618        // that inserts a module directly.
2619        self.mark_configured(module.module_id());
2620        let mut modules = self
2621            .modules
2622            .lock()
2623            .unwrap_or_else(|poisoned| poisoned.into_inner());
2624        modules.insert(module.module_id().to_string(), module)
2625    }
2626
2627    /// Record that the supervisor has taken on `module_id`. Called before the
2628    /// module's first process is spawned, so that by the time that process can
2629    /// register, [`Self::is_configured`] already answers true.
2630    fn mark_configured(&self, module_id: &str) {
2631        self.configured_ids
2632            .lock()
2633            .unwrap_or_else(|poisoned| poisoned.into_inner())
2634            .insert(module_id.to_string());
2635    }
2636
2637    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2638    /// before it was ever put on the roster. A module already on the roster
2639    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2640    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2641        let modules = self
2642            .modules
2643            .lock()
2644            .unwrap_or_else(|poisoned| poisoned.into_inner());
2645        if !modules.contains_key(module_id) {
2646            self.configured_ids
2647                .lock()
2648                .unwrap_or_else(|poisoned| poisoned.into_inner())
2649                .remove(module_id);
2650        }
2651    }
2652
2653    /// Whether `module_id` is a module this daemon supervises: on the roster,
2654    /// or about to be (its process is being spawned right now).
2655    ///
2656    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2657    /// for scopes: a supervised module's process can register and sync before
2658    /// [`Self::get`] can return it, and in that window it is still a module
2659    /// that will sync, not one that never will.
2660    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2661        self.configured_ids
2662            .lock()
2663            .unwrap_or_else(|poisoned| poisoned.into_inner())
2664            .contains(module_id)
2665    }
2666
2667    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2668        let modules = self
2669            .modules
2670            .lock()
2671            .unwrap_or_else(|poisoned| poisoned.into_inner());
2672        modules.get(module_id).cloned()
2673    }
2674
2675    pub(crate) fn record_late_health_answer(
2676        &self,
2677        module_id: &str,
2678        latency_ms: u64,
2679    ) -> Result<bool, SuperviseError> {
2680        let Some(module) = self.get(module_id) else {
2681            return Ok(false);
2682        };
2683        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2684            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2685            state.health.last_late_answer_latency_ms = Some(latency_ms);
2686            // A late answer is an answer: the module served the probe, just past
2687            // the deadline. Leaving the miss streak in place while logging
2688            // "proves the module is alive" is how a CPU-starved module that
2689            // answers every probe a few seconds late still marches to the
2690            // threshold and gets killed — the exact kill class `NoAnswer` is
2691            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2692            // is degradation, and degradation reports; it does not restart.
2693            state.health.consecutive_failures = 0;
2694        })?;
2695        Ok(true)
2696    }
2697
2698    /// Arm the one-shot marker for the module process that this caller
2699    /// deliberately initiated severance against. Generic connection teardown
2700    /// must not call this:
2701    /// a surviving process would otherwise retain an exemption for a later
2702    /// genuine crash.
2703    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2704        let Some(module) = self.get(module_id) else {
2705            return Ok(false);
2706        };
2707        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2708        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2709            return Ok(false);
2710        };
2711        drop(snapshot);
2712        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2713    }
2714
2715    pub fn list(&self) -> Vec<SupervisedModule> {
2716        let modules = self
2717            .modules
2718            .lock()
2719            .unwrap_or_else(|poisoned| poisoned.into_inner());
2720        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2721        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2722        modules
2723    }
2724
2725    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2726        self.activity_images
2727            .lock()
2728            .unwrap_or_else(|p| p.into_inner())
2729            .remove(module_id);
2730        self.spawn_nonces
2731            .lock()
2732            .unwrap_or_else(|poisoned| poisoned.into_inner())
2733            .remove(module_id);
2734        self.close_swap(module_id);
2735        let mut reserved_nonces = self
2736            .reserved_nonces
2737            .lock()
2738            .unwrap_or_else(|poisoned| poisoned.into_inner());
2739        if reserved_nonces.contains_key(module_id) {
2740            // The old nonce must die with the removed process, but the exact-id
2741            // gate remains until an operator explicitly releases it.
2742            reserved_nonces.insert(module_id.to_string(), None);
2743        }
2744        drop(reserved_nonces);
2745        self.reserved_prefix_owners
2746            .lock()
2747            .unwrap_or_else(|poisoned| poisoned.into_inner())
2748            .retain(|_, owner| owner != module_id);
2749        let removed = self
2750            .modules
2751            .lock()
2752            .unwrap_or_else(|poisoned| poisoned.into_inner())
2753            .remove(module_id);
2754        self.configured_ids
2755            .lock()
2756            .unwrap_or_else(|poisoned| poisoned.into_inner())
2757            .remove(module_id);
2758        removed
2759    }
2760
2761    /// Remember a module removed by a non-preview rescan so route.open can
2762    /// distinguish that intentional removal from an unknown id.
2763    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2764        self.removal_tombstones
2765            .lock()
2766            .unwrap_or_else(|poisoned| poisoned.into_inner())
2767            .insert(module_id.to_string(), unix_ms_now());
2768    }
2769
2770    /// Return how long ago a rescan removed this module in milliseconds.
2771    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2772        self.removal_tombstones
2773            .lock()
2774            .unwrap_or_else(|poisoned| poisoned.into_inner())
2775            .get(module_id)
2776            .copied()
2777            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2778    }
2779
2780    /// Retire a reserved-id gate only after its module has left supervision.
2781    ///
2782    /// A retained gate has no live nonce (`None`), so releasing any other entry
2783    /// would weaken a currently configured or otherwise active reservation.
2784    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2785        if self.get(module_id).is_some() {
2786            return false;
2787        }
2788        let mut reserved_nonces = self
2789            .reserved_nonces
2790            .lock()
2791            .unwrap_or_else(|poisoned| poisoned.into_inner());
2792        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2793            return false;
2794        }
2795        reserved_nonces.remove(module_id);
2796        true
2797    }
2798
2799    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2800        Arc::clone(&self.operation_lock)
2801    }
2802}
2803
2804/// Process supervisor for subc-owned singleton modules.
2805#[derive(Debug, Clone)]
2806pub struct Supervisor {
2807    registry: Arc<Registry>,
2808    restart_policy: RestartPolicy,
2809    drain_timeout: Duration,
2810    connection_file_path: Option<PathBuf>,
2811    capture_logs_dir: Option<PathBuf>,
2812    forwarding: Option<Arc<ForwardingTable>>,
2813    process_liveness: Arc<SupervisorProcessLiveness>,
2814    supervisor_handle: Option<SupervisorHandle>,
2815    health: HealthConfig,
2816    daemon_start_clock: crate::clock::StartClock,
2817    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2818    spawn_events: SpawnEventFeed,
2819    provenance_probe: ExecutableIdentityProbe,
2820    /// Every process spawned through this supervisor (and its clones) and not
2821    /// yet reaped, so daemon shutdown can end them.
2822    child_roster: ChildRoster,
2823    #[cfg(target_os = "linux")]
2824    cgroup_placement: Option<subc_cgroup::Placement>,
2825    #[cfg(test)]
2826    test_after_first_spawn: AfterFirstSpawnHook,
2827}
2828
2829/// A roster entry whose monitor and first process have not started yet.
2830/// Keeping the command receiver here lets discovery serve status immediately
2831/// while start/stop requests wait until bootstrap has published the connection file.
2832pub(crate) struct RegisteredConfiguredModule {
2833    spec: ModuleSpec,
2834    runtime: SupervisorRuntimeConfig,
2835    module: SupervisedModule,
2836    commands: mpsc::Receiver<SupervisorCommand>,
2837}
2838
2839/// Test-only hook run on the path that takes on a new module, right after its
2840/// first `spawn_child` returns (the process exists and could already be
2841/// registering) and before that process is handed to the module's supervise
2842/// loop and put on the roster. Lets a test observe what a fast child would see
2843/// in that window without racing a real one.
2844#[cfg(test)]
2845#[derive(Clone, Default)]
2846struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2847
2848#[cfg(test)]
2849type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2850
2851#[cfg(test)]
2852impl fmt::Debug for AfterFirstSpawnHook {
2853    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2854        f.write_str("AfterFirstSpawnHook")
2855    }
2856}
2857
2858#[cfg(test)]
2859impl AfterFirstSpawnHook {
2860    fn run(&self, module_id: &str) {
2861        if let Some(hook) = &self.0 {
2862            hook(module_id);
2863        }
2864    }
2865}
2866
2867impl Supervisor {
2868    #[cfg(test)]
2869    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2870        let supervisor = Self::new(registry, policy);
2871        #[cfg(target_os = "macos")]
2872        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2873        supervisor
2874    }
2875    /// Verify the trampoline once when configured. Missing private OS support
2876    /// refuses every macOS launch by name but does not stop the daemon's control
2877    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2878    /// entry point; the library must not exec an arbitrary hosting program.
2879    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2880        let path = path.into();
2881        #[cfg(target_os = "macos")]
2882        {
2883            let result = probe_privacy_trampoline(&path).map(|()| path);
2884            if let Err(cause) = &result {
2885                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2886            }
2887            self.child_roster.set_privacy_trampoline(result);
2888        }
2889        #[cfg(not(target_os = "macos"))]
2890        let _ = path;
2891        self
2892    }
2893    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2894    /// capability-requirement log to read.
2895    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2896        self.child_roster.shutdown_flag()
2897    }
2898
2899    /// The first step of an announced daemon shutdown, before the notice and
2900    /// before any connection is closed.
2901    ///
2902    /// Sets the daemon-shutdown flag first: from here on no module is
2903    /// respawned (crash restart, operator restart, or swap), and every child
2904    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2905    /// the module exits on the EOF this shutdown gives it or is signalled by a
2906    /// service manager that kills the whole cgroup. Then writes the journal's
2907    /// shutdown marker, which records the instant and closes this daemon
2908    /// incarnation's stretch of the journal.
2909    #[cfg(unix)]
2910    pub(crate) fn begin_daemon_shutdown(&self) {
2911        self.child_roster.close();
2912        if let Some(journal) = &self.terminal_journal {
2913            journal.stamp_shutdown();
2914        }
2915    }
2916
2917    /// Announce a cut while established connections can still carry replies.
2918    /// These budgets bound the attempt to deliver notices and the wait for
2919    /// routes to drain; neither guarantees delivery or child completion, and
2920    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2921    #[cfg(unix)]
2922    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2923        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2924        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2925        let Some(forwarding) = &self.forwarding else {
2926            return Ok(());
2927        };
2928        let notice_started = Instant::now();
2929        let modules = forwarding
2930            .begin_daemon_drain()
2931            .map_err(SuperviseError::Forwarding)?;
2932        let deadline_ms =
2933            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2934        let shutdown_reason = self
2935            .supervisor_handle
2936            .as_ref()
2937            .map(SupervisorHandle::take_daemon_shutdown_reason)
2938            .unwrap_or(RouteCloseReason::Disable);
2939        let mut modules_told = 0;
2940        let mut notices = tokio::task::JoinSet::new();
2941        let mut drains = Vec::new();
2942        for connection in forwarding
2943            .module_connections()
2944            .map_err(SuperviseError::Forwarding)?
2945        {
2946            let target = ModuleDrainTarget {
2947                endpoint: connection.endpoint,
2948                sink: connection.sink,
2949                negotiated_ver: connection.negotiated_ver,
2950                abandoned_bindings: Vec::new(),
2951                excluded_subscriptions: 0,
2952            };
2953            notices.spawn(async move {
2954                (
2955                    true,
2956                    send_module_draining(
2957                        &connection.module_id,
2958                        shutdown_reason,
2959                        deadline_ms,
2960                        &target,
2961                        true,
2962                    )
2963                    .await,
2964                )
2965            });
2966        }
2967        for (module_id, endpoint) in modules {
2968            let Some(target) = forwarding
2969                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2970                .map_err(SuperviseError::Forwarding)?
2971            else {
2972                continue;
2973            };
2974            let routes = forwarding
2975                .endpoint_routes(target.endpoint)
2976                .map_err(SuperviseError::Forwarding)?;
2977            // Only an announced restart promises that modules are coming back.
2978            // Client route notices retain their existing reconnect semantics.
2979            let mut recipients = Vec::new();
2980            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2981            for route in routes {
2982                let client = route.goodbye_target;
2983                if let Some((_, channels)) = clients
2984                    .iter_mut()
2985                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2986                {
2987                    channels.push(client.channel);
2988                } else {
2989                    let channel = client.channel;
2990                    clients.push((client, vec![channel]));
2991                }
2992            }
2993            for (client, mut channels) in clients {
2994                channels.sort_unstable();
2995                channels.dedup();
2996                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2997                    module_id: module_id.clone(),
2998                    channels,
2999                    reason: RouteCloseReason::Restart,
3000                })
3001                .expect("route closing serializes");
3002                recipients.push((client.sink, client.negotiated_ver, closing));
3003            }
3004            for (sink, version, body) in recipients {
3005                notices.spawn(async move {
3006                    let frame = Frame::build_with_version(
3007                        version,
3008                        FrameType::Push,
3009                        control_flags(),
3010                        0,
3011                        0,
3012                        0,
3013                        body,
3014                    )
3015                    .expect("bounded lifecycle notice frame builds");
3016                    (false, sink.send_flushed(frame).await.is_ok())
3017                });
3018            }
3019            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3020            drains.push((module_id, target.endpoint, gauges));
3021        }
3022        // A quiet forwarding table is not proof that queued notices reached the
3023        // socket. Wait for writer flush acknowledgements before testing quiescence.
3024        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3025        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3026            match result {
3027                Ok((true, true)) => modules_told += 1,
3028                Ok((false, true)) => {}
3029                result => warn!(?result, "daemon shutdown notice delivery failed"),
3030            }
3031        }
3032        notices.abort_all();
3033        info!(
3034            ?shutdown_reason,
3035            modules_told, "daemon shutdown module notices delivered"
3036        );
3037        let notice_elapsed = notice_started.elapsed();
3038        let drain_started = Instant::now();
3039        let deadline = drain_started + DRAIN_BUDGET;
3040        let mut waits = tokio::task::JoinSet::new();
3041        for (module_id, endpoint, gauges) in drains {
3042            let forwarding = Arc::clone(forwarding);
3043            let mut runtime = self.runtime_config();
3044            runtime.health.cadence = Duration::from_millis(100);
3045            waits.spawn(async move {
3046                let wait_result = wait_for_forwarding_quiescence(
3047                    &forwarding,
3048                    &module_id,
3049                    &runtime,
3050                    endpoint,
3051                    deadline,
3052                    &gauges,
3053                    DrainScope::Endpoint(endpoint),
3054                    &mut DrainGaugeTrace::new(&gauges),
3055                )
3056                .await;
3057                // Store the measurement before the task returns: quiescence
3058                // may be reached just as the deadline stops the join loop from
3059                // collecting results. Unfinished or failed waits report false.
3060                forwarding
3061                    .record_daemon_drain_result(
3062                        endpoint,
3063                        drained_after_quiescence_wait(&wait_result),
3064                    )
3065                    .map_err(SuperviseError::Forwarding)?;
3066                wait_result
3067            });
3068        }
3069        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3070            if !matches!(result, Ok(Ok(true))) {
3071                warn!(?result, "daemon shutdown drain did not reach quiescence");
3072            }
3073        }
3074        // Measure only the forwarding drain, not child teardown or the caller's
3075        // signal delivery and process-reaping delays.
3076        info!(
3077            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3078            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3079            "daemon shutdown forwarding drain complete"
3080        );
3081        Ok(())
3082    }
3083
3084    /// The last step of a graceful daemon shutdown, after the notice and the
3085    /// drain: send every registered module a module GOODBYE, the same planned
3086    /// stop signal `ck module stop` gives, then close every connection so each
3087    /// subc module sees EOF and starts its own teardown, then end every
3088    /// supervised child that has not exited
3089    /// by its own deadline (its drain budget, capped). Modules lead their own
3090    /// process groups, so a
3091    /// service manager's group kill no longer reaches them; without this a
3092    /// child that does not stop on EOF (every `protocol: "none"` child, which
3093    /// has no connection) would outlive the daemon. Every wait is bounded (see
3094    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3095    #[cfg(unix)]
3096    pub(crate) async fn end_children_for_daemon_shutdown(
3097        &self,
3098        already_escalated: bool,
3099        escalate: impl std::future::Future<Output = ()>,
3100    ) {
3101        tokio::pin!(escalate);
3102        let mut escalated = already_escalated;
3103        if let Some(forwarding) = &self.forwarding {
3104            let reason = CloseReason::new(
3105                "daemon_shutdown",
3106                "the daemon is exiting after its shutdown notice and drain",
3107            );
3108            if escalated {
3109                // The operator asked to stop waiting: queue the GOODBYEs but
3110                // do not wait for them to be written.
3111                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3112            } else {
3113                tokio::select! {
3114                    biased;
3115                    _ = escalate.as_mut() => {
3116                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3117                        escalated = true;
3118                    }
3119                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3120                }
3121            }
3122            let closed = forwarding.close_all_connections(&reason);
3123            debug!(closed, "closed established connections for daemon shutdown");
3124        }
3125        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3126        // already completed and must not be polled again; the child shutdown
3127        // wait is told it is escalated and gets a future that never fires.
3128        let escalated_here = escalated && !already_escalated;
3129        let remaining_escalate = async move {
3130            if escalated_here {
3131                std::future::pending::<()>().await;
3132            } else {
3133                escalate.await;
3134            }
3135        };
3136        crate::child_roster::end_children_for_daemon_shutdown(
3137            &self.child_roster,
3138            escalated,
3139            remaining_escalate,
3140        )
3141        .await;
3142    }
3143
3144    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3145        Self {
3146            registry,
3147            restart_policy,
3148            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3149            connection_file_path: None,
3150            capture_logs_dir: None,
3151            forwarding: None,
3152            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3153            supervisor_handle: None,
3154            health: HealthConfig::default(),
3155            daemon_start_clock: crate::clock::StartClock::capture(),
3156            terminal_journal: None,
3157            spawn_events: SpawnEventFeed::default(),
3158            provenance_probe: ExecutableIdentityProbe::default(),
3159            child_roster: ChildRoster::default(),
3160            #[cfg(target_os = "linux")]
3161            cgroup_placement: None,
3162            #[cfg(test)]
3163            test_after_first_spawn: AfterFirstSpawnHook::default(),
3164        }
3165    }
3166
3167    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3168        self.drain_timeout = drain_timeout;
3169        self
3170    }
3171
3172    /// The drain budget a module gets when its config entry sets none.
3173    pub(crate) fn default_drain_timeout(&self) -> Duration {
3174        self.drain_timeout
3175    }
3176
3177    pub fn with_process_liveness(
3178        mut self,
3179        process_liveness: Arc<SupervisorProcessLiveness>,
3180    ) -> Self {
3181        self.process_liveness = process_liveness;
3182        self
3183    }
3184
3185    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3186        self.connection_file_path = Some(connection_file_path.into());
3187        self
3188    }
3189
3190    /// Enables daemon-owned capture files for supervised stdout and stderr.
3191    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3192        self.capture_logs_dir = Some(logs_dir.into());
3193        self
3194    }
3195
3196    /// Names this daemon lifetime in spawn events, independently of whether a
3197    /// terminal journal is configured.
3198    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3199        // A millisecond start stamp can repeat after clock rollback or a rapid
3200        // restart. Use the connection file's random daemon_id instead: it already
3201        // identifies this daemon lifetime independently of the wall clock.
3202        self.spawn_events.configure_incarnation(daemon_incarnation);
3203        self
3204    }
3205
3206    /// Enables best-effort history shared by every supervised module. Without
3207    /// it, terminal history is kept only in each module's in-memory ring.
3208    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3209        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3210        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3211            path,
3212            daemon_incarnation,
3213        )));
3214        this
3215    }
3216
3217    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3218        self.child_roster.counters = forwarding.counters();
3219        self.forwarding = Some(forwarding);
3220        self
3221    }
3222
3223    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3224        self.spawn_events = supervisor_handle.spawn_events.clone();
3225        self.supervisor_handle = Some(supervisor_handle);
3226        self
3227    }
3228
3229    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3230        self.health = health;
3231        self
3232    }
3233
3234    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3235    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3236    /// record is kept.
3237    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3238        self.child_roster.record_to(path.into());
3239        self
3240    }
3241
3242    #[cfg(target_os = "linux")]
3243    pub fn with_cgroup_placement(
3244        mut self,
3245        cgroup_placement: Option<subc_cgroup::Placement>,
3246    ) -> Self {
3247        self.child_roster
3248            .set_cgroup_placement(cgroup_placement.clone());
3249        self.cgroup_placement = cgroup_placement;
3250        self
3251    }
3252
3253    /// Spawn `spec.program` and start monitoring it.
3254    ///
3255    /// The child is expected to parse `--subc <connection-file-path>`, read the
3256    /// TCP+key connection file, authenticate to the already-running listener, and
3257    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3258    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3259        validate_spec(&spec)?;
3260        self.establish_identity(&spec);
3261
3262        let runtime = self.runtime_config();
3263        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3264        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3265        let registration_watch = watch_registration(&self.registry, &spec);
3266        let spawned = spawn_child(
3267            &spec,
3268            runtime.connection_file_path.as_deref(),
3269            self.supervisor_handle.as_ref(),
3270            &runtime.stderr_ring,
3271            runtime.capture_logs_dir.as_deref(),
3272            &runtime.child_roster,
3273            #[cfg(target_os = "linux")]
3274            runtime.cgroup_placement.as_ref(),
3275        );
3276        #[cfg(test)]
3277        self.test_after_first_spawn.run(&spec.module_id);
3278        let mut child = match spawned {
3279            Ok(child) => child,
3280            Err(err) => {
3281                // Unlike the configured paths, a failed `spawn` leaves nothing
3282                // on the roster, so the module must not stay marked configured.
3283                self.abandon_unrostered(&spec.module_id);
3284                return Err(err);
3285            }
3286        };
3287        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3288        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3289        if let Some(spawn) = activity_spawn {
3290            spawn.complete(&snapshot);
3291        }
3292
3293        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3294    }
3295
3296    /// Make `spec`'s module count as configured, with its identity gates
3297    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3298    /// exists.
3299    ///
3300    /// Every path that takes on a new module calls this before `spawn_child`.
3301    /// The order is the point: the child can connect, register, sync its
3302    /// scopes and ask about them as soon as it is spawned, and the module is
3303    /// only put on the roster after `spawn_child` returns. Were the mark set
3304    /// with the roster entry, a fast child would see its own owner reported
3305    /// as not configured, and a scoped `route.open` in that window would be
3306    /// refused as terminal `scope_not_live` ("will never sync") instead of
3307    /// retryable `scope_not_synced`.
3308    fn establish_identity(&self, spec: &ModuleSpec) {
3309        if let Some(supervisor_handle) = &self.supervisor_handle {
3310            supervisor_handle.apply_identity_configuration(spec);
3311            supervisor_handle.mark_configured(&spec.module_id);
3312        }
3313    }
3314
3315    /// Take back [`Self::establish_identity`]'s configured mark when the
3316    /// module will not be put on the roster after all.
3317    fn abandon_unrostered(&self, module_id: &str) {
3318        if let Some(supervisor_handle) = &self.supervisor_handle {
3319            supervisor_handle.unmark_configured_unless_rostered(module_id);
3320        }
3321    }
3322
3323    /// Record a freshly spawned first process as running. On failure the
3324    /// module never reaches the roster, so its configured mark is taken back.
3325    fn mark_first_process_running(
3326        &self,
3327        spec: &ModuleSpec,
3328        runtime: &SupervisorRuntimeConfig,
3329        snapshot: &SharedSnapshot,
3330        child: &mut SupervisedChild,
3331    ) -> Result<(), SuperviseError> {
3332        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3333            containment::terminate_blocking(&runtime.child_roster.containment);
3334            self.abandon_unrostered(&spec.module_id);
3335            return Err(err);
3336        }
3337        self.process_liveness
3338            .track(spec.module_id.clone(), Arc::clone(snapshot));
3339        Ok(())
3340    }
3341
3342    /// Start supervising a module declared in daemon configuration.
3343    ///
3344    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3345    /// failures in the supervisor handle so operator-facing `supervisor.list`
3346    /// reflects every configured module while daemon startup continues.
3347    pub fn supervise_configured(
3348        &self,
3349        spec: ModuleSpec,
3350        enabled: bool,
3351    ) -> Result<SupervisedModule, SuperviseError> {
3352        self.supervise_configured_requested(
3353            spec,
3354            enabled,
3355            self.health.clone(),
3356            None,
3357            self.restart_policy,
3358            DEFAULT_REGISTER_TIMEOUT,
3359            None,
3360        )
3361    }
3362
3363    /// Supervise a configured module with its own health, drain, and crash
3364    /// budget. The restart policy is per-module because the config file is:
3365    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3366    /// module that is expensive to restart should not be forced onto the same
3367    /// budget as one that is cheap.
3368    pub fn supervise_configured_with_health(
3369        &self,
3370        spec: ModuleSpec,
3371        enabled: bool,
3372        health: HealthConfig,
3373        drain_timeout_ms: Option<u64>,
3374        restart_policy: RestartPolicy,
3375    ) -> Result<SupervisedModule, SuperviseError> {
3376        self.supervise_configured_requested(
3377            spec,
3378            enabled,
3379            health,
3380            drain_timeout_ms,
3381            restart_policy,
3382            DEFAULT_REGISTER_TIMEOUT,
3383            None,
3384        )
3385    }
3386
3387    #[allow(clippy::too_many_arguments)]
3388    pub(crate) fn supervise_configured_requested(
3389        &self,
3390        spec: ModuleSpec,
3391        enabled: bool,
3392        health: HealthConfig,
3393        drain_timeout_ms: Option<u64>,
3394        restart_policy: RestartPolicy,
3395        register_timeout: Duration,
3396        requester: Option<Principal>,
3397    ) -> Result<SupervisedModule, SuperviseError> {
3398        let registered = self.register_configured_with_health(
3399            spec,
3400            enabled,
3401            health,
3402            drain_timeout_ms,
3403            restart_policy,
3404            register_timeout,
3405        )?;
3406        let daemon_initiated = requester.is_none();
3407        self.start_registered_configured(registered, requester, daemon_initiated)
3408    }
3409
3410    /// Record a configured module as starting or disabled, without launching a
3411    /// process or running its monitor. Bootstrap registers the entire roster
3412    /// before publishing discovery, then activates it once children can read
3413    /// the connection file passed in their `--subc` argument.
3414    pub(crate) fn register_configured_with_health(
3415        &self,
3416        spec: ModuleSpec,
3417        enabled: bool,
3418        health: HealthConfig,
3419        drain_timeout_ms: Option<u64>,
3420        restart_policy: RestartPolicy,
3421        register_timeout: Duration,
3422    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3423        validate_spec(&spec)?;
3424        self.establish_identity(&spec);
3425
3426        let mut runtime = self.runtime_config();
3427        runtime.health = health;
3428        runtime.restart_policy = restart_policy;
3429        runtime.register_timeout = register_timeout;
3430        if let Some(ms) = drain_timeout_ms {
3431            runtime.drain_timeout = Duration::from_millis(ms);
3432            *runtime
3433                .effective_drain_timeout
3434                .lock()
3435                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3436        }
3437        let snapshot = if enabled {
3438            SupervisorSnapshot::starting()
3439        } else {
3440            SupervisorSnapshot::disabled()
3441        };
3442        let snapshot = {
3443            let mut snapshot = snapshot;
3444            if enabled {
3445                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3446                {
3447                    snapshot.waiting_for_previous_pid = Some(pid);
3448                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3449                }
3450            }
3451            snapshot
3452        };
3453        let (module, commands) =
3454            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3455        Ok(RegisteredConfiguredModule {
3456            spec,
3457            runtime,
3458            module,
3459            commands,
3460        })
3461    }
3462
3463    /// Activate a previously registered module. Queued control requests are
3464    /// served by the monitor after the initial configured spawn has settled.
3465    pub(crate) fn start_registered_configured(
3466        &self,
3467        registered: RegisteredConfiguredModule,
3468        requester: Option<Principal>,
3469        daemon_initiated: bool,
3470    ) -> Result<SupervisedModule, SuperviseError> {
3471        let RegisteredConfiguredModule {
3472            spec,
3473            mut runtime,
3474            module,
3475            commands,
3476        } = registered;
3477        runtime.start_requester = requester;
3478        runtime.recovery_is_daemon = daemon_initiated;
3479        let snapshot = &module.inner.snapshot;
3480        let should_spawn = {
3481            let state = lock_snapshot(snapshot)?;
3482            state.enabled && state.waiting_for_previous_pid.is_none()
3483        };
3484        let child = if should_spawn {
3485            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3486            let registration_watch = watch_registration(&self.registry, &spec);
3487            let spawned = spawn_child(
3488                &spec,
3489                runtime.connection_file_path.as_deref(),
3490                self.supervisor_handle.as_ref(),
3491                &runtime.stderr_ring,
3492                runtime.capture_logs_dir.as_deref(),
3493                &runtime.child_roster,
3494                #[cfg(target_os = "linux")]
3495                runtime.cgroup_placement.as_ref(),
3496            );
3497            #[cfg(test)]
3498            self.test_after_first_spawn.run(&spec.module_id);
3499            match spawned {
3500                Ok(mut child) => {
3501                    arm_registration_deadline(
3502                        &mut child,
3503                        runtime.register_timeout,
3504                        registration_watch,
3505                    );
3506                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3507                    if let Some(spawn) = activity_spawn {
3508                        spawn.complete(snapshot);
3509                    }
3510                    Some(child)
3511                }
3512                Err(err) => {
3513                    if runtime.health.critical {
3514                        error!(
3515                            module_id = %spec.module_id,
3516                            program = %spec.program.display(),
3517                            error = %err,
3518                            "critical configured module failed to spawn; marking failed and alerting"
3519                        );
3520                    } else {
3521                        error!(
3522                            module_id = %spec.module_id,
3523                            program = %spec.program.display(),
3524                            error = %err,
3525                            "configured module failed to spawn; marking failed and continuing"
3526                        );
3527                    }
3528                    fail_snapshot(
3529                        snapshot,
3530                        Some(&spec.module_id),
3531                        None,
3532                        &runtime.failure_context(daemon_initiated),
3533                    );
3534                    None
3535                }
3536            }
3537        } else {
3538            None
3539        };
3540        self.start_monitor(spec, runtime, &module, child, commands);
3541        Ok(module)
3542    }
3543
3544    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3545        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3546        SupervisorRuntimeConfig {
3547            registry: self.registry.clone(),
3548            recovery_is_daemon: true,
3549            start_requester: None,
3550            pending_activity_restart: false,
3551            provenance_probe: self.provenance_probe.clone(),
3552            scheduled_respawn: Arc::default(),
3553            deferred_reload_reply: Arc::default(),
3554            restart_policy: self.restart_policy,
3555            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3556            drain_timeout: self.drain_timeout,
3557            // Shared with this module's roster copy: daemon shutdown waits on
3558            // each child for the module's own drain budget, as resolved now.
3559            child_roster: self
3560                .child_roster
3561                .for_module(Arc::clone(&effective_drain_timeout)),
3562            effective_drain_timeout,
3563            default_drain_timeout: self.drain_timeout,
3564            health: self.health.clone(),
3565            connection_file_path: self.connection_file_path.clone(),
3566            capture_logs_dir: self.capture_logs_dir.clone(),
3567            forwarding: self.forwarding.clone(),
3568            supervisor_handle: self.supervisor_handle.clone(),
3569            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3570            terminal_ring: Arc::new(Mutex::new(
3571                TerminalRing::new(
3572                    TerminalRingConfig::default(),
3573                    self.daemon_start_clock.started_at_ms(),
3574                )
3575                .with_start_clock(self.daemon_start_clock)
3576                .with_journal(self.terminal_journal.clone())
3577                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3578            )),
3579            spawn_events: self.spawn_events.clone(),
3580            #[cfg(target_os = "linux")]
3581            cgroup_placement: self.cgroup_placement.clone(),
3582            #[cfg(test)]
3583            test_seed_stale_facts_before_enable_spawn: false,
3584            #[cfg(test)]
3585            test_reload_exit_record_gate: None,
3586        }
3587    }
3588
3589    fn supervised_module(
3590        &self,
3591        spec: ModuleSpec,
3592        runtime: SupervisorRuntimeConfig,
3593        snapshot: SharedSnapshot,
3594        child: Option<SupervisedChild>,
3595    ) -> SupervisedModule {
3596        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3597        self.start_monitor(spec, runtime, &module, child, commands);
3598        module
3599    }
3600
3601    fn register_module(
3602        &self,
3603        spec: &ModuleSpec,
3604        runtime: &SupervisorRuntimeConfig,
3605        snapshot: SharedSnapshot,
3606    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3607        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3608            spec: spec.clone(),
3609            health: runtime.health.clone(),
3610        }));
3611        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3612        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3613        // The module's OWN policy, which may be its per-module config rather than
3614        // the supervisor-wide one; status must report the budget the supervise
3615        // loop actually enforces.
3616        let restart_policy = runtime.restart_policy;
3617        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3618        let (tx, rx) = mpsc::channel(4);
3619        let module_id = spec.module_id.clone();
3620        let module = SupervisedModule {
3621            inner: Arc::new(SupervisedModuleInner {
3622                module_id: module_id.clone(),
3623                registry: Arc::clone(&self.registry),
3624                snapshot,
3625                configuration,
3626                stderr_ring,
3627                terminal_ring,
3628                commands: tx,
3629                monitor: Mutex::new(None),
3630                restart_policy,
3631                effective_drain_timeout,
3632                provenance_probe: self.provenance_probe.clone(),
3633                retained: runtime.child_roster.retained_view(),
3634            }),
3635        };
3636        // Identity gates and the configured mark must precede the roster entry,
3637        // so a client never sees a module without its admission policy.
3638        if let Some(supervisor_handle) = &self.supervisor_handle {
3639            supervisor_handle.insert(module.clone());
3640        }
3641        (module, rx)
3642    }
3643
3644    fn start_monitor(
3645        &self,
3646        spec: ModuleSpec,
3647        runtime: SupervisorRuntimeConfig,
3648        module: &SupervisedModule,
3649        child: Option<SupervisedChild>,
3650        commands: mpsc::Receiver<SupervisorCommand>,
3651    ) {
3652        let monitor = tokio::spawn(supervise_loop(
3653            spec,
3654            runtime,
3655            Arc::clone(&self.registry),
3656            Arc::clone(&self.process_liveness),
3657            Arc::clone(&module.inner.snapshot),
3658            child,
3659            commands,
3660        ));
3661        *module
3662            .inner
3663            .monitor
3664            .lock()
3665            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3666    }
3667}
3668
3669impl Default for Supervisor {
3670    fn default() -> Self {
3671        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3672    }
3673}
3674
3675/// Handle to one supervised child process.
3676#[derive(Clone)]
3677pub struct SupervisedModule {
3678    inner: Arc<SupervisedModuleInner>,
3679}
3680
3681struct SupervisedModuleInner {
3682    retained: crate::child_roster::RetainedView,
3683    module_id: String,
3684    registry: Arc<Registry>,
3685    snapshot: SharedSnapshot,
3686    configuration: Arc<Mutex<SupervisedConfiguration>>,
3687    stderr_ring: Arc<Mutex<StderrRing>>,
3688    terminal_ring: Arc<Mutex<TerminalRing>>,
3689    commands: mpsc::Sender<SupervisorCommand>,
3690    monitor: Mutex<Option<JoinHandle<()>>>,
3691    /// Copied from the supervisor's runtime config at spawn so `status()` can
3692    /// report the restart budget without reaching back into the supervisor. The
3693    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3694    restart_policy: RestartPolicy,
3695    effective_drain_timeout: Arc<Mutex<Duration>>,
3696    provenance_probe: ExecutableIdentityProbe,
3697}
3698
3699impl fmt::Debug for SupervisedModule {
3700    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3701        f.debug_struct("SupervisedModule")
3702            .field("module_id", &self.inner.module_id)
3703            .field("status", &self.status())
3704            .finish_non_exhaustive()
3705    }
3706}
3707
3708impl SupervisedModule {
3709    pub fn module_id(&self) -> &str {
3710        &self.inner.module_id
3711    }
3712
3713    /// Test-only: put one probe miss on the streak, the way
3714    /// `handle_health_probe_failure` does, so tests can assert what a later
3715    /// event does to the streak without driving the whole probe loop.
3716    #[cfg(test)]
3717    pub(crate) fn record_health_probe_failure_for_test(
3718        &self,
3719        detail: &str,
3720    ) -> Result<(), SuperviseError> {
3721        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3722            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3723            state.health.detail = Some(detail.to_string());
3724        })
3725    }
3726
3727    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3728        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3729    }
3730
3731    pub(crate) fn registration_end_reason(
3732        &self,
3733    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3734        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3735        Ok(match snapshot.state {
3736            ModuleState::Draining if snapshot.draining_to_replace => {
3737                Some(RegistrationEndReason::SupervisorRestart)
3738            }
3739            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3740            ModuleState::Restarting if snapshot.draining_to_replace => {
3741                Some(RegistrationEndReason::SupervisorRestart)
3742            }
3743            _ => None,
3744        })
3745    }
3746
3747    /// The module's retained stderr, newest lines last.
3748    ///
3749    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3750    /// module, `supervisor.list` renders every module, and putting it in the
3751    /// shared snapshot would make each status read carry a payload almost nobody
3752    /// asked for. Callers that want the text ask for it.
3753    pub fn stderr_tail(
3754        &self,
3755        max_lines: Option<usize>,
3756        max_bytes: Option<usize>,
3757    ) -> StderrTailSnapshot {
3758        self.inner
3759            .stderr_ring
3760            .lock()
3761            .unwrap_or_else(|poisoned| poisoned.into_inner())
3762            .snapshot(max_lines, max_bytes)
3763    }
3764
3765    /// The module's bounded terminal history, oldest retained exit first.
3766    ///
3767    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3768    /// daemon whose in-memory history was necessarily reset.
3769    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3770        self.inner
3771            .terminal_ring
3772            .lock()
3773            .unwrap_or_else(|poisoned| poisoned.into_inner())
3774            .snapshot()
3775    }
3776
3777    /// Retained observations from the current ring and all journal generations.
3778    ///
3779    /// Blocking: this reads the journal files. Async callers use
3780    /// `Self::read_durable_terminal_history`.
3781    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3782        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3783    }
3784
3785    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3786    /// read (up to every retained generation) never occupies a runtime worker.
3787    /// Fails only if the blocking task could not finish (runtime shutdown or a
3788    /// panic in the read).
3789    pub(crate) async fn read_durable_terminal_history(
3790        &self,
3791    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3792        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3793        let module_id = self.inner.module_id.clone();
3794        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3795            .await
3796    }
3797
3798    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3799        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3800            .map(|(status, _)| status)
3801    }
3802
3803    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3804        let state = lock_snapshot(&self.inner.snapshot)?;
3805        Ok(state
3806            .failed_retry
3807            .pending
3808            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3809            .map(|retry| {
3810                unix_ms_now().saturating_add(
3811                    u64::try_from(
3812                        retry
3813                            .deadline
3814                            .saturating_duration_since(Instant::now())
3815                            .as_millis(),
3816                    )
3817                    .unwrap_or(u64::MAX),
3818                )
3819            }))
3820    }
3821
3822    pub(crate) fn record_deliberate_severance(
3823        &self,
3824        identity: ProcessIdentity,
3825    ) -> Result<bool, SuperviseError> {
3826        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3827        if snapshot.pid != Some(identity.pid)
3828            || snapshot.process_start_time != Some(identity.start_time)
3829        {
3830            return Ok(false);
3831        }
3832        snapshot.deliberate_severance = Some(identity);
3833        Ok(true)
3834    }
3835
3836    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3837    ///
3838    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3839    /// does not produce reader-observability logs.
3840    pub(crate) fn status_for_control(
3841        &self,
3842        caller: &'static str,
3843    ) -> Result<ModuleStatus, SuperviseError> {
3844        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3845            .map(|(status, _)| status)
3846    }
3847
3848    fn status_with_snapshot_lock(
3849        &self,
3850        snapshot: &SharedSnapshot,
3851        caller: Option<&'static str>,
3852    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3853        let mut guard = match caller {
3854            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3855            None => lock_snapshot(snapshot)?,
3856        };
3857        // Read the budget through the pruning path so a reader sees the same
3858        // in-window count the restart decision would use, not a stale total.
3859        let restart_count =
3860            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3861        let snapshot = guard.clone();
3862        drop(guard);
3863        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3864            SuperviseError::StatePoisoned {
3865                module_id: Some(self.inner.module_id.clone()),
3866            }
3867        })?;
3868        let registration_active = self
3869            .inner
3870            .registry
3871            .get_module(&self.inner.module_id)
3872            .map_err(SuperviseError::Registry)?
3873            .is_some();
3874        let protocol = snapshot
3875            .spawned_protocol
3876            .unwrap_or(self.declared_protocol()?);
3877        let running_process =
3878            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3879        // Registration is the difference between the two protocols and the only
3880        // one: a subc module that has not registered cannot serve a request even
3881        // though its process is up, and a `none` module never registers at all,
3882        // so requiring it there would pin `live` to false for the whole life of
3883        // a perfectly healthy process.
3884        let live = match protocol {
3885            ModuleProtocol::Subc => running_process && registration_active,
3886            ModuleProtocol::None => running_process,
3887        };
3888        // Read only the retained cache: a status read must not resend signals,
3889        // rewrite the record, or keep the containment resource alive.
3890        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3891        let carry_configured = self
3892            .inner
3893            .configuration
3894            .lock()
3895            .map_err(|_| SuperviseError::StatePoisoned {
3896                module_id: Some(self.inner.module_id.clone()),
3897            })?
3898            .spec
3899            .carry_containment;
3900        let recovery_block = carries_containment(
3901            carry_configured,
3902            self.inner
3903                .retained
3904                .has_carried_container(&self.inner.module_id),
3905        )
3906        .then(|| self.inner.retained.recovery_block())
3907        .flatten();
3908        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3909        let pid = snapshot.reported_pid();
3910        let mut health = snapshot.health;
3911        if let Some(detail) = recovery_block
3912            .as_ref()
3913            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3914        {
3915            health.detail = Some(match health.detail {
3916                Some(previous) => format!("{previous}; {detail}"),
3917                None => detail.clone(),
3918            });
3919        }
3920
3921        Ok((
3922            ModuleStatus {
3923                waiting_for_previous_pid: retained
3924                    .as_ref()
3925                    .map(|(pid, _)| *pid)
3926                    .filter(|pid| *pid > 0)
3927                    .or(snapshot.waiting_for_previous_pid),
3928                waiting_for_previous_detail: recovery_block
3929                    .or_else(|| retained.map(|(_, detail)| detail))
3930                    .or(snapshot.waiting_for_previous_detail),
3931                module_id: self.inner.module_id.clone(),
3932                launch_nonce_source,
3933                state: snapshot.state,
3934                enabled: snapshot.enabled,
3935                process_alive: snapshot.process_alive,
3936                registration_active,
3937                protocol,
3938                live,
3939                restart_count,
3940                lifetime_restarts: snapshot.lifetime_restarts,
3941                spawn_generation: snapshot.spawn_generation,
3942                max_restarts: self.inner.restart_policy.max_restarts,
3943                restart_window: self.inner.restart_policy.window,
3944                drain_timeout,
3945                restart_backoff: self.inner.restart_policy.backoff,
3946                restart_max_backoff: self.inner.restart_policy.max_backoff,
3947                pid,
3948                spawned_at_ms: snapshot.spawned_at_ms,
3949                spawned_from: snapshot.spawned_from,
3950                launch_spec_digest: snapshot.launch_spec_digest,
3951                process_start_time: snapshot.process_start_time,
3952                last_exit: snapshot.last_exit,
3953                health,
3954            },
3955            snapshot.spawned_file_identity.clone(),
3956        ))
3957    }
3958
3959    #[cfg(test)]
3960    pub(crate) fn hold_snapshot_for_test(
3961        &self,
3962        acquired: std::sync::mpsc::Sender<()>,
3963        hold: Duration,
3964    ) -> std::thread::JoinHandle<()> {
3965        let snapshot = Arc::clone(&self.inner.snapshot);
3966        std::thread::spawn(move || {
3967            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3968            acquired
3969                .send(())
3970                .expect("test receiver waits for snapshot lock");
3971            std::thread::sleep(hold);
3972        })
3973    }
3974
3975    /// The status and the running-image check for `supervisor.provenance`,
3976    /// taken from one status read. The exec acknowledgement can land between
3977    /// two separate reads, and the reply would then pair "no pid yet" with an
3978    /// image observed after the module started, which describes no single
3979    /// moment.
3980    pub(crate) async fn status_and_running_image_agreement(
3981        &self,
3982    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3983        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3984        let image = self
3985            .inner
3986            .provenance_probe
3987            .observe(
3988                status.pid,
3989                status.spawned_from.as_deref(),
3990                identity,
3991                status.process_start_time,
3992            )
3993            .await;
3994        Ok((status, image))
3995    }
3996
3997    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3998        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3999            Ok(snapshot) => snapshot.clone(),
4000            Err(_) => {
4001                return subc_control::RunningImageAgreement::Unavailable {
4002                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
4003                };
4004            }
4005        };
4006        self.inner
4007            .provenance_probe
4008            .observe(
4009                snapshot.reported_pid(),
4010                snapshot.spawned_from.as_deref(),
4011                snapshot.spawned_file_identity.clone(),
4012                snapshot.process_start_time,
4013            )
4014            .await
4015    }
4016
4017    /// Memory and CPU time of the module's current process, read now. Only the
4018    /// process the supervisor spawned is read, not processes it has started.
4019    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4020        #[cfg(windows)]
4021        {
4022            let image = lock_snapshot(&self.inner.snapshot)
4023                .ok()
4024                .and_then(|snapshot| {
4025                    snapshot
4026                        .spawned_file_identity
4027                        .as_ref()
4028                        .and_then(|identity| identity.windows_image.clone())
4029                });
4030            match image {
4031                Some(image) => {
4032                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
4033                }
4034                None => subc_control::ChildResourceUsage::Unavailable {
4035                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4036                },
4037            }
4038        }
4039        #[cfg(not(windows))]
4040        {
4041            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4042                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4043                Err(_) => {
4044                    return subc_control::ChildResourceUsage::Unavailable {
4045                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4046                    }
4047                }
4048            };
4049            crate::child_resources::read(pid, start_time)
4050        }
4051    }
4052
4053    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4054        self.inner
4055            .snapshot
4056            .try_lock()
4057            .map(|snapshot| snapshot.subtree_target.clone())
4058            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4059    }
4060
4061    #[cfg(test)]
4062    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4063        self.inner.snapshot.lock().unwrap().subtree_target = target;
4064    }
4065
4066    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4067        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4068        Ok(match snapshot.state {
4069            ModuleState::Restarting => true,
4070            ModuleState::Failed | ModuleState::Disabled => false,
4071            _ if snapshot.enabled && snapshot.respawn_pending => true,
4072            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4073        })
4074    }
4075
4076    #[cfg(test)]
4077    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4078        self.is_warming_with_snapshot_lock(None)
4079    }
4080
4081    pub(crate) fn is_warming_for_control(
4082        &self,
4083        caller: &'static str,
4084    ) -> Result<bool, SuperviseError> {
4085        self.is_warming_with_snapshot_lock(Some(caller))
4086    }
4087
4088    fn is_warming_with_snapshot_lock(
4089        &self,
4090        caller: Option<&'static str>,
4091    ) -> Result<bool, SuperviseError> {
4092        let snapshot = match caller {
4093            Some(caller) => {
4094                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4095            }
4096            None => lock_snapshot(&self.inner.snapshot)?,
4097        }
4098        .clone();
4099        Ok(matches!(
4100            snapshot.state,
4101            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4102        ))
4103    }
4104
4105    /// Drain the module and stop monitoring it.
4106    pub async fn drain(&self) -> Result<(), SuperviseError> {
4107        self.stop().await
4108    }
4109
4110    pub(crate) async fn retire_requested(
4111        &self,
4112        requester: Option<Principal>,
4113    ) -> Result<(), SuperviseError> {
4114        match self.state()? {
4115            ModuleState::Stopped => return Ok(()),
4116            ModuleState::Failed
4117                if lock_snapshot(&self.inner.snapshot)?
4118                    .failed_retry
4119                    .pending
4120                    .is_none() =>
4121            {
4122                return Ok(())
4123            }
4124            ModuleState::Failed
4125            | ModuleState::Starting
4126            | ModuleState::Running
4127            | ModuleState::Unresponsive
4128            | ModuleState::Restarting
4129            | ModuleState::Draining
4130            | ModuleState::Disabled => {}
4131        }
4132
4133        let (reply_tx, reply_rx) = oneshot::channel();
4134        self.inner
4135            .commands
4136            .send(SupervisorCommand::Retire {
4137                requester,
4138                reply: reply_tx,
4139            })
4140            .await
4141            .map_err(|_| SuperviseError::CommandClosed {
4142                module_id: self.inner.module_id.clone(),
4143            })?;
4144        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4145            module_id: self.inner.module_id.clone(),
4146        })?
4147    }
4148
4149    pub async fn stop(&self) -> Result<(), SuperviseError> {
4150        self.stop_requested(None).await
4151    }
4152
4153    pub(crate) async fn stop_requested(
4154        &self,
4155        requester: Option<Principal>,
4156    ) -> Result<(), SuperviseError> {
4157        match self.state()? {
4158            ModuleState::Stopped => return Ok(()),
4159            ModuleState::Failed
4160                if lock_snapshot(&self.inner.snapshot)?
4161                    .failed_retry
4162                    .pending
4163                    .is_none() =>
4164            {
4165                return Ok(())
4166            }
4167            ModuleState::Failed
4168            | ModuleState::Starting
4169            | ModuleState::Running
4170            | ModuleState::Unresponsive
4171            | ModuleState::Restarting
4172            | ModuleState::Draining
4173            | ModuleState::Disabled => {}
4174        }
4175
4176        let (reply_tx, reply_rx) = oneshot::channel();
4177        self.inner
4178            .commands
4179            .send(SupervisorCommand::Drain {
4180                requester,
4181                reply: reply_tx,
4182            })
4183            .await
4184            .map_err(|_| SuperviseError::CommandClosed {
4185                module_id: self.inner.module_id.clone(),
4186            })?;
4187        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4188            module_id: self.inner.module_id.clone(),
4189        })?
4190    }
4191
4192    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4193        self.restart_requested(drain_timeout_ms, None).await
4194    }
4195
4196    pub(crate) async fn restart_requested(
4197        &self,
4198        drain_timeout_ms: Option<u64>,
4199        requester: Option<Principal>,
4200    ) -> Result<(), SuperviseError> {
4201        let queued_at = Instant::now();
4202        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4203        let (reply_tx, reply_rx) = oneshot::channel();
4204        self.inner
4205            .commands
4206            .send(SupervisorCommand::Restart {
4207                requester,
4208                drain_timeout_ms,
4209                received_at_generation,
4210                queued_at,
4211                reply: reply_tx,
4212            })
4213            .await
4214            .map_err(|_| SuperviseError::CommandClosed {
4215                module_id: self.inner.module_id.clone(),
4216            })?;
4217        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4218            module_id: self.inner.module_id.clone(),
4219        })?
4220    }
4221
4222    /// Blue/green restart: see `SupervisorCommand::Swap` and the
4223    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4224    /// process then drains in the background of the supervise loop) or has
4225    /// failed, leaving the old process serving.
4226    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4227        self.swap_requested(ready_timeout, None).await
4228    }
4229
4230    pub(crate) async fn swap_requested(
4231        &self,
4232        ready_timeout: Option<Duration>,
4233        requester: Option<Principal>,
4234    ) -> Result<(), SuperviseError> {
4235        let (reply_tx, reply_rx) = oneshot::channel();
4236        self.inner
4237            .commands
4238            .send(SupervisorCommand::Swap {
4239                requester,
4240                ready_timeout,
4241                reply: reply_tx,
4242            })
4243            .await
4244            .map_err(|_| SuperviseError::CommandClosed {
4245                module_id: self.inner.module_id.clone(),
4246            })?;
4247        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4248            module_id: self.inner.module_id.clone(),
4249        })?
4250    }
4251
4252    pub async fn reload(&self) -> Result<(), SuperviseError> {
4253        self.reload_requested(None).await
4254    }
4255
4256    pub(crate) async fn reload_requested(
4257        &self,
4258        requester: Option<Principal>,
4259    ) -> Result<(), SuperviseError> {
4260        let (reply_tx, reply_rx) = oneshot::channel();
4261        self.inner
4262            .commands
4263            .send(SupervisorCommand::Reload {
4264                requester,
4265                reply: reply_tx,
4266            })
4267            .await
4268            .map_err(|_| SuperviseError::CommandClosed {
4269                module_id: self.inner.module_id.clone(),
4270            })?;
4271        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4272            module_id: self.inner.module_id.clone(),
4273        })?
4274    }
4275
4276    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4277        self.set_enabled_requested(enabled, None).await
4278    }
4279
4280    pub(crate) async fn set_enabled_requested(
4281        &self,
4282        enabled: bool,
4283        requester: Option<Principal>,
4284    ) -> Result<bool, SuperviseError> {
4285        let (reply_tx, reply_rx) = oneshot::channel();
4286        self.inner
4287            .commands
4288            .send(SupervisorCommand::SetEnabled {
4289                requester,
4290                enabled,
4291                reply: reply_tx,
4292            })
4293            .await
4294            .map_err(|_| SuperviseError::CommandClosed {
4295                module_id: self.inner.module_id.clone(),
4296            })?;
4297        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4298            module_id: self.inner.module_id.clone(),
4299        })?
4300    }
4301
4302    /// The current process's protocol, or the configured protocol when down.
4303    /// A rescan stores the next launch spec without changing how an existing
4304    /// process registers, serves routes, is probed, or exits.
4305    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4306        let configured = self
4307            .inner
4308            .configuration
4309            .lock()
4310            .map_err(|_| SuperviseError::StatePoisoned {
4311                module_id: Some(self.inner.module_id.clone()),
4312            })?
4313            .spec
4314            .protocol;
4315        let state = lock_snapshot(&self.inner.snapshot)?;
4316        Ok(state.spawned_protocol.unwrap_or(configured))
4317    }
4318
4319    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4320        let configuration =
4321            self.inner
4322                .configuration
4323                .lock()
4324                .map_err(|_| SuperviseError::StatePoisoned {
4325                    module_id: Some(self.inner.module_id.clone()),
4326                })?;
4327        Ok((configuration.spec.clone(), configuration.health.clone()))
4328    }
4329
4330    /// Replace this module's launch spec, keeping its health and drain policy,
4331    /// the way a rescan does for a changed config entry. The running process is
4332    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4333    #[cfg(any(test, feature = "test-support"))]
4334    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4335        let (_, health) = self.configuration()?;
4336        let drain_timeout_ms = u64::try_from(
4337            self.inner
4338                .effective_drain_timeout
4339                .lock()
4340                .unwrap_or_else(|poisoned| poisoned.into_inner())
4341                .as_millis(),
4342        )
4343        .ok();
4344        self.update_configuration(spec, health, drain_timeout_ms)
4345            .await
4346    }
4347
4348    pub(crate) async fn update_configuration(
4349        &self,
4350        spec: ModuleSpec,
4351        health: HealthConfig,
4352        drain_timeout_ms: Option<u64>,
4353    ) -> Result<(), SuperviseError> {
4354        if spec.module_id != self.inner.module_id {
4355            return Err(SuperviseError::InvalidSpec {
4356                reason: "a supervised module's module_id cannot be changed".to_string(),
4357            });
4358        }
4359        validate_spec(&spec)?;
4360        let (reply_tx, reply_rx) = oneshot::channel();
4361        self.inner
4362            .commands
4363            .send(SupervisorCommand::UpdateConfiguration {
4364                spec: spec.clone(),
4365                health: health.clone(),
4366                drain_timeout_ms,
4367                reply: reply_tx,
4368            })
4369            .await
4370            .map_err(|_| SuperviseError::CommandClosed {
4371                module_id: self.inner.module_id.clone(),
4372            })?;
4373        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4374            module_id: self.inner.module_id.clone(),
4375        })?;
4376        let mut configuration =
4377            self.inner
4378                .configuration
4379                .lock()
4380                .map_err(|_| SuperviseError::StatePoisoned {
4381                    module_id: Some(self.inner.module_id.clone()),
4382                })?;
4383        configuration.spec = spec;
4384        configuration.health = health;
4385        Ok(())
4386    }
4387}
4388
4389impl Drop for SupervisedModuleInner {
4390    fn drop(&mut self) {
4391        let Ok(mut monitor) = self.monitor.lock() else {
4392            return;
4393        };
4394        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4395            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4396                state.state = ModuleState::Stopped;
4397                clear_current_process_facts(state);
4398            });
4399            monitor.abort();
4400        }
4401        let _ = monitor.take();
4402    }
4403}
4404
4405#[derive(Debug)]
4406enum SupervisorCommand {
4407    Drain {
4408        requester: Option<Principal>,
4409        reply: oneshot::Sender<Result<(), SuperviseError>>,
4410    },
4411    Retire {
4412        requester: Option<Principal>,
4413        reply: oneshot::Sender<Result<(), SuperviseError>>,
4414    },
4415    Restart {
4416        requester: Option<Principal>,
4417        /// Operator override for this one restart's drain budget, in ms. `None`
4418        /// uses the module's configured/default budget; `Some(0)` cuts
4419        /// immediately (wedge bounce: a stuck request never settles, so
4420        /// waiting only delays recovery).
4421        drain_timeout_ms: Option<u64>,
4422        /// The module's `spawn_generation` when the request was received, before
4423        /// it waited in the command queue. A queued restart whose module has
4424        /// since spawned a newer process is already satisfied (see the handler).
4425        received_at_generation: u64,
4426        /// When the request entered the command queue, so the handler can log
4427        /// how long it waited behind the loop's other work.
4428        queued_at: Instant,
4429        reply: oneshot::Sender<Result<(), SuperviseError>>,
4430    },
4431    Reload {
4432        requester: Option<Principal>,
4433        reply: oneshot::Sender<Result<(), SuperviseError>>,
4434    },
4435    SetEnabled {
4436        requester: Option<Principal>,
4437        enabled: bool,
4438        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4439    },
4440    UpdateConfiguration {
4441        spec: ModuleSpec,
4442        health: HealthConfig,
4443        /// Per-module drain override from the new config; `None` re-resolves to
4444        /// the supervisor-wide default.
4445        drain_timeout_ms: Option<u64>,
4446        reply: oneshot::Sender<()>,
4447    },
4448    Swap {
4449        requester: Option<Principal>,
4450        /// How long the candidate may take to register and declare itself
4451        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4452        ready_timeout: Option<Duration>,
4453        /// Answered at cutover or failure; the incumbent's drain follows.
4454        reply: oneshot::Sender<Result<(), SuperviseError>>,
4455    },
4456}
4457
4458#[derive(Debug)]
4459pub enum SuperviseError {
4460    InvalidSpec {
4461        reason: String,
4462    },
4463    Spawn {
4464        program: PathBuf,
4465        source: io::Error,
4466        cgroup_path: Option<PathBuf>,
4467    },
4468    Cgroup {
4469        module_id: String,
4470        source: io::Error,
4471    },
4472    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4473    /// than spawn a reserved module without its identity binding.
4474    LaunchNonce {
4475        reason: String,
4476    },
4477    Wait {
4478        module_id: String,
4479        source: io::Error,
4480    },
4481    Kill {
4482        module_id: String,
4483        source: io::Error,
4484    },
4485    Forwarding(ForwardingError),
4486    Registry(RegistryError),
4487    ReloadUnavailable {
4488        module_id: String,
4489        reason: String,
4490    },
4491    /// An operator restart/reload was requested for a module that is currently
4492    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4493    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4494    /// by a restart, so these commands are rejected instead of re-enabling it.
4495    Disabled {
4496        module_id: String,
4497    },
4498    ReloadFailed {
4499        module_id: String,
4500        reason: String,
4501    },
4502    RegistrationStillActive {
4503        module_id: String,
4504        waited: Duration,
4505    },
4506    StatePoisoned {
4507        module_id: Option<String>,
4508    },
4509    CommandClosed {
4510        module_id: String,
4511    },
4512    /// A restart or reload arrived while a swap's candidate was warming. The
4513    /// swap owns the module until it cuts over or fails; a stop or disable
4514    /// would have aborted it instead.
4515    SwapInProgress {
4516        module_id: String,
4517    },
4518    /// A swap was refused before anything was spawned.
4519    SwapRefused {
4520        module_id: String,
4521        reason: SwapRefusal,
4522    },
4523    /// A swap spawned a candidate and gave up on it. The candidate has been
4524    /// killed and its slot freed; the incumbent was left serving and was never
4525    /// drained, except in the one `CutoverLost` case described on that arm.
4526    SwapFailed {
4527        module_id: String,
4528        arm: SwapFailureArm,
4529        detail: String,
4530        /// How the candidate exited, when it exited on its own before the
4531        /// supervisor gave up on it.
4532        candidate_exit: Option<ExitReport>,
4533    },
4534}
4535
4536/// Why a swap was refused before a candidate was spawned.
4537#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4538pub enum SwapRefusal {
4539    /// The module's config does not declare `overlap: "safe"`.
4540    OverlapExclusive,
4541    /// The module is not registered, so there is no incumbent to keep serving
4542    /// and nothing a swap would improve on; a plain restart is the tool.
4543    NotRegistered,
4544    /// The module does not speak the subc wire, so a candidate could never
4545    /// register or declare itself ready.
4546    ProtocolNone,
4547    /// The supervisor lacks the forwarding table (to cut routes over) or the
4548    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4549    NotConfigured,
4550    /// A swap is already open for this module.
4551    AlreadySwapping,
4552}
4553
4554impl SwapRefusal {
4555    pub fn as_str(self) -> &'static str {
4556        match self {
4557            Self::OverlapExclusive => "overlap_exclusive",
4558            Self::NotRegistered => "not_registered",
4559            Self::ProtocolNone => "protocol_none",
4560            Self::NotConfigured => "not_configured",
4561            Self::AlreadySwapping => "already_swapping",
4562        }
4563    }
4564}
4565
4566/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4567/// serving and undrained; see `CutoverLost`.
4568#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4569pub enum SwapFailureArm {
4570    /// The candidate process could not be started.
4571    SpawnFailed,
4572    /// The candidate did not register within the readiness budget.
4573    NeverRegistered,
4574    /// The candidate registered but did not declare itself ready in time.
4575    NeverReady,
4576    /// The candidate exited before cutover.
4577    CandidateExited,
4578    /// The candidate declared itself ready but failed its health probe.
4579    CandidateUnhealthy,
4580    /// An operator stop, disable or retire arrived while the candidate warmed.
4581    /// The candidate was killed and the operator's command then carried out on
4582    /// the incumbent.
4583    Interrupted,
4584    /// The candidate's connection closed at the moment of cutover. If it
4585    /// closed before forwarding moved, the incumbent is untouched. If it closed
4586    /// between the forwarding and registry halves of cutover, forwarding can no
4587    /// longer route to the incumbent, so the module is restarted plainly.
4588    CutoverLost,
4589}
4590
4591impl SwapFailureArm {
4592    pub fn as_str(self) -> &'static str {
4593        match self {
4594            Self::SpawnFailed => "spawn_failed",
4595            Self::NeverRegistered => "never_registered",
4596            Self::NeverReady => "never_ready",
4597            Self::CandidateExited => "candidate_exited",
4598            Self::CandidateUnhealthy => "candidate_unhealthy",
4599            Self::Interrupted => "interrupted",
4600            Self::CutoverLost => "cutover_lost",
4601        }
4602    }
4603}
4604
4605impl fmt::Display for SuperviseError {
4606    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4607        match self {
4608            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4609            Self::Spawn {
4610                program,
4611                source,
4612                cgroup_path: Some(cgroup_path),
4613            } => write!(
4614                f,
4615                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4616                cgroup_path.display(),
4617                program.display()
4618            ),
4619            Self::Spawn {
4620                program,
4621                source,
4622                cgroup_path: None,
4623            } => write!(
4624                f,
4625                "failed to spawn module '{}': {source}",
4626                program.display()
4627            ),
4628            Self::Cgroup { module_id, source } => {
4629                write!(
4630                    f,
4631                    "failed to prepare cgroup for module '{module_id}': {source}"
4632                )
4633            }
4634            Self::LaunchNonce { reason } => {
4635                write!(
4636                    f,
4637                    "failed to generate reserved-module launch nonce: {reason}"
4638                )
4639            }
4640            Self::Wait { module_id, source } => {
4641                write!(f, "failed to wait for module '{module_id}': {source}")
4642            }
4643            Self::Kill { module_id, source } => {
4644                write!(f, "failed to kill module '{module_id}': {source}")
4645            }
4646            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4647            Self::Registry(err) => write!(f, "registry error: {err}"),
4648            Self::ReloadUnavailable { module_id, reason } => {
4649                write!(f, "reload unavailable for module '{module_id}': {reason}")
4650            }
4651            Self::Disabled { module_id } => {
4652                write!(
4653                    f,
4654                    "module '{module_id}' is disabled; enable it before restart or reload"
4655                )
4656            }
4657            Self::ReloadFailed { module_id, reason } => {
4658                write!(f, "reload failed for module '{module_id}': {reason}")
4659            }
4660            Self::RegistrationStillActive { module_id, waited } => write!(
4661                f,
4662                "module '{module_id}' registration remained active after waiting {waited:?}"
4663            ),
4664            Self::StatePoisoned { module_id } => match module_id {
4665                Some(module_id) => {
4666                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4667                }
4668                None => write!(f, "supervisor state was poisoned"),
4669            },
4670            Self::CommandClosed { module_id } => {
4671                write!(
4672                    f,
4673                    "supervisor command channel for module '{module_id}' is closed"
4674                )
4675            }
4676            Self::SwapInProgress { module_id } => write!(
4677                f,
4678                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4679            ),
4680            Self::SwapRefused { module_id, reason } => match reason {
4681                SwapRefusal::OverlapExclusive => write!(
4682                    f,
4683                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4684                ),
4685                SwapRefusal::NotRegistered => write!(
4686                    f,
4687                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4688                ),
4689                SwapRefusal::ProtocolNone => write!(
4690                    f,
4691                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4692                ),
4693                SwapRefusal::NotConfigured => write!(
4694                    f,
4695                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4696                ),
4697                SwapRefusal::AlreadySwapping => {
4698                    write!(f, "module '{module_id}' is already being swapped")
4699                }
4700            },
4701            Self::SwapFailed {
4702                module_id,
4703                arm,
4704                detail,
4705                ..
4706            } => write!(
4707                f,
4708                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4709                arm.as_str()
4710            ),
4711        }
4712    }
4713}
4714
4715impl Error for SuperviseError {
4716    fn source(&self) -> Option<&(dyn Error + 'static)> {
4717        match self {
4718            Self::Spawn { source, .. }
4719            | Self::Cgroup { source, .. }
4720            | Self::Wait { source, .. }
4721            | Self::Kill { source, .. } => Some(source),
4722            Self::Forwarding(err) => Some(err),
4723            Self::Registry(err) => Some(err),
4724            Self::LaunchNonce { .. }
4725            | Self::InvalidSpec { .. }
4726            | Self::ReloadUnavailable { .. }
4727            | Self::Disabled { .. }
4728            | Self::ReloadFailed { .. }
4729            | Self::RegistrationStillActive { .. }
4730            | Self::StatePoisoned { .. }
4731            | Self::CommandClosed { .. }
4732            | Self::SwapInProgress { .. }
4733            | Self::SwapRefused { .. }
4734            | Self::SwapFailed { .. } => None,
4735        }
4736    }
4737}
4738
4739pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4740    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4741        return Err(SuperviseError::InvalidSpec {
4742            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4743        });
4744    }
4745    if spec.module_id.trim().is_empty() {
4746        return Err(SuperviseError::InvalidSpec {
4747            reason: "module_id must not be empty".to_string(),
4748        });
4749    }
4750
4751    Ok(())
4752}
4753
4754#[derive(Debug, Default)]
4755struct HealthProbeRuntime {
4756    spawn_generation: Option<u64>,
4757    configured_health: Option<HealthConfig>,
4758    registered_connection: Option<crate::ConnectionId>,
4759    advertised: bool,
4760    next_probe_at: Option<Instant>,
4761    probe_index: u64,
4762}
4763
4764fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4765    lock_snapshot(snapshot)
4766        .ok()
4767        .and_then(|state| state.spawned_protocol)
4768        .unwrap_or(spec.protocol)
4769}
4770
4771impl HealthProbeRuntime {
4772    fn refresh_registration(
4773        &mut self,
4774        spec: &ModuleSpec,
4775        runtime: &SupervisorRuntimeConfig,
4776        registry: &Registry,
4777        snapshot: &SharedSnapshot,
4778    ) {
4779        let generation = lock_snapshot(snapshot)
4780            .ok()
4781            .map(|state| state.spawn_generation);
4782        if self.spawn_generation != generation {
4783            // A replacement cannot reuse a deadline that is already due for its
4784            // predecessor. HTTP children get a full cadence to start; wire
4785            // children still wait for their own health-advertising HELLO.
4786            *self = Self {
4787                spawn_generation: generation,
4788                ..Self::default()
4789            };
4790        }
4791        if self.configured_health.as_ref() != Some(&runtime.health) {
4792            self.configured_health = Some(runtime.health.clone());
4793            self.next_probe_at = None;
4794            self.registered_connection = None;
4795            self.probe_index = 0;
4796        }
4797        // A non-wire process never registers. Only an explicitly configured
4798        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4799        // health failure for that kind of process.
4800        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4801            self.registered_connection = None;
4802            self.advertised = runtime.health.http.is_some();
4803            if !self.advertised {
4804                self.next_probe_at = None;
4805                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4806                    let unknown = ModuleHealthStatus::default();
4807                    if state.health != unknown {
4808                        state.health = unknown;
4809                    }
4810                });
4811            } else if self.next_probe_at.is_none() {
4812                self.next_probe_at = Some(
4813                    Instant::now()
4814                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4815                );
4816            }
4817            return;
4818        }
4819
4820        let registration = match registry.get_module(&spec.module_id) {
4821            Ok(registration) => registration,
4822            Err(err) => {
4823                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4824                self.advertised = false;
4825                self.next_probe_at = None;
4826                return;
4827            }
4828        };
4829
4830        let Some(registration) = registration else {
4831            self.registered_connection = None;
4832            self.advertised = false;
4833            self.next_probe_at = None;
4834            return;
4835        };
4836
4837        let advertised = registration
4838            .control_ops
4839            .iter()
4840            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4841        if !advertised {
4842            self.registered_connection = Some(registration.connection_id);
4843            self.advertised = false;
4844            self.next_probe_at = None;
4845            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4846                if state.health.status != SupervisorHealthStatus::Unknown
4847                    || state.health.consecutive_failures != 0
4848                    || state.health.last_probe_ms.is_some()
4849                    || state.health.detail.is_some()
4850                    || state.health.metrics.is_some()
4851                {
4852                    state.health.status = SupervisorHealthStatus::Unknown;
4853                    state.health.consecutive_failures = 0;
4854                    state.health.last_probe_ms = None;
4855                    state.health.detail = None;
4856                    state.health.metrics = None;
4857                }
4858            });
4859            return;
4860        }
4861
4862        let reregistered = self.registered_connection != Some(registration.connection_id);
4863        self.registered_connection = Some(registration.connection_id);
4864        self.advertised = true;
4865        if reregistered || self.next_probe_at.is_none() {
4866            self.probe_index = 0;
4867            self.next_probe_at = Some(
4868                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4869            );
4870            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4871                state.health.status = SupervisorHealthStatus::Unknown;
4872                state.health.consecutive_failures = 0;
4873                state.health.detail = None;
4874                state.health.metrics = None;
4875            });
4876        }
4877    }
4878
4879    fn wake_at(&self) -> Option<Instant> {
4880        if !self.advertised {
4881            return None;
4882        }
4883        self.next_probe_at
4884    }
4885
4886    fn due(&self) -> bool {
4887        self.advertised
4888            && self
4889                .next_probe_at
4890                .is_some_and(|next| Instant::now() >= next)
4891    }
4892
4893    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4894        self.probe_index = self.probe_index.wrapping_add(1);
4895        self.next_probe_at = Some(
4896            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4897        );
4898    }
4899}
4900
4901/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4902///
4903/// This was a struct with a single `message: String`, and every one of the
4904/// fifteen construction sites collapsed into it. Each site knows exactly what it
4905/// saw -- the lane is gone, the module did not answer in time, the module
4906/// answered with the wrong thing -- and `handle_health_probe_failure` then
4907/// treated all of them identically: increment a counter, compare to a threshold,
4908/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4909/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4910///
4911/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4912///
4913/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4914///   answer on it again.
4915/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4916///   AND with a perfectly healthy one that lost a CPU race -- which is what
4917///   happens under machine load, and is how this supervisor killed a healthy
4918///   module three times in one day.
4919/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4920///   Restarting on it is defensible, but it is not the silence case and should
4921///   never be counted as one.
4922/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4923///   anything, so it cannot be evidence about the module at all.
4924///
4925/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4926/// one that fires most often, and while every variant collapsed into one string
4927/// it carried the same weight as the strongest.
4928///
4929/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4930/// DESIGN and a reader stopping at it gets the build backwards: the restart
4931/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4932/// probes still increment the failure streak and drive escalation at the
4933/// threshold (see `is_proof_of_death` below for why that is deliberate and
4934/// what gates the change). Absence of evidence restarts modules today.
4935#[derive(Debug)]
4936enum HealthProbeEvidence {
4937    /// The module's control lane is gone. Proof of death.
4938    LaneDead,
4939    /// No reply within the deadline. Proves nothing about the module's state.
4940    NoAnswer,
4941    /// The module replied, but not with a usable health report. Proves it is alive.
4942    BadAnswer,
4943    /// The daemon could not ask. Says nothing about the module.
4944    Misconfigured,
4945}
4946
4947#[derive(Debug)]
4948struct HealthProbeError {
4949    evidence: HealthProbeEvidence,
4950    message: String,
4951}
4952
4953impl HealthProbeError {
4954    fn lane_dead(message: impl Into<String>) -> Self {
4955        Self::with(HealthProbeEvidence::LaneDead, message)
4956    }
4957
4958    fn no_answer(message: impl Into<String>) -> Self {
4959        Self::with(HealthProbeEvidence::NoAnswer, message)
4960    }
4961
4962    fn bad_answer(message: impl Into<String>) -> Self {
4963        Self::with(HealthProbeEvidence::BadAnswer, message)
4964    }
4965
4966    fn misconfigured(message: impl Into<String>) -> Self {
4967        Self::with(HealthProbeEvidence::Misconfigured, message)
4968    }
4969
4970    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4971        Self {
4972            evidence,
4973            message: message.into(),
4974        }
4975    }
4976
4977    /// Whether this observation is proof the module cannot serve.
4978    ///
4979    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4980    /// variant that fires under CPU starvation, and treating it as proof is the
4981    /// defect this enum exists to make impossible to reintroduce silently.
4982    ///
4983    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4984    /// to restart also needs a bound for the case it excludes -- a genuinely
4985    /// wedged module, alive but never answering -- and that bound must come from
4986    /// the distribution of real late-answer latencies, which nothing measures
4987    /// yet. Landing the classification first makes the later change a one-line
4988    /// decision against evidence that already exists, rather than two unproven
4989    /// changes at once.
4990    #[allow(dead_code)]
4991    fn is_proof_of_death(&self) -> bool {
4992        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4993    }
4994
4995    /// Short stable label for logs and the health snapshot.
4996    ///
4997    /// An operator reading `ck health` currently cannot tell "the module is gone"
4998    /// from "the module did not answer in five seconds", because both render as
4999    /// prose in the same field. These labels are what make the two
5000    /// distinguishable at a glance, and they are what a later restart-policy
5001    /// change will be argued from.
5002    fn label(&self) -> &'static str {
5003        match self.evidence {
5004            HealthProbeEvidence::LaneDead => "lane-dead",
5005            HealthProbeEvidence::NoAnswer => "no-answer",
5006            HealthProbeEvidence::BadAnswer => "bad-answer",
5007            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5008        }
5009    }
5010}
5011
5012impl fmt::Display for HealthProbeError {
5013    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5014        f.write_str(&self.message)
5015    }
5016}
5017
5018async fn run_health_probe_cycle(
5019    spec: &ModuleSpec,
5020    runtime: &SupervisorRuntimeConfig,
5021    registry: &Registry,
5022    process_liveness: &SupervisorProcessLiveness,
5023    snapshot: &SharedSnapshot,
5024    child: &mut Option<SupervisedChild>,
5025) {
5026    let now_ms = unix_ms_now();
5027    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5028        .then_some(runtime.health.http.as_deref())
5029        .flatten();
5030    let result = match http {
5031        Some(url) => probe_http_health(url, runtime.health.deadline).await,
5032        None => probe_module_health(&spec.module_id, runtime, None).await,
5033    };
5034    match result {
5035        Ok(report) => {
5036            handle_health_report(
5037                spec,
5038                runtime,
5039                registry,
5040                process_liveness,
5041                snapshot,
5042                child,
5043                report,
5044                now_ms,
5045            )
5046            .await;
5047        }
5048        Err(err) => {
5049            if http.is_some() {
5050                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5051                    state.health.status = SupervisorHealthStatus::Failing;
5052                });
5053            }
5054            handle_health_probe_failure(
5055                spec,
5056                runtime,
5057                registry,
5058                process_liveness,
5059                snapshot,
5060                child,
5061                err,
5062                now_ms,
5063            )
5064            .await;
5065        }
5066    }
5067}
5068
5069pub(crate) struct HttpProbeTarget<'a> {
5070    address: std::net::SocketAddr,
5071    localhost: bool,
5072    authority: &'a str,
5073    path: String,
5074}
5075
5076/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
5077/// or TLS. A URL cannot turn a local health check into an outbound connection.
5078pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5079    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5080        return Err("must not contain whitespace, controls, or a fragment".into());
5081    }
5082    let rest = url
5083        .strip_prefix("http://")
5084        .ok_or("must use plain http://")?;
5085    let split = rest.find(['/', '?']).unwrap_or(rest.len());
5086    let (authority, suffix) = rest.split_at(split);
5087    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5088        ("::1", rest)
5089    } else {
5090        let split = authority.find(':').unwrap_or(authority.len());
5091        authority.split_at(split)
5092    };
5093    let ip: std::net::IpAddr = match host {
5094        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5095        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5096        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5097    };
5098    let port = if port.is_empty() {
5099        80
5100    } else {
5101        port.strip_prefix(':')
5102            .and_then(|p| p.parse::<u16>().ok())
5103            .filter(|p| *p > 0)
5104            .ok_or("must have a valid nonzero TCP port")?
5105    };
5106    let path = if suffix.is_empty() {
5107        "/".into()
5108    } else if suffix.starts_with('?') {
5109        format!("/{suffix}")
5110    } else {
5111        suffix.into()
5112    };
5113    Ok(HttpProbeTarget {
5114        address: std::net::SocketAddr::new(ip, port),
5115        localhost: host == "localhost",
5116        authority,
5117        path,
5118    })
5119}
5120
5121async fn probe_http_health(
5122    url: &str,
5123    deadline: Duration,
5124) -> Result<HealthReport, HealthProbeError> {
5125    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5126    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5127    // Keep partial diagnostics outside the timed future so cancellation does
5128    // not discard a status line or body bytes already received.
5129    let mut response_status = String::new();
5130    let mut body = Vec::new();
5131    let probe = async {
5132        // Resolve localhost ourselves so a hosts-file override cannot turn
5133        // this into an outbound request, while IPv6-only local servers work.
5134        let connection = match tokio::net::TcpStream::connect(target.address).await {
5135            Err(_) if target.localhost => {
5136                tokio::net::TcpStream::connect((
5137                    std::net::Ipv6Addr::LOCALHOST,
5138                    target.address.port(),
5139                ))
5140                .await
5141            }
5142            result => result,
5143        };
5144        let mut stream = connection.map_err(|error| {
5145            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5146        })?;
5147        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5148            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5149        let mut reader = BufReader::new(stream);
5150        let mut budget = 16 * 1024;
5151        let status = http_line(&mut reader, &mut budget).await?;
5152        let mut words = status.split_ascii_whitespace();
5153        let version = words.next();
5154        let code = words
5155            .next()
5156            .filter(|word| word.len() == 3)
5157            .and_then(|word| word.parse::<u16>().ok());
5158        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5159            || !code.is_some_and(|code| (100..600).contains(&code))
5160        {
5161            return Err(HealthProbeError::bad_answer(format!(
5162                "invalid HTTP status: {status}"
5163            )));
5164        }
5165        let code = code.expect("validated status code");
5166        response_status = status.clone();
5167        let mut length = None;
5168        let mut chunked = false;
5169        loop {
5170            let line = http_line(&mut reader, &mut budget).await?;
5171            if line.is_empty() {
5172                break;
5173            }
5174            if let Some((name, value)) = line.split_once(':') {
5175                if name.eq_ignore_ascii_case("content-length") {
5176                    length = Some(value.trim().parse::<u64>().map_err(|_| {
5177                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
5178                    })?);
5179                } else if name.eq_ignore_ascii_case("transfer-encoding") {
5180                    chunked = value.trim().eq_ignore_ascii_case("chunked");
5181                }
5182            }
5183        }
5184        if chunked {
5185            while body.len() < 200 {
5186                let line = http_line(&mut reader, &mut budget).await?;
5187                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5188                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5189                if size == 0 {
5190                    break;
5191                }
5192                let count = size.min((200 - body.len()) as u64) as usize;
5193                let start = body.len();
5194                (&mut reader)
5195                    .take(count as u64)
5196                    .read_to_end(&mut body)
5197                    .await
5198                    .map_err(|error| {
5199                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5200                    })?;
5201                if body.len() - start != count {
5202                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5203                }
5204                if size > count as u64 || body.len() == 200 {
5205                    break;
5206                }
5207                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5208                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5209                }
5210            }
5211        } else {
5212            reader
5213                .take(length.unwrap_or(200).min(200))
5214                .read_to_end(&mut body)
5215                .await
5216                .map_err(|error| {
5217                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5218                })?;
5219        }
5220        if (200..300).contains(&code) {
5221            Ok(HealthReport::ok())
5222        } else {
5223            Err(HealthProbeError::bad_answer(
5224                "HTTP health endpoint returned non-2xx",
5225            ))
5226        }
5227    };
5228    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5229        Err(HealthProbeError::no_answer(format!(
5230            "HTTP probe timed out after {deadline:?}"
5231        )))
5232    });
5233    if let Err(error) = &mut result {
5234        if !response_status.is_empty() {
5235            error.message = format!(
5236                "{}; {response_status}: {}",
5237                error.message,
5238                String::from_utf8_lossy(&body)
5239            );
5240        }
5241    }
5242    result
5243}
5244
5245async fn http_line(
5246    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5247    remaining: &mut usize,
5248) -> Result<String, HealthProbeError> {
5249    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5250    let mut line = Vec::new();
5251    (&mut *reader)
5252        .take(*remaining as u64)
5253        .read_until(b'\n', &mut line)
5254        .await
5255        .map_err(|error| {
5256            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5257        })?;
5258    *remaining -= line.len();
5259    if !line.ends_with(b"\r\n") {
5260        return Err(HealthProbeError::bad_answer(
5261            "HTTP headers are incomplete or exceed 16 KiB",
5262        ));
5263    }
5264    line.truncate(line.len() - 2);
5265    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5266}
5267
5268async fn probe_module_health(
5269    module_id: &str,
5270    runtime: &SupervisorRuntimeConfig,
5271    drain_deadline: Option<Instant>,
5272) -> Result<HealthReport, HealthProbeError> {
5273    let Some(forwarding) = runtime.forwarding.as_ref() else {
5274        return Err(HealthProbeError::misconfigured(
5275            "supervisor was not configured with a forwarding table",
5276        ));
5277    };
5278    let probe_started_at = Instant::now();
5279    let mut deadline = probe_started_at + runtime.health.deadline;
5280    if let Some(drain_deadline) = drain_deadline {
5281        deadline = deadline.min(drain_deadline);
5282    }
5283    let pending = if drain_deadline.is_some() {
5284        forwarding.begin_drain_health_probe_rpc_for(
5285            module_id,
5286            MODULE_CONTROL_OP_HEALTH_CHECK,
5287            probe_started_at,
5288            deadline,
5289        )
5290    } else {
5291        forwarding.begin_health_probe_rpc_for(
5292            module_id,
5293            MODULE_CONTROL_OP_HEALTH_CHECK,
5294            probe_started_at,
5295            deadline,
5296        )
5297    }
5298    .map_err(|err| {
5299        // The endpoint is not registered, so there is no live control lane to
5300        // ask. That is the module being absent, not slow.
5301        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5302    })?;
5303    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5304}
5305
5306/// [`probe_module_health`] for one endpoint rather than the id's active one.
5307///
5308/// A swap probes two processes that no by-id lookup reaches: its candidate
5309/// before cutover, and its superseded incumbent (for busy gauges) while the
5310/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5311/// bounds the by-id drain probe.
5312async fn probe_endpoint_health(
5313    endpoint: crate::ModuleEndpointId,
5314    runtime: &SupervisorRuntimeConfig,
5315    deadline_cap: Option<Instant>,
5316) -> Result<HealthReport, HealthProbeError> {
5317    let Some(forwarding) = runtime.forwarding.as_ref() else {
5318        return Err(HealthProbeError::misconfigured(
5319            "supervisor was not configured with a forwarding table",
5320        ));
5321    };
5322    let probe_started_at = Instant::now();
5323    let mut deadline = probe_started_at + runtime.health.deadline;
5324    if let Some(cap) = deadline_cap {
5325        deadline = deadline.min(cap);
5326    }
5327    let pending = forwarding
5328        .begin_endpoint_health_probe_rpc_for(
5329            endpoint,
5330            MODULE_CONTROL_OP_HEALTH_CHECK,
5331            probe_started_at,
5332            deadline,
5333        )
5334        .map_err(|err| {
5335            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5336        })?;
5337    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5338}
5339
5340/// Send a begun health probe and classify its answer.
5341async fn await_health_probe(
5342    forwarding: &ForwardingTable,
5343    pending: PendingModuleControlRpc,
5344    deadline: Instant,
5345    probe_budget: Duration,
5346) -> Result<HealthReport, HealthProbeError> {
5347    let PendingModuleControlRpc {
5348        endpoint,
5349        module_sink,
5350        negotiated_ver,
5351        corr,
5352        receiver,
5353    } = pending;
5354    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5355        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5356    })?;
5357    let frame = Frame::build_with_version(
5358        negotiated_ver,
5359        FrameType::Request,
5360        control_flags(),
5361        0,
5362        0,
5363        corr,
5364        body,
5365    )
5366    .map_err(|err| {
5367        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5368    })?;
5369
5370    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5371    // blocks waiting for capacity when the module's egress queue is full, and an
5372    // unbounded await here freezes the whole supervision actor (it stops polling
5373    // Child::wait and supervisor commands), making the module unrecoverable
5374    // in-band. On timeout the probe fails like any transport failure.
5375    match timeout_at(deadline, module_sink.send(frame)).await {
5376        Ok(Ok(())) => {}
5377        Ok(Err(err)) => {
5378            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5379            // A closed sink means the module's egress channel is gone -- the
5380            // receiving half is dropped when its connection tears down. Proof.
5381            return Err(HealthProbeError::lane_dead(format!(
5382                "failed to send health.check: {err}"
5383            )));
5384        }
5385        Err(_elapsed) => {
5386            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5387            // A full egress queue means the module is not draining its socket, which
5388            // is consistent with a wedged module AND with one whose reader is merely
5389            // starved. Silence, not proof.
5390            return Err(HealthProbeError::no_answer(
5391                "health.check send timed out before enqueue (module egress full)",
5392            ));
5393        }
5394    }
5395
5396    match timeout_at(deadline, receiver).await {
5397        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5398        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5399        // and those prove it is alive even though the probe failed.
5400        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5401            response.health_report().ok_or_else(|| {
5402                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5403            })
5404        }
5405        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5406            format!("health.check rejected: {}", body.message),
5407        )),
5408        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5409            Err(HealthProbeError::lane_dead(message))
5410        }
5411        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5412            Err(HealthProbeError::bad_answer(message))
5413        }
5414        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5415            Err(HealthProbeError::bad_answer(format!(
5416                "expected module-control op '{expected}', got '{actual}'"
5417            )))
5418        }
5419        // A reply that crosses the deadline before this waiter observes it is
5420        // still proof of life. The forwarding path records its end-to-end latency
5421        // before delivering this classification.
5422        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5423            "module answered health.check after its daemon deadline",
5424        )),
5425        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5426            "health.check waiter was canceled before the module responded",
5427        )),
5428        Err(_) => {
5429            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5430            Err(HealthProbeError::no_answer(format!(
5431                "module did not answer health.check within {probe_budget:?}"
5432            )))
5433        }
5434    }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn handle_health_report(
5439    spec: &ModuleSpec,
5440    runtime: &SupervisorRuntimeConfig,
5441    registry: &Registry,
5442    process_liveness: &SupervisorProcessLiveness,
5443    snapshot: &SharedSnapshot,
5444    child: &mut Option<SupervisedChild>,
5445    report: HealthReport,
5446    now_ms: u64,
5447) {
5448    let status = supervisor_health_status(report.status);
5449    let detail = report.detail.clone();
5450    let metrics = truncate_health_metrics(report.metrics);
5451    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5452        state.health.status = status;
5453        state.health.last_probe_ms = Some(now_ms);
5454        state.health.detail = detail.clone();
5455        state.health.metrics = metrics.clone();
5456        state.health.consecutive_failures = 0;
5457        state.failed_retry.observe_health(
5458            report.status == HealthStatus::Ok,
5459            runtime.restart_policy.window,
5460        );
5461    });
5462
5463    let action = match report.status {
5464        HealthStatus::Ok => return,
5465        HealthStatus::Degraded => runtime.health.on_degraded,
5466        HealthStatus::Failing => runtime.health.on_failing,
5467    };
5468    apply_l3_health_action(
5469        spec,
5470        runtime,
5471        registry,
5472        process_liveness,
5473        snapshot,
5474        child,
5475        status,
5476        detail.as_deref(),
5477        action,
5478        now_ms,
5479    )
5480    .await;
5481}
5482
5483#[allow(clippy::too_many_arguments)]
5484async fn handle_health_probe_failure(
5485    spec: &ModuleSpec,
5486    runtime: &SupervisorRuntimeConfig,
5487    registry: &Registry,
5488    process_liveness: &SupervisorProcessLiveness,
5489    snapshot: &SharedSnapshot,
5490    child: &mut Option<SupervisedChild>,
5491    err: HealthProbeError,
5492    now_ms: u64,
5493) {
5494    let threshold = runtime.health.failure_threshold.max(1);
5495    let mut failures = 0;
5496    // Carry the evidence class into the operator-visible detail. Without it,
5497    // "module did not answer within 5s" and "the control lane is gone" are two
5498    // prose strings in the same field, and the reader has to know the codebase to
5499    // tell which one is proof of anything.
5500    let detail = format!("[{}] {err}", err.label());
5501    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5502        state
5503            .failed_retry
5504            .observe_health(false, runtime.restart_policy.window);
5505        // A failed wire probe invalidates the last report, even before the
5506        // restart threshold. HTTP probes already mark failures as Failing.
5507        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5508            state.health.status = SupervisorHealthStatus::Unknown;
5509        }
5510        state.health.last_probe_ms = Some(now_ms);
5511        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5512        state.health.detail = Some(detail.clone());
5513        state.health.metrics = None;
5514        failures = state.health.consecutive_failures;
5515    });
5516
5517    if failures < threshold {
5518        warn!(
5519            module_id = %spec.module_id,
5520            consecutive_failures = failures,
5521            threshold,
5522            evidence = err.label(),
5523            detail = %detail,
5524            "health.check probe failed"
5525        );
5526        return;
5527    }
5528
5529    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5530        state.state = ModuleState::Unresponsive;
5531        state.health.status = SupervisorHealthStatus::Unresponsive;
5532    });
5533    // The evidence class is logged at the kill site because this is the line an
5534    // operator reads after an unexplained restart. A streak of `no-answer` under
5535    // machine load is the known false-positive shape; a `lane-dead` is not.
5536    if runtime.health.critical {
5537        error!(
5538            module_id = %spec.module_id,
5539            status = "unresponsive",
5540            evidence = err.label(),
5541            detail = %detail,
5542            "critical module health alert"
5543        );
5544    } else {
5545        warn!(
5546            module_id = %spec.module_id,
5547            status = "unresponsive",
5548            evidence = err.label(),
5549            detail = %detail,
5550            "module health threshold breached"
5551        );
5552    }
5553    if let Err(err) = health_restart_child(
5554        spec,
5555        runtime,
5556        registry,
5557        process_liveness,
5558        snapshot,
5559        child,
5560        SupervisorHealthStatus::Unresponsive,
5561        Some(&detail),
5562        now_ms,
5563    )
5564    .await
5565    {
5566        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5567    }
5568}
5569
5570#[allow(clippy::too_many_arguments)]
5571async fn apply_l3_health_action(
5572    spec: &ModuleSpec,
5573    runtime: &SupervisorRuntimeConfig,
5574    registry: &Registry,
5575    process_liveness: &SupervisorProcessLiveness,
5576    snapshot: &SharedSnapshot,
5577    child: &mut Option<SupervisedChild>,
5578    status: SupervisorHealthStatus,
5579    detail: Option<&str>,
5580    action: HealthAction,
5581    now_ms: u64,
5582) {
5583    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5584    match action {
5585        HealthAction::Report => {
5586            info!(
5587                module_id = %spec.module_id,
5588                status = ?status,
5589                detail,
5590                "module reported non-ok health"
5591            );
5592        }
5593        HealthAction::Alert => {
5594            error!(
5595                module_id = %spec.module_id,
5596                status = ?status,
5597                detail,
5598                "module health alert"
5599            );
5600        }
5601        HealthAction::Restart => {
5602            if let Err(err) = health_restart_child(
5603                spec,
5604                runtime,
5605                registry,
5606                process_liveness,
5607                snapshot,
5608                child,
5609                status,
5610                detail,
5611                now_ms,
5612            )
5613            .await
5614            {
5615                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5616            }
5617        }
5618    }
5619}
5620
5621#[allow(clippy::too_many_arguments)]
5622async fn health_restart_child(
5623    spec: &ModuleSpec,
5624    runtime: &SupervisorRuntimeConfig,
5625    registry: &Registry,
5626    process_liveness: &SupervisorProcessLiveness,
5627    snapshot: &SharedSnapshot,
5628    child: &mut Option<SupervisedChild>,
5629    status: SupervisorHealthStatus,
5630    detail: Option<&str>,
5631    now_ms: u64,
5632) -> Result<(), SuperviseError> {
5633    let (enabled, schedule) = {
5634        let mut state = lock_snapshot(snapshot)?;
5635        let enabled = state.enabled;
5636        let schedule = if enabled {
5637            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5638        } else {
5639            None
5640        };
5641        (enabled, schedule)
5642    };
5643
5644    if !enabled {
5645        return Err(SuperviseError::Disabled {
5646            module_id: spec.module_id.clone(),
5647        });
5648    }
5649
5650    if schedule.is_none() {
5651        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5652        error!(
5653            module_id = %spec.module_id,
5654            status = ?status,
5655            detail,
5656            max_restarts = runtime.restart_policy.max_restarts,
5657            window_secs = runtime.restart_policy.window.as_secs(),
5658            reason = %runtime.restart_policy.budget_exhausted_detail(),
5659            "health restart budget exhausted; marking module failed"
5660        );
5661        let stop_notice = begin_forwarding_drain_if_configured(
5662            spec,
5663            runtime,
5664            registry,
5665            snapshot,
5666            Some(true),
5667            RouteCloseReason::Disable,
5668        )
5669        .await?;
5670        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5671            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5672        })?;
5673        let result = drain_optional_child(
5674            &spec.module_id,
5675            spec.protocol,
5676            stop_notice,
5677            registry,
5678            runtime.forwarding.as_deref(),
5679            snapshot,
5680            &runtime.terminal_ring,
5681            &runtime.spawn_events,
5682            child,
5683            runtime.drain_timeout,
5684            ModuleState::Failed,
5685            Some(true),
5686            &runtime.failure_context(true),
5687        )
5688        .await;
5689        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5690            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5691                state.failed_retry.schedule(&spec.module_id);
5692            }
5693        })?;
5694        result?;
5695        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5696        return Ok(());
5697    }
5698
5699    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5700    let mut restart_count = 0;
5701    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5702        restart_count = state.crash_restarts.len();
5703        state.state = ModuleState::Unresponsive;
5704        state.health.status = status;
5705        state.health.last_action = Some(HealthAction::Restart.to_string());
5706        state.health.last_action_ms = Some(now_ms);
5707    })?;
5708    warn!(
5709        module_id = %spec.module_id,
5710        status = ?status,
5711        detail,
5712        restart_count,
5713        restart_in_window = schedule.restart_in_window,
5714        delay_ms = schedule.delay.as_millis() as u64,
5715        "health-triggered module restart"
5716    );
5717
5718    let stop_notice = begin_forwarding_drain_if_configured(
5719        spec,
5720        runtime,
5721        registry,
5722        snapshot,
5723        Some(true),
5724        RouteCloseReason::Restart,
5725    )
5726    .await?;
5727    drain_optional_child(
5728        &spec.module_id,
5729        spec.protocol,
5730        stop_notice,
5731        registry,
5732        runtime.forwarding.as_deref(),
5733        snapshot,
5734        &runtime.terminal_ring,
5735        &runtime.spawn_events,
5736        child,
5737        runtime.drain_timeout,
5738        ModuleState::Restarting,
5739        Some(true),
5740        &runtime.failure_context(true),
5741    )
5742    .await?;
5743    schedule_respawn(
5744        runtime,
5745        snapshot,
5746        &spec.module_id,
5747        schedule.delay,
5748        RespawnKind::Spawn,
5749        true,
5750    )
5751}
5752
5753fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5754    if let Some(reply) = runtime
5755        .deferred_reload_reply
5756        .lock()
5757        .unwrap_or_else(|p| p.into_inner())
5758        .take()
5759    {
5760        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5761            module_id: module_id.to_string(),
5762            reason: reason.to_string(),
5763        }));
5764    }
5765}
5766
5767fn schedule_respawn(
5768    runtime: &SupervisorRuntimeConfig,
5769    snapshot: &SharedSnapshot,
5770    module_id: &str,
5771    delay: Duration,
5772    kind: RespawnKind,
5773    daemon_initiated: bool,
5774) -> Result<(), SuperviseError> {
5775    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5776    update_snapshot(snapshot, Some(module_id), |state| {
5777        state.respawn_pending = true
5778    })?;
5779    *runtime
5780        .scheduled_respawn
5781        .lock()
5782        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5783        deadline: Instant::now() + delay,
5784        kind,
5785        daemon_initiated,
5786    });
5787    Ok(())
5788}
5789
5790fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5791    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5792        state.health.last_action = Some(action);
5793        state.health.last_action_ms = Some(now_ms);
5794    });
5795}
5796
5797fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5798    match status {
5799        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5800        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5801        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5802    }
5803}
5804
5805/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5806/// returned to every `supervisor.list` and `supervisor.health` caller.
5807///
5808/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5809/// path: that request exists to return a module's complete metrics object, and
5810/// `ck health <module-id>` documents it as the way to see what the cached view
5811/// truncates. The asymmetry is the feature.
5812///
5813/// So a new caller must decide which side it is on rather than assume the cap is
5814/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5815/// the truncation that path exists to avoid.
5816fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5817    let metrics = metrics?;
5818    match serde_json::to_vec(&metrics) {
5819        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5820            "truncated": true,
5821            "original_bytes": encoded.len(),
5822        })),
5823        Ok(_) | Err(_) => Some(metrics),
5824    }
5825}
5826
5827/// Spread health probes so a fleet-wide restart does not converge them.
5828///
5829/// The delay is derived from the module id and probe index rather than a random
5830/// source, so it is deterministic per module: a module keeps its own offset
5831/// across daemon restarts instead of re-rolling into a collision.
5832fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5833    if cadence.is_zero() {
5834        return Duration::ZERO;
5835    }
5836    let cadence_ms = cadence.as_millis() as u64;
5837    // This early return is REDUNDANT, deliberately, and a mutation run will show
5838    // it surviving removal. Recording why here so the next person to notice does
5839    // not have to re-derive it:
5840    //
5841    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5842    //   a zero cadence and builds the Duration from whole milliseconds, so a
5843    //   sub-millisecond cadence cannot come from config.
5844    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5845    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5846    //   -- exactly what this returns.
5847    //
5848    // Kept as a guard against a future widening of the config parser (accepting
5849    // microseconds, say), which would make the sub-millisecond case reachable.
5850    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5851    // divides by zero. Remove this and nothing changes.
5852    if cadence_ms == 0 {
5853        return cadence;
5854    }
5855    // Note that this never returns less than one cadence, including for the FIRST
5856    // probe. So a freshly registered module reports health `unknown` for a full
5857    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5858    // ready to answer.
5859    //
5860    // That is a property of the supervisor's schedule, not of any module: an
5861    // operator watching a restart sees `unknown` and cannot tell it from a module
5862    // that is slow to warm. Measured on two unrelated modules, both flipping to
5863    // `ok` between 22s and 32s after restart.
5864    //
5865    // Left as-is because spreading the first probe is what keeps a fleet-wide
5866    // restart from firing fourteen simultaneous probes into a cold machine. The
5867    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5868    // that thundering herd for a faster first reading.
5869    let jitter_span = (cadence_ms / 10).max(1);
5870    let hash = module_id.as_bytes().iter().fold(
5871        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5872        |acc, byte| {
5873            acc.wrapping_mul(1099511628211)
5874                .wrapping_add(u64::from(*byte))
5875        },
5876    );
5877    cadence + Duration::from_millis(hash % jitter_span)
5878}
5879
5880#[cfg(test)]
5881mod tests {
5882    use super::*;
5883
5884    #[test]
5885    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5886        let handle = SupervisorHandle::new();
5887        let module_id = "readded-tombstone";
5888        handle.record_rescan_removal(module_id);
5889        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5890
5891        handle.apply_identity_configuration(&ModuleSpec {
5892            module_id: module_id.to_string(),
5893            program: PathBuf::from("/test/module"),
5894            args: Vec::new(),
5895            env: Vec::new(),
5896            reserved: false,
5897            reserved_prefixes: Vec::new(),
5898            protocol: ModuleProtocol::Subc,
5899            overlap: Default::default(),
5900            carry_containment: false,
5901        });
5902
5903        assert!(
5904            handle.removal_tombstone_age_ms(module_id).is_none(),
5905            "a re-added module must not retain a stale removal tombstone"
5906        );
5907    }
5908
5909    /// What one module's owner looked like from the control plane at the
5910    /// instant after its first process was spawned.
5911    #[derive(Debug, PartialEq, Eq)]
5912    struct OwnerInSpawnWindow {
5913        module_id: String,
5914        configured: bool,
5915        on_roster: bool,
5916        admission_refusal: Option<&'static str>,
5917    }
5918
5919    /// A supervised module's process can connect, register, sync its scopes
5920    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5921    /// the module on the roster until afterward; configured supervision does
5922    /// so before spawning. In both cases the owner must
5923    /// already read as configured, so a scoped `route.open` against it is
5924    /// refused as retryable `scope_not_synced` and not as terminal
5925    /// `scope_not_live` ("will never sync").
5926    ///
5927    /// The hook runs just after each initial spawn attempt, before returning to
5928    /// the caller, so no race with a real child's registration is needed.
5929    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5930    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5931        use crate::scopes::ScopeTable;
5932        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5933
5934        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5935        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5936            module_id: module_id.to_string(),
5937            program,
5938            args: Vec::new(),
5939            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5940                .into_iter()
5941                .map(|key| (key.to_string(), dir.path().display().to_string()))
5942                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5943                .collect(),
5944            reserved: false,
5945            reserved_prefixes: Vec::new(),
5946            protocol: ModuleProtocol::Subc,
5947            overlap: Default::default(),
5948            carry_containment: false,
5949        };
5950        let live = super::terminal_history_tests::fake_aft_stub_path();
5951        let missing = dir.path().join("definitely-missing-module");
5952
5953        let handle = SupervisorHandle::new();
5954        let mut supervisor =
5955            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5956                .with_handle(handle.clone());
5957        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5958        let hook_handle = handle.clone();
5959        let hook_observed = Arc::clone(&observed);
5960        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5961            // Exactly what the control plane computes for a scoped route.open
5962            // naming this module as the owner of a scope it has not synced.
5963            let configured = hook_handle.is_configured(module_id);
5964            let selector = ScopeSelector::new(
5965                Principal::Reserved {
5966                    module_id: module_id.to_string(),
5967                },
5968                "s".to_string(),
5969                Some(1),
5970            );
5971            let carrier = Principal::Reserved {
5972                module_id: "carrier".to_string(),
5973            };
5974            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5975                .admit(&carrier, module_id, &selector, configured)
5976            {
5977                Ok(_) => None,
5978                Err(refusal) => Some(refusal.code),
5979            };
5980            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5981                module_id: module_id.to_string(),
5982                configured,
5983                on_roster: hook_handle.get(module_id).is_some(),
5984                admission_refusal,
5985            });
5986        })));
5987
5988        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5989        let configured = supervisor
5990            .supervise_configured(stub("configured", live.clone()), true)
5991            .unwrap();
5992        let with_health = supervisor
5993            .supervise_configured_with_health(
5994                stub("with-health", live.clone()),
5995                true,
5996                HealthConfig::default(),
5997                None,
5998                RestartPolicy::default(),
5999            )
6000            .unwrap();
6001        // The failed-spawn path still puts the module on the roster (as
6002        // failed), so it is configured throughout.
6003        let failed = supervisor
6004            .supervise_configured_with_health(
6005                stub("failed-spawn", missing.clone()),
6006                true,
6007                HealthConfig::default(),
6008                None,
6009                RestartPolicy::default(),
6010            )
6011            .unwrap();
6012        // A failed plain `spawn` puts nothing on the roster, so its mark is
6013        // taken back once the spawn has failed.
6014        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6015
6016        let expected = [
6017            "plain",
6018            "configured",
6019            "with-health",
6020            "failed-spawn",
6021            "spawn-error",
6022        ]
6023        .into_iter()
6024        .map(|module_id| OwnerInSpawnWindow {
6025            module_id: module_id.to_string(),
6026            configured: true,
6027            on_roster: !matches!(module_id, "plain" | "spawn-error"),
6028            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6029        })
6030        .collect::<Vec<_>>();
6031        assert_eq!(*observed.lock().unwrap(), expected);
6032
6033        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6034            assert!(
6035                handle.get(module_id).is_some(),
6036                "{module_id} is on the roster"
6037            );
6038            assert!(
6039                handle.is_configured(module_id),
6040                "{module_id} stays configured"
6041            );
6042        }
6043        assert!(handle.get("spawn-error").is_none());
6044        assert!(
6045            !handle.is_configured("spawn-error"),
6046            "a plain spawn that failed must not leave its module marked configured"
6047        );
6048
6049        // Leaving the roster clears the mark with it.
6050        handle.retire("failed-spawn");
6051        assert!(!handle.is_configured("failed-spawn"));
6052
6053        for module in [plain, configured, with_health] {
6054            module.stop().await.unwrap();
6055        }
6056        drop(failed);
6057    }
6058
6059    /// A configured module reads as starting until the supervisor has admitted
6060    /// its first process (exec confirmed, identity durably recorded), even
6061    /// though that process is already running by then and can register over
6062    /// HELLO, which is what lists it in catalog.list. So a client that needs
6063    /// the supervisor state must wait for that state; seeing the module in
6064    /// the catalog does not mean supervisor.list already says running.
6065    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6066    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6067        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6068        let spec = ModuleSpec {
6069            module_id: "spawned-not-admitted".to_string(),
6070            program: super::terminal_history_tests::fake_aft_stub_path(),
6071            args: Vec::new(),
6072            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6073                .into_iter()
6074                .map(|key| (key.to_string(), dir.path().display().to_string()))
6075                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6076                .collect(),
6077            reserved: false,
6078            reserved_prefixes: Vec::new(),
6079            protocol: ModuleProtocol::Subc,
6080            overlap: Default::default(),
6081            carry_containment: false,
6082        };
6083        let handle = SupervisorHandle::new();
6084        let mut supervisor =
6085            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6086                .with_handle(handle.clone());
6087        let observed = Arc::new(Mutex::new(None));
6088        let hook_handle = handle.clone();
6089        let hook_observed = Arc::clone(&observed);
6090        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6091            let state = hook_handle
6092                .get(module_id)
6093                .expect("a configured module is on the roster before it is spawned")
6094                .state()
6095                .unwrap();
6096            *hook_observed.lock().unwrap() = Some(state);
6097        })));
6098
6099        let module = supervisor.supervise_configured(spec, true).unwrap();
6100        assert_eq!(
6101            *observed.lock().unwrap(),
6102            Some(ModuleState::Starting),
6103            "between the spawn and its admission the supervisor still reports starting"
6104        );
6105        assert_eq!(module.state().unwrap(), ModuleState::Running);
6106        module.stop().await.unwrap();
6107    }
6108
6109    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6110        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6111        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6112            snapshot.process_alive = true;
6113            snapshot.pid = Some(41);
6114            snapshot.spawned_at_ms = Some(42);
6115            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6116            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6117            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6118                device: 43,
6119                inode: 44,
6120                #[cfg(windows)]
6121                windows_image: None,
6122            });
6123        })
6124        .unwrap();
6125        snapshot
6126    }
6127
6128    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6129        let snapshot = lock_snapshot(snapshot).unwrap();
6130        assert!(!snapshot.process_alive);
6131        assert_eq!(snapshot.pid, None);
6132        assert_eq!(snapshot.spawned_at_ms, None);
6133        assert_eq!(snapshot.spawned_from, None);
6134        assert_eq!(snapshot.launch_spec_digest, None);
6135        assert_eq!(snapshot.spawned_file_identity, None);
6136    }
6137
6138    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6139    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6140        let supervisor =
6141            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6142        let mut runtime = supervisor.runtime_config();
6143        runtime.test_seed_stale_facts_before_enable_spawn = true;
6144        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6145        let mut child = None;
6146        let spec = ModuleSpec {
6147            module_id: "failed-enable-clears-facts".to_string(),
6148            program: PathBuf::from("/definitely/missing/failed-enable-module"),
6149            args: Vec::new(),
6150            env: Vec::new(),
6151            reserved: false,
6152            reserved_prefixes: Vec::new(),
6153            protocol: ModuleProtocol::Subc,
6154            overlap: Default::default(),
6155            carry_containment: false,
6156        };
6157
6158        let result = set_child_enabled(
6159            &spec,
6160            &runtime,
6161            &supervisor.registry,
6162            &supervisor.process_liveness,
6163            &snapshot,
6164            &mut child,
6165            true,
6166            &runtime.failure_context(false),
6167        )
6168        .await;
6169
6170        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6171        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6172        assert_snapshot_process_facts_cleared(&snapshot);
6173    }
6174
6175    #[tokio::test]
6176    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6177        let supervisor =
6178            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6179        let runtime = supervisor.runtime_config();
6180        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6181            ModuleState::Restarting,
6182            true,
6183        )));
6184        let spec = ModuleSpec {
6185            module_id: "start-stranded-restarting".to_string(),
6186            program: super::terminal_history_tests::fake_aft_stub_path(),
6187            args: Vec::new(),
6188            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6189            reserved: false,
6190            reserved_prefixes: Vec::new(),
6191            protocol: ModuleProtocol::None,
6192            overlap: Default::default(),
6193            carry_containment: false,
6194        };
6195        let mut child = None;
6196        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6197        assert!(!super::set_child_enabled(
6198            &spec,
6199            &runtime,
6200            &Registry::default(),
6201            &supervisor.process_liveness,
6202            &snapshot,
6203            &mut child,
6204            true,
6205            &runtime.failure_context(false),
6206        )
6207        .await
6208        .unwrap());
6209        assert!(child.is_none());
6210        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6211        assert!(super::set_child_enabled(
6212            &spec,
6213            &runtime,
6214            &Registry::default(),
6215            &supervisor.process_liveness,
6216            &snapshot,
6217            &mut child,
6218            true,
6219            &runtime.failure_context(false),
6220        )
6221        .await
6222        .unwrap());
6223        assert_eq!(
6224            lock_snapshot(&snapshot).unwrap().state,
6225            ModuleState::Running
6226        );
6227        let mut child = child.unwrap();
6228        child.start_kill().unwrap();
6229        child.wait().await.unwrap();
6230    }
6231
6232    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6233    async fn failed_reload_spawn_clears_current_process_facts() {
6234        let supervisor =
6235            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6236        let mut runtime = supervisor.runtime_config();
6237        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6238        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6239        let mut child = None;
6240        let spec = ModuleSpec {
6241            module_id: "failed-reload-clears-facts".to_string(),
6242            program: PathBuf::from("/unused/failed-reload-module"),
6243            args: Vec::new(),
6244            env: Vec::new(),
6245            reserved: false,
6246            reserved_prefixes: Vec::new(),
6247            protocol: ModuleProtocol::Subc,
6248            overlap: Default::default(),
6249            carry_containment: false,
6250        };
6251
6252        let result = handle_reload_spawn_failure(
6253            &spec,
6254            &runtime,
6255            &supervisor.process_liveness,
6256            &snapshot,
6257            &mut child,
6258            "forced reload spawn failure".to_string(),
6259        )
6260        .await;
6261
6262        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6263        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6264        assert_snapshot_process_facts_cleared(&snapshot);
6265    }
6266
6267    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6268    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6269        let supervisor =
6270            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6271        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6272        let module = supervisor.supervised_module(
6273            ModuleSpec {
6274                module_id: "drop-clears-facts".to_string(),
6275                program: PathBuf::from("/unused/drop-module"),
6276                args: Vec::new(),
6277                env: Vec::new(),
6278                reserved: false,
6279                reserved_prefixes: Vec::new(),
6280                protocol: ModuleProtocol::Subc,
6281                overlap: Default::default(),
6282                carry_containment: false,
6283            },
6284            supervisor.runtime_config(),
6285            Arc::clone(&snapshot),
6286            None,
6287        );
6288        assert!(!module
6289            .inner
6290            .monitor
6291            .lock()
6292            .unwrap()
6293            .as_ref()
6294            .unwrap()
6295            .is_finished());
6296
6297        drop(module);
6298
6299        assert_eq!(
6300            lock_snapshot(&snapshot).unwrap().state,
6301            ModuleState::Stopped
6302        );
6303        assert_snapshot_process_facts_cleared(&snapshot);
6304    }
6305
6306    #[cfg(unix)]
6307    #[tokio::test]
6308    async fn rescan_preserves_running_protocol_until_respawn() {
6309        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6310        let initial = ModuleSpec {
6311            module_id: "rescan-protocol".into(),
6312            program: PathBuf::from("/bin/sleep"),
6313            args: vec!["60".into()],
6314            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6315                .into_iter()
6316                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6317                .collect(),
6318            reserved: false,
6319            reserved_prefixes: vec![],
6320            protocol: ModuleProtocol::None,
6321            overlap: Default::default(),
6322            carry_containment: false,
6323        };
6324        let supervisor =
6325            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6326        let module = supervisor.spawn(initial.clone()).unwrap();
6327        assert!(module.status().unwrap().live);
6328        let mut next = initial;
6329        next.protocol = ModuleProtocol::Subc;
6330        module
6331            .update_configuration(next.clone(), HealthConfig::default(), None)
6332            .await
6333            .unwrap();
6334        assert!(
6335            module.status().unwrap().live,
6336            "rescan must not require HELLO from the old non-wire process"
6337        );
6338        let runtime = supervisor.runtime_config();
6339        let action = on_child_exit(
6340            &next,
6341            RestartPolicy::default(),
6342            &supervisor.registry,
6343            &module.inner.snapshot,
6344            &runtime.terminal_ring,
6345            &runtime.spawn_events,
6346            &runtime.child_roster,
6347            ExitReport {
6348                kind: ExitKind::Clean,
6349                code: Some(0),
6350                signal: None,
6351                at_ms: unix_ms_now(),
6352            },
6353            true,
6354        )
6355        .await;
6356        assert!(
6357            matches!(action, NextAction::Restart { .. }),
6358            "the old non-wire process's clean exit must restart"
6359        );
6360        module.drain().await.unwrap();
6361    }
6362
6363    #[cfg(unix)]
6364    #[tokio::test]
6365    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6366        use std::os::unix::fs::PermissionsExt;
6367        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6368        let script = dir.join("module.sh");
6369        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6370        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6371        let record_path = dir.join("live-children.json");
6372        let supervisor =
6373            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6374                .with_live_children_record(&record_path);
6375        for (program, args) in [
6376            (PathBuf::from("sleep"), vec!["60".into()]),
6377            (script, vec![]),
6378        ] {
6379            let spec = ModuleSpec {
6380                module_id: "image-identity".into(),
6381                program: program.clone(),
6382                args,
6383                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6384                    .into_iter()
6385                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6386                    .collect(),
6387                reserved: false,
6388                reserved_prefixes: vec![],
6389                protocol: ModuleProtocol::None,
6390                overlap: Default::default(),
6391                carry_containment: false,
6392            };
6393            let module = supervisor.spawn(spec).unwrap();
6394            #[cfg(target_os = "macos")]
6395            {
6396                // SETEXEC confirmation is asynchronous; the orphan record must
6397                // identify the final image, never the intermediate trampoline.
6398                let deadline = Instant::now() + Duration::from_secs(5);
6399                while crate::live_children::read_record(&record_path)
6400                    .unwrap()
6401                    .iter()
6402                    .all(|entry| entry.executable.is_none())
6403                {
6404                    assert!(Instant::now() < deadline, "module image was not confirmed");
6405                    tokio::time::sleep(Duration::from_millis(5)).await;
6406                }
6407            }
6408            let entry = crate::live_children::read_record(&record_path)
6409                .unwrap()
6410                .pop()
6411                .unwrap();
6412            let observed = subc_os::Process::open(entry.pid)
6413                .unwrap()
6414                .unwrap()
6415                .observe()
6416                .unwrap();
6417            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6418            module.drain().await.unwrap();
6419            assert_eq!(
6420                verdict,
6421                crate::live_children::IdentityVerdict::Matches,
6422                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6423            );
6424        }
6425    }
6426
6427    #[cfg(unix)]
6428    fn http_fixture(
6429        dir: &std::path::Path,
6430        url: &str,
6431        threshold: u32,
6432    ) -> crate::daemon_config::ConfiguredModule {
6433        let path = dir.join("subc.jsonc");
6434        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6435            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6436            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6437            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6438            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6439        }}}).to_string()).unwrap();
6440        crate::daemon_config::load(&path)
6441            .unwrap()
6442            .unwrap()
6443            .modules
6444            .pop()
6445            .unwrap()
6446    }
6447
6448    #[cfg(unix)]
6449    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6450        timeout(Duration::from_secs(5), async {
6451            loop {
6452                if module.status().unwrap().health.status == status {
6453                    break;
6454                }
6455                sleep(Duration::from_millis(5)).await;
6456            }
6457        })
6458        .await
6459        .unwrap_or_else(|_| {
6460            panic!(
6461                "expected {status:?}, got {:?}",
6462                module.status().unwrap().health
6463            )
6464        });
6465    }
6466
6467    #[cfg(unix)]
6468    #[tokio::test]
6469    async fn http_health_status_flips_ok_failing_ok() {
6470        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6471        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6472        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6473        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6474        let serving_status = status.clone();
6475        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6476        let server = tokio::spawn(async move {
6477            loop {
6478                let (mut stream, _) = listener.accept().await.unwrap();
6479                let mut request = [0u8; 2048];
6480                let count = stream.read(&mut request).await.unwrap();
6481                assert!(count > 0, "a probe must send an HTTP request");
6482                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6483                let body = if code == 200 {
6484                    "ready"
6485                } else {
6486                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6487                };
6488                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6489                let _ = stream.write_all(response.as_bytes()).await;
6490            }
6491        });
6492        let configured = http_fixture(&dir, &url, 1000);
6493        let module =
6494            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6495                .supervise_configured_with_health(
6496                    configured.module_spec(),
6497                    true,
6498                    configured.health,
6499                    configured.drain_timeout_ms,
6500                    configured.restart,
6501                )
6502                .unwrap();
6503        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6504        status.store(503, std::sync::atomic::Ordering::SeqCst);
6505        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6506        assert!(module
6507            .status()
6508            .unwrap()
6509            .health
6510            .detail
6511            .unwrap()
6512            .contains("scratch failure"));
6513        status.store(200, std::sync::atomic::Ordering::SeqCst);
6514        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6515        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6516        let before = module.status().unwrap();
6517        let (spec, mut health) = module.configuration().unwrap();
6518        health.http = None;
6519        module
6520            .update_configuration(spec.clone(), health.clone(), Some(10))
6521            .await
6522            .unwrap();
6523        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6524        health.http = Some(url);
6525        module
6526            .update_configuration(spec, health, Some(10))
6527            .await
6528            .unwrap();
6529        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6530        assert_eq!(
6531            module.status().unwrap().pid,
6532            before.pid,
6533            "changing a probe must apply live, not restart its process"
6534        );
6535        let (spec, mut health) = module.configuration().unwrap();
6536        health.failure_threshold = 2;
6537        module
6538            .update_configuration(spec, health, Some(10))
6539            .await
6540            .unwrap();
6541        status.store(503, std::sync::atomic::Ordering::SeqCst);
6542        timeout(Duration::from_secs(5), async {
6543            while module.status().unwrap().spawn_generation == before.spawn_generation {
6544                sleep(Duration::from_millis(5)).await;
6545            }
6546        })
6547        .await
6548        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6549        module.drain().await.unwrap();
6550        server.abort();
6551    }
6552
6553    #[cfg(unix)]
6554    #[tokio::test]
6555    async fn http_health_replacement_recovers_after_one_restart() {
6556        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6557        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6558        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6559        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6560        let mut configured = http_fixture(&dir, &url, 2);
6561        configured.restart.max_restarts = 2;
6562        let module =
6563            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6564                .supervise_configured_with_health(
6565                    configured.module_spec(),
6566                    true,
6567                    configured.health,
6568                    configured.drain_timeout_ms,
6569                    configured.restart,
6570                )
6571                .unwrap();
6572        let first_generation = module.status().unwrap().spawn_generation;
6573        let observed = module.clone();
6574        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6575        let server = tokio::spawn(async move {
6576            let mut generation = first_generation;
6577            let mut probes = 0;
6578            loop {
6579                let (mut stream, _) = listener.accept().await.unwrap();
6580                let mut request = [0; 2048];
6581                assert!(stream.read(&mut request).await.unwrap() > 0);
6582                let current = observed.status().unwrap().spawn_generation;
6583                if current != generation {
6584                    generation = current;
6585                    probes = 0;
6586                }
6587                probes += 1;
6588                // The original process never recovers. Each replacement has one
6589                // transient miss, then answers successfully for its remaining life.
6590                let healthy = generation != first_generation && probes > 1;
6591                let code = if healthy { 200 } else { 503 };
6592                let response = format!(
6593                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6594                );
6595                let _ = stream.write_all(response.as_bytes()).await;
6596                if healthy {
6597                    healthy_tx.send(()).await.unwrap();
6598                }
6599            }
6600        });
6601        timeout(Duration::from_secs(5), async {
6602            let mut healthy_probes = 0;
6603            loop {
6604                if module.state().unwrap() == ModuleState::Failed {
6605                    break;
6606                }
6607                while healthy_rx.try_recv().is_ok() {
6608                    healthy_probes += 1;
6609                }
6610                if healthy_probes >= 3
6611                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6612                {
6613                    break;
6614                }
6615                sleep(Duration::from_millis(5)).await;
6616            }
6617        })
6618        .await
6619        .expect("the replacement must settle healthy without exhausting its restart budget");
6620        let status = module.status().unwrap();
6621        module.drain().await.unwrap();
6622        server.abort();
6623        assert_eq!(status.state, ModuleState::Running);
6624        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6625        assert_eq!(status.health.consecutive_failures, 0);
6626        assert_eq!(status.lifetime_restarts, 1);
6627        assert_eq!(status.restart_count, 1);
6628    }
6629
6630    #[cfg(unix)]
6631    #[tokio::test]
6632    async fn http_health_sustained_failure_exhausts_restart_budget() {
6633        use std::collections::BTreeMap;
6634        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6635        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6636        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6637        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6638        let mut configured = http_fixture(&dir, &url, 2);
6639        configured.restart.max_restarts = 2;
6640        let module =
6641            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6642                .supervise_configured_with_health(
6643                    configured.module_spec(),
6644                    true,
6645                    configured.health,
6646                    configured.drain_timeout_ms,
6647                    configured.restart,
6648                )
6649                .unwrap();
6650        let observed = module.clone();
6651        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6652        let served = probes.clone();
6653        let server = tokio::spawn(async move {
6654            loop {
6655                let (mut stream, _) = listener.accept().await.unwrap();
6656                let mut request = [0; 2048];
6657                assert!(stream.read(&mut request).await.unwrap() > 0);
6658                *served
6659                    .lock()
6660                    .unwrap()
6661                    .entry(observed.status().unwrap().spawn_generation)
6662                    .or_default() += 1;
6663                let _ = stream
6664                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6665                    .await;
6666            }
6667        });
6668        timeout(Duration::from_secs(5), async {
6669            while module.state().unwrap() != ModuleState::Failed {
6670                sleep(Duration::from_millis(5)).await;
6671            }
6672        })
6673        .await
6674        .expect("sustained failures must still exhaust the health restart budget");
6675        let status = module.status().unwrap();
6676        server.abort();
6677        assert_eq!(status.restart_count, 2);
6678        assert_eq!(status.lifetime_restarts, 2);
6679        assert!(!status.process_alive);
6680        assert_eq!(
6681            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6682            vec![2, 2, 2]
6683        );
6684    }
6685
6686    #[cfg(unix)]
6687    #[tokio::test]
6688    async fn spawn_clears_the_previous_process_health() {
6689        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6690        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6691        configured.health.cadence = Duration::from_secs(3600);
6692        let module =
6693            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6694                .supervise_configured_with_health(
6695                    configured.module_spec(),
6696                    false,
6697                    configured.health,
6698                    configured.drain_timeout_ms,
6699                    configured.restart,
6700                )
6701                .unwrap();
6702        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6703            state.health.status = SupervisorHealthStatus::Unresponsive;
6704            state.health.consecutive_failures = 3;
6705            state.health.detail = Some("old process refused connections".into());
6706            state.health.metrics = Some(serde_json::json!({"old": true}));
6707            state.health.last_probe_ms = Some(123);
6708            state.health.last_action = Some("restart".into());
6709            state.health.last_action_ms = Some(124);
6710        })
6711        .unwrap();
6712        module.set_enabled(true).await.unwrap();
6713        let health = module.status().unwrap().health;
6714        module.drain().await.unwrap();
6715        assert_eq!(health.consecutive_failures, 0);
6716        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6717        assert_eq!(health.detail, None);
6718        assert_eq!(health.metrics, None);
6719        assert_eq!(health.last_probe_ms, None);
6720        assert_eq!(health.last_action.as_deref(), Some("restart"));
6721        assert_eq!(health.last_action_ms, Some(124));
6722    }
6723
6724    #[cfg(unix)]
6725    #[tokio::test]
6726    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6727        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6728        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6729        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6730        drop(unused);
6731        let configured = http_fixture(&dir, &url, 2);
6732        let module =
6733            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6734                .supervise_configured_with_health(
6735                    configured.module_spec(),
6736                    true,
6737                    configured.health,
6738                    configured.drain_timeout_ms,
6739                    configured.restart,
6740                )
6741                .unwrap();
6742        let before = module.status().unwrap().spawn_generation;
6743        timeout(Duration::from_secs(5), async {
6744            loop {
6745                let status = module.status().unwrap();
6746                if status.spawn_generation > before {
6747                    assert!(status.lifetime_restarts > 0);
6748                    break;
6749                }
6750                sleep(Duration::from_millis(5)).await;
6751            }
6752        })
6753        .await
6754        .expect("sustained HTTP refusal must trigger the health restart policy");
6755        module.drain().await.unwrap();
6756    }
6757
6758    #[cfg(unix)]
6759    #[tokio::test]
6760    async fn http_health_timeout_honours_deadline() {
6761        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6762        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6763        let server = tokio::spawn(async move {
6764            let _held = listener.accept().await.unwrap();
6765            std::future::pending::<()>().await;
6766        });
6767        let error = timeout(
6768            Duration::from_secs(1),
6769            probe_http_health(&url, Duration::from_millis(10)),
6770        )
6771        .await
6772        .expect("the probe must enforce its own deadline")
6773        .unwrap_err();
6774        server.abort();
6775        assert!(error.to_string().contains("timed out"));
6776    }
6777
6778    #[cfg(unix)]
6779    #[tokio::test]
6780    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6781        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6782        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6783        let url = format!(
6784            "http://localhost:{}/healthz",
6785            listener.local_addr().unwrap().port()
6786        );
6787        let server = tokio::spawn(async move {
6788            let (mut stream, _) = listener.accept().await.unwrap();
6789            let mut request = [0u8; 2048];
6790            assert!(stream.read(&mut request).await.unwrap() > 0);
6791            stream
6792                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6793                .await
6794                .unwrap();
6795        });
6796        // The deadline only bounds a hang. A probe that never tried the IPv6
6797        // address would be refused on 127.0.0.1 and fail at once, so a longer
6798        // deadline does not weaken the assertion; one second timed out under a
6799        // loaded parallel test run.
6800        assert_eq!(
6801            probe_http_health(&url, Duration::from_secs(10))
6802                .await
6803                .unwrap()
6804                .status,
6805            HealthStatus::Ok
6806        );
6807        server.await.unwrap();
6808    }
6809
6810    #[cfg(unix)]
6811    #[tokio::test]
6812    async fn http_health_timeout_keeps_partial_status_and_body() {
6813        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6814        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6815        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6816        let server = tokio::spawn(async move {
6817            let (mut stream, _) = listener.accept().await.unwrap();
6818            let mut request = [0u8; 2048];
6819            assert!(stream.read(&mut request).await.unwrap() > 0);
6820            stream
6821                .write_all(
6822                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6823                )
6824                .await
6825                .unwrap();
6826            std::future::pending::<()>().await;
6827        });
6828        let error = probe_http_health(&url, Duration::from_secs(1))
6829            .await
6830            .unwrap_err()
6831            .to_string();
6832        server.abort();
6833        assert!(
6834            error.contains("timed out")
6835                && error.contains("503 Unavailable")
6836                && error.contains("partial diagnostic"),
6837            "{error}"
6838        );
6839    }
6840
6841    #[cfg(unix)]
6842    #[tokio::test]
6843    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6844        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6845        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6846        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6847        let server = tokio::spawn(async move {
6848            let (mut stream, _) = listener.accept().await.unwrap();
6849            let mut request = [0u8; 2048];
6850            assert!(stream.read(&mut request).await.unwrap() > 0);
6851            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6852            let response = format!(
6853                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6854                body.len()
6855            );
6856            stream.write_all(response.as_bytes()).await.unwrap();
6857        });
6858        let error = probe_http_health(&url, Duration::from_secs(1))
6859            .await
6860            .unwrap_err()
6861            .to_string();
6862        server.await.unwrap();
6863        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6864        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6865        assert!(!error.contains("not-in-diagnostic"));
6866    }
6867
6868    #[cfg(unix)]
6869    #[tokio::test]
6870    async fn http_health_real_nats_server_monitoring() {
6871        if std::process::Command::new("nats-server")
6872            .arg("--version")
6873            .env("XDG_DATA_HOME", std::env::temp_dir())
6874            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6875            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6876            .output()
6877            .is_err()
6878        {
6879            eprintln!(
6880                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6881            );
6882            return;
6883        }
6884        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6885        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6886        let port = monitor.local_addr().unwrap().port();
6887        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6888        let client_port = client.local_addr().unwrap().port();
6889        let config = dir.join("server.conf");
6890        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6891        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6892        configured.program = PathBuf::from("nats-server");
6893        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6894        drop(monitor);
6895        drop(client);
6896        let module =
6897            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6898                .supervise_configured_with_health(
6899                    configured.module_spec(),
6900                    true,
6901                    configured.health,
6902                    configured.drain_timeout_ms,
6903                    configured.restart,
6904                )
6905                .unwrap();
6906        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6907        module.drain().await.unwrap();
6908    }
6909
6910    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6911    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6912        let supervisor =
6913            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6914        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6915        let initial = ModuleSpec {
6916            module_id: "rescan-preserves-spawn-facts".to_string(),
6917            program: PathBuf::from("/spawned/module"),
6918            args: Vec::new(),
6919            env: Vec::new(),
6920            reserved: false,
6921            reserved_prefixes: Vec::new(),
6922            protocol: ModuleProtocol::Subc,
6923            overlap: Default::default(),
6924            carry_containment: false,
6925        };
6926        let module = supervisor.supervised_module(
6927            initial.clone(),
6928            supervisor.runtime_config(),
6929            snapshot,
6930            None,
6931        );
6932        let before = module.status().unwrap();
6933        let mut replacement = initial;
6934        replacement.program = PathBuf::from("/rescanned/replacement-module");
6935
6936        module
6937            .update_configuration(replacement, HealthConfig::default(), None)
6938            .await
6939            .unwrap();
6940
6941        let after = module.status().unwrap();
6942        assert_eq!(after.pid, before.pid);
6943        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6944        assert_eq!(after.spawned_from, before.spawned_from);
6945        drop(module);
6946    }
6947}
6948
6949fn unix_ms_now() -> u64 {
6950    SystemTime::now()
6951        .duration_since(UNIX_EPOCH)
6952        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6953        .unwrap_or(0)
6954}
6955
6956async fn supervise_loop(
6957    mut spec: ModuleSpec,
6958    mut runtime: SupervisorRuntimeConfig,
6959    registry: Arc<Registry>,
6960    process_liveness: Arc<SupervisorProcessLiveness>,
6961    snapshot: SharedSnapshot,
6962    mut child: Option<SupervisedChild>,
6963    mut commands: mpsc::Receiver<SupervisorCommand>,
6964) {
6965    let mut health_probe = HealthProbeRuntime::default();
6966    // Registry writes (including embedded callers) notify this module only.
6967    // Subscribe before the first refresh; watch retains changes that arrive
6968    // while commands or probes are running. No polling fallback is needed.
6969    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6970        Ok(changes) => Some(changes),
6971        Err(err) => {
6972            // A poisoned registry cannot accept further writes, so it cannot
6973            // recover via a timer. Keep exit and command handling alive.
6974            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6975            None
6976        }
6977    };
6978    // All restart backoffs run here, including health and operator requests.
6979    // While one is pending the loop serves commands, so disable or drain can
6980    // cancel the replacement without spawning a process just to stop it.
6981    let mut pending_respawn: Option<PendingRespawn> = None;
6982    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6983    // before anything else so a stop that interrupted a swap runs at once.
6984    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6985    let mut previous_instance_recheck = tokio::time::interval_at(
6986        Instant::now() + Duration::from_secs(5),
6987        Duration::from_secs(5),
6988    );
6989    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6990    loop {
6991        #[cfg(test)]
6992        {
6993            lock_snapshot(&snapshot).unwrap().actor_select = None;
6994        }
6995        #[cfg(target_os = "macos")]
6996        if let Some(active) = child.as_mut() {
6997            active.confirm_privacy_exec().await;
6998        }
6999        if let Some(scheduled) = runtime
7000            .scheduled_respawn
7001            .lock()
7002            .unwrap_or_else(|p| p.into_inner())
7003            .take()
7004        {
7005            runtime.recovery_is_daemon = scheduled.daemon_initiated;
7006            pending_respawn = Some(scheduled);
7007        }
7008        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7009            pending_respawn = None;
7010            cancel_deferred_reload(
7011                &runtime,
7012                &spec.module_id,
7013                "respawn cancelled by a supervisor command",
7014            );
7015        }
7016        if child.is_none()
7017            && pending_respawn.is_none()
7018            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7019        {
7020            cancel_deferred_reload(
7021                &runtime,
7022                &spec.module_id,
7023                "respawn cancelled before a replacement was spawned",
7024            );
7025            let mut stranded = false;
7026            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7027                state.respawn_pending = false;
7028                state.coalesced_restart_pending = false;
7029                if matches!(
7030                    state.state,
7031                    ModuleState::Restarting
7032                        | ModuleState::Starting
7033                        | ModuleState::Draining
7034                        | ModuleState::Unresponsive
7035                ) {
7036                    error!(module_id = %spec.module_id, state = ?state.state,
7037                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7038                    stranded = true;
7039                }
7040            });
7041            if stranded {
7042                fail_snapshot(
7043                    &snapshot,
7044                    Some(&spec.module_id),
7045                    None,
7046                    &runtime.failure_context(runtime.recovery_is_daemon),
7047                );
7048            }
7049        }
7050        if child.is_none()
7051            && (runtime.child_roster.is_closed()
7052                || pending_respawn.is_none()
7053                || !lock_snapshot(&snapshot)
7054                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7055        {
7056            containment::terminate(&runtime.child_roster.containment).await;
7057        }
7058        if let Some(command) = requeued.pop_front() {
7059            if !handle_supervisor_command(
7060                command,
7061                &mut spec,
7062                &mut runtime,
7063                &registry,
7064                &process_liveness,
7065                &snapshot,
7066                &mut child,
7067                &mut commands,
7068                &mut requeued,
7069            )
7070            .await
7071            {
7072                containment::terminate(&runtime.child_roster.containment).await;
7073                return;
7074            }
7075            if child.is_some() || !respawn_still_pending(&snapshot) {
7076                pending_respawn = None;
7077                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7078                    state.respawn_pending = false
7079                });
7080            }
7081            continue;
7082        }
7083        if child.is_some() {
7084            runtime.recovery_is_daemon = true;
7085            #[cfg(test)]
7086            {
7087                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7088            }
7089            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
7090            let active = child.as_mut().expect("child checked above");
7091            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7092                active.registration_deadline = None;
7093            }
7094            let registration_deadline = active.registration_deadline;
7095            let no_checks = runtime.health.http.is_none()
7096                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7097                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7098            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7099                state
7100                    .failed_retry
7101                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
7102            });
7103            let wake_at = match (health_probe.wake_at(), uptime_reset) {
7104                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7105                (probe, None) => probe,
7106            };
7107            let wake_at = match (wake_at, registration_deadline) {
7108                (Some(wake), Some(register)) => Some(wake.min(register)),
7109                (wake, register) => wake.or(register),
7110            };
7111            #[cfg(test)]
7112            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7113            let probe_sleep = supervisor_sleep(wake_at);
7114            tokio::pin!(probe_sleep);
7115            // A `protocol: "none"` child (a plain process that never registers
7116            // over the subc wire, such as nats-server) with no HTTP health
7117            // check has nothing to probe, so it waits only for its exit or a
7118            // supervisor command. A rescan that adds an HTTP check is a command.
7119            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7120            #[cfg(test)]
7121            {
7122                let registration_pending = wire_child
7123                    && registration_changes
7124                        .as_ref()
7125                        .is_some_and(|changes| changes.has_changed().unwrap());
7126                let mut state = lock_snapshot(&snapshot).unwrap();
7127                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7128                    generation: state.spawn_generation,
7129                    turn: state.actor_turns,
7130                    registered_connection: health_probe.registered_connection,
7131                    next_probe_at: health_probe.next_probe_at,
7132                    wake_after,
7133                });
7134            }
7135            let active_child = child.as_mut().expect("child checked above");
7136            tokio::select! {
7137                wait_result = active_child.wait() => {
7138                    if let Ok(mut state) = lock_snapshot(&snapshot) {
7139                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7140                    }
7141                    #[cfg(test)]
7142                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7143                    // Every arm below that gives up on the CHILD must keep the
7144                    // supervision task itself alive (child = None, loop
7145                    // continues into command-serving mode). Returning here
7146                    // closes the command channel, which makes the module
7147                    // permanently unrestartable in-band: a clean child exit
7148                    // of an enabled module once wedged the fleet this way
7149                    // ('supervisor command channel is closed') and required a
7150                    // full daemon restart to recover.
7151                    let exit_report = match wait_result {
7152                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7153                        Err(err) => {
7154                            active_child.drain_stderr(&spec.module_id).await;
7155                            fail_after_wait_error(
7156                                &spec.module_id,
7157                                &snapshot,
7158                                &runtime.terminal_ring,
7159                                &runtime.spawn_events,
7160                                format!("failed to wait for supervised module: {err}"),
7161                            &runtime.failure_context(true),
7162                            );
7163                            untrack_if_registration_released(
7164                                &process_liveness,
7165                                &registry,
7166                                &spec.module_id,
7167                                &snapshot,
7168                            );
7169                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7170                            child = None;
7171                            continue;
7172                        }
7173                    };
7174                    active_child.drain_stderr(&spec.module_id).await;
7175
7176                    let next = on_child_exit(
7177                        &spec,
7178                        runtime.restart_policy,
7179                        &registry,
7180                        &snapshot,
7181                        &runtime.terminal_ring,
7182                        &runtime.spawn_events,
7183                        &runtime.child_roster,
7184                        exit_report,
7185                        true,
7186                    ).await;
7187                    runtime.start_requester = None;
7188                    runtime.pending_activity_restart = false;
7189                    // The exit is recorded, so a daemon shutdown may stop
7190                    // waiting for this child (see `SupervisedChild::wait`).
7191                    let carry = matches!(next, NextAction::Restart { .. })
7192                        && active_child.spawn_failure.is_none()
7193                        && !runtime.child_roster.is_closed();
7194                    if let Err(error) = active_child.finish_exit(carry).await {
7195                        containment::terminate(&runtime.child_roster.containment).await;
7196                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7197                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7198                            &runtime.failure_context(true));
7199                        child = None;
7200                        continue;
7201                    }
7202                    match next {
7203                        NextAction::Stop { registration_released } => {
7204                            if registration_released {
7205                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7206                            }
7207                            child = None;
7208                        }
7209                        NextAction::Restart { schedule } => {
7210                            let delay = schedule.map_or(
7211                                runtime.restart_policy.delay_for_restart(0),
7212                                |schedule| schedule.delay,
7213                            );
7214                            if let Some(schedule) = schedule {
7215                                log_crash_respawn(&spec.module_id, schedule);
7216                            }
7217                            // The exited child is fully recorded at this point,
7218                            // so release it and count the backoff down in the
7219                            // command-serving branch below rather than sleeping
7220                            // here: commands cannot be received from inside this
7221                            // select arm, and an operator disable or drain that
7222                            // arrives during the backoff must cancel the pending
7223                            // respawn instead of waiting for it to spawn first.
7224                            child = None;
7225                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7226                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7227                        }
7228                    }
7229                }
7230                command = commands.recv() => {
7231                    #[cfg(test)]
7232                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7233                    let Some(command) = command else {
7234                        containment::terminate(&runtime.child_roster.containment).await;
7235                        return;
7236                    };
7237                    if !handle_supervisor_command(
7238                        command,
7239                        &mut spec,
7240                        &mut runtime,
7241                        &registry,
7242                        &process_liveness,
7243                        &snapshot,
7244                        &mut child,
7245                        &mut commands,
7246                        &mut requeued,
7247                    ).await {
7248                        containment::terminate(&runtime.child_roster.containment).await;
7249                        return;
7250                    }
7251                }
7252                _ = async {
7253                    match registration_changes.as_mut() {
7254                        Some(changes) => { let _ = changes.changed().await; }
7255                        None => std::future::pending().await,
7256                    }
7257                }, if wire_child => {
7258                    #[cfg(test)]
7259                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7260                }
7261                _ = &mut probe_sleep => {
7262                    #[cfg(test)]
7263                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7264                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7265                        // Recheck after the wake: HELLO, exit or shutdown may
7266                        // have become ready in the same executor tick.
7267                        active_child.registration_deadline = None;
7268                        if !runtime.child_roster.is_closed()
7269                            && !active_child.registered_since_spawn()
7270                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7271                        {
7272                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7273                                wait_secs = runtime.register_timeout.as_secs(),
7274                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7275                            active_child.spawn_failure = Some("registration_timeout".into());
7276                            if let Err(error) = stop_unregistered_child(active_child).await {
7277                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7278                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7279                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7280                                    &runtime.failure_context(true));
7281                                child = None;
7282                            }
7283                            // The ordinary exit handler records the child's exit,
7284                            // spends crash budget, and schedules recovery.
7285                            continue;
7286                        }
7287                    }
7288                    if health_probe.due() {
7289                        run_health_probe_cycle(
7290                            &spec,
7291                            &runtime,
7292                            &registry,
7293                            &process_liveness,
7294                            &snapshot,
7295                            &mut child,
7296                        ).await;
7297                        if child.is_none() {
7298                            runtime.start_requester = None;
7299                            runtime.pending_activity_restart = false;
7300                        }
7301                        if child.is_some() {
7302                            health_probe.schedule_next(&spec, runtime.health.cadence);
7303                        }
7304                    }
7305                }
7306            }
7307        } else if let Some(pending) = pending_respawn {
7308            tokio::select! {
7309                _ = runtime.child_roster.closed() => {
7310                    pending_respawn = None;
7311                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7312                        state.state = ModuleState::Stopped;
7313                        state.respawn_pending = false;
7314                    });
7315                    containment::terminate(&runtime.child_roster.containment).await;
7316                }
7317                _ = sleep_until(pending.deadline) => {
7318                    pending_respawn = None;
7319                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7320                    // A command handled below while the backoff elapsed may
7321                    // have stopped the module; never respawn past an operator's
7322                    // disable or drain.
7323                    if !respawn_still_pending(&snapshot) {
7324                        continue;
7325                    }
7326                    // The daemon began shutting down during the backoff: the
7327                    // spawn would be refused anyway, and refusing it here
7328                    // leaves the module stopped instead of reporting a
7329                    // failed restart.
7330                    if runtime.child_roster.is_closed() {
7331                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7332                            state.state = ModuleState::Stopped;
7333                        });
7334                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7335                        continue;
7336                    }
7337                    if let Err(err) = release_dead_registration(
7338                        &registry,
7339                        runtime.forwarding.as_deref(),
7340                        &snapshot,
7341                        &spec.module_id,
7342                        &runtime.failure_context(runtime.recovery_is_daemon),
7343                    ).await {
7344                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7345                        continue;
7346                    }
7347
7348                    if matches!(pending.kind, RespawnKind::Reload) {
7349                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7350                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7351                        if result.is_ok() && runtime.pending_activity_restart {
7352                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7353                        }
7354                        runtime.pending_activity_restart = false;
7355                        if let Some(reply) = reply { let _ = reply.send(result); }
7356                        continue;
7357                    }
7358                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7359                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7360                        Ok(next_child) => {
7361                            child = Some(next_child);
7362                            if runtime.pending_activity_restart {
7363                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7364                                runtime.pending_activity_restart = false;
7365                            }
7366                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7367                        }
7368                        Err(err) => {
7369                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7370                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7371                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7372                        }
7373                    }
7374                }
7375                command = commands.recv() => {
7376                    let Some(command) = command else {
7377                        containment::terminate(&runtime.child_roster.containment).await;
7378                        return;
7379                    };
7380                    if !handle_supervisor_command(
7381                        command,
7382                        &mut spec,
7383                        &mut runtime,
7384                        &registry,
7385                        &process_liveness,
7386                        &snapshot,
7387                        &mut child,
7388                        &mut commands,
7389                        &mut requeued,
7390                    ).await {
7391                        containment::terminate(&runtime.child_roster.containment).await;
7392                        return;
7393                    }
7394                    // Reconcile the pending respawn with what the command did:
7395                    // a start may already have spawned a fresh child,
7396                    // while a disable or drain moved the snapshot out of the
7397                    // state the respawn was counting down from.
7398                    if child.is_some() || !respawn_still_pending(&snapshot) {
7399                        pending_respawn = None;
7400                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7401                    }
7402                }
7403            }
7404        } else {
7405            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7406                (state.enabled && state.state == ModuleState::Failed)
7407                    .then_some(state.failed_retry.pending)
7408                    .flatten()
7409            });
7410            let command = if lock_snapshot(&snapshot)
7411                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7412            {
7413                tokio::select! {
7414                    command = commands.recv() => command,
7415                    _ = previous_instance_recheck.tick() => {
7416                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7417                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7418                            if enabled && !runtime.child_roster.is_closed() {
7419                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7420                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7421                                    Ok(next_child) => child = Some(next_child),
7422                                    Err(error) => {
7423                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7424                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7425                                    }
7426                                }
7427                            } else {
7428                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7429                                    state.waiting_for_previous_pid = None;
7430                                    state.waiting_for_previous_detail = None;
7431                                });
7432                            }
7433                        }
7434                        continue;
7435                    }
7436                }
7437            } else {
7438                tokio::select! {
7439                    biased;
7440                    command = commands.recv() => command,
7441                    _ = async {
7442                        match slow_retry {
7443                            Some(retry) => sleep_until(retry.deadline).await,
7444                            None => std::future::pending().await,
7445                        }
7446                    } => {
7447                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7448                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7449                            state.failed_retry.pending = None;
7450                        });
7451                        if runtime.child_roster.is_closed() {
7452                            continue;
7453                        }
7454                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7455                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7456                        if let Err(error) = set_child_enabled(
7457                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7458                            &runtime.failure_context(true),
7459                        ).await {
7460                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7461                        }
7462                        continue;
7463                    }
7464                }
7465            };
7466            let Some(command) = command else {
7467                containment::terminate(&runtime.child_roster.containment).await;
7468                return;
7469            };
7470            if !handle_supervisor_command(
7471                command,
7472                &mut spec,
7473                &mut runtime,
7474                &registry,
7475                &process_liveness,
7476                &snapshot,
7477                &mut child,
7478                &mut commands,
7479                &mut requeued,
7480            )
7481            .await
7482            {
7483                containment::terminate(&runtime.child_roster.containment).await;
7484                return;
7485            }
7486        }
7487    }
7488}
7489
7490// Preserve the deadline until the future is polled. Starting a relative sleep
7491// after the executor yields would restart part of the countdown.
7492async fn supervisor_sleep(wake_at: Option<Instant>) {
7493    match wake_at {
7494        Some(deadline) => sleep_until(deadline).await,
7495        None => std::future::pending().await,
7496    }
7497}
7498
7499fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7500    info!(
7501        module_id,
7502        restart_in_window = schedule.restart_in_window,
7503        delay_ms = schedule.delay.as_millis() as u64,
7504        "respawning after crash"
7505    );
7506}
7507
7508/// Whether the respawn a backoff was counting down to is still wanted. A
7509/// disable or drain handled while the backoff elapsed moves the snapshot out
7510/// of `Restarting`, and the operator's stop must win over the pending respawn,
7511/// so every sleep-then-spawn path re-validates against the live snapshot
7512/// instead of assuming the state it left behind still holds.
7513fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7514    matches!(
7515        lock_snapshot(snapshot),
7516        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7517    )
7518}
7519
7520enum NextAction {
7521    Stop {
7522        registration_released: bool,
7523    },
7524    Restart {
7525        schedule: Option<CrashRestartSchedule>,
7526    },
7527}
7528
7529#[allow(clippy::too_many_arguments)]
7530async fn handle_supervisor_command(
7531    command: SupervisorCommand,
7532    spec: &mut ModuleSpec,
7533    runtime: &mut SupervisorRuntimeConfig,
7534    registry: &Arc<Registry>,
7535    process_liveness: &SupervisorProcessLiveness,
7536    snapshot: &SharedSnapshot,
7537    child: &mut Option<SupervisedChild>,
7538    commands: &mut mpsc::Receiver<SupervisorCommand>,
7539    requeued: &mut VecDeque<SupervisorCommand>,
7540) -> bool {
7541    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7542        runtime.recovery_is_daemon = false;
7543    }
7544    if matches!(
7545        command,
7546        SupervisorCommand::Drain { .. }
7547            | SupervisorCommand::Retire { .. }
7548            | SupervisorCommand::SetEnabled { .. }
7549            | SupervisorCommand::Restart { .. }
7550    ) {
7551        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7552            state.failed_retry.cancel()
7553        });
7554    }
7555    let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7556        runtime
7557            .child_roster
7558            .previous_instance(&spec.module_id)
7559            .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7560    });
7561    let command = if let Some(error) = blocked {
7562        match command {
7563            SupervisorCommand::Restart { reply, .. }
7564            | SupervisorCommand::Reload { reply, .. }
7565            | SupervisorCommand::Swap { reply, .. } => {
7566                let _ = reply.send(Err(error));
7567                return true;
7568            }
7569            SupervisorCommand::SetEnabled {
7570                enabled: true,
7571                reply,
7572                ..
7573            } => {
7574                let _ = reply.send(Err(error));
7575                return true;
7576            }
7577            other => other,
7578        }
7579    } else {
7580        command
7581    };
7582    match command {
7583        SupervisorCommand::Drain { requester, reply } => {
7584            // A plain stop runs no forwarding drain, so nothing reaches the
7585            // module over its connection before the wait: ask by signal.
7586            let result = drain_optional_child(
7587                &spec.module_id,
7588                spec.protocol,
7589                StopNotice::NotSent,
7590                registry,
7591                runtime.forwarding.as_deref(),
7592                snapshot,
7593                &runtime.terminal_ring,
7594                &runtime.spawn_events,
7595                child,
7596                runtime.drain_timeout,
7597                ModuleState::Stopped,
7598                None,
7599                &FailureContext::operator(),
7600            )
7601            .await;
7602            containment::terminate(&runtime.child_roster.containment).await;
7603            let registration_released = result.is_ok();
7604            if registration_released {
7605                runtime.record_activity_transition(
7606                    "subc.module_stopped",
7607                    &spec.module_id,
7608                    requester,
7609                );
7610            }
7611            let _ = reply.send(result);
7612            if registration_released {
7613                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7614            }
7615            false
7616        }
7617        SupervisorCommand::Retire { requester, reply } => {
7618            let result = async {
7619                let stop_notice = begin_forwarding_drain_if_configured(
7620                    spec,
7621                    runtime,
7622                    registry,
7623                    snapshot,
7624                    None,
7625                    RouteCloseReason::Disable,
7626                )
7627                .await?;
7628                drain_optional_child(
7629                    &spec.module_id,
7630                    spec.protocol,
7631                    stop_notice,
7632                    registry,
7633                    runtime.forwarding.as_deref(),
7634                    snapshot,
7635                    &runtime.terminal_ring,
7636                    &runtime.spawn_events,
7637                    child,
7638                    runtime.drain_timeout,
7639                    ModuleState::Stopped,
7640                    None,
7641                    &FailureContext::operator(),
7642                )
7643                .await
7644            }
7645            .await;
7646            containment::terminate(&runtime.child_roster.containment).await;
7647            let registration_released = result.is_ok();
7648            if registration_released {
7649                runtime.record_activity_transition(
7650                    "subc.module_stopped",
7651                    &spec.module_id,
7652                    requester,
7653                );
7654            }
7655            let _ = reply.send(result);
7656            if registration_released {
7657                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7658            }
7659            false
7660        }
7661        SupervisorCommand::Restart {
7662            requester,
7663            drain_timeout_ms,
7664            received_at_generation,
7665            queued_at,
7666            reply,
7667        } => {
7668            // Without this line a restart that waited in the queue (behind a
7669            // health probe cycle or another command) was invisible: the log
7670            // showed only the drain timing out, minutes after the operator's call.
7671            info!(
7672                module_id = %spec.module_id,
7673                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7674                "restart command dequeued"
7675            );
7676            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7677            // caller whose own request lane rides the module being restarted: the
7678            // caller's in-flight request keeps the drain from quiescing, the drain
7679            // keeps the restart from completing, and the completion keeps the reply
7680            // from releasing the caller — so the drain always timed out and cut the
7681            // initiator with a GOODBYE, even on a healthy module. Replying once the
7682            // restart is validated lets a self-lane caller settle, which is exactly
7683            // what makes the drain succeed. Completion is observable via
7684            // supervisor.list / module status; a post-ack failure lands the module
7685            // in a visible terminal state below rather than in a reply nobody can
7686            // receive.
7687            let validation = match lock_snapshot(snapshot) {
7688                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7689                    module_id: spec.module_id.clone(),
7690                }),
7691                Ok(_) => Ok(()),
7692                Err(err) => Err(err),
7693            };
7694            let initiated = validation.is_ok();
7695            let _ = reply.send(validation);
7696            // A restart asks for a fresh process. Commands run one at a time,
7697            // so a restart queued behind another restart (two operator calls
7698            // in quick succession) is dequeued the moment the first one has
7699            // spawned its replacement -- before that process has sent HELLO.
7700            // Running it would drain and kill the process the first restart
7701            // just produced, which is the opposite of what both callers asked
7702            // for. If a process spawned after this request was received is
7703            // still supervised, the request is already satisfied. Not when the
7704            // configuration changed since that spawn: then the newer process
7705            // predates the spec this restart may exist to apply.
7706            // Generation publication can lag a child that is already serving
7707            // while its identity is recorded. Only process creation after request
7708            // receipt satisfies a restart, not publication after receipt.
7709            let satisfied_by_generation = if initiated
7710                && child
7711                    .as_ref()
7712                    .is_some_and(|child| child.spawned_at > queued_at)
7713            {
7714                lock_snapshot(snapshot).ok().and_then(|state| {
7715                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7716                })
7717            } else {
7718                None
7719            };
7720            let satisfied_by_pending = initiated
7721                && child.is_none()
7722                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7723                    // No child exists yet, so this replacement will necessarily
7724                    // be created after the queued request, unlike a published
7725                    // generation whose process may already have been serving.
7726                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7727                    if pending {
7728                        state.coalesced_restart_pending = true;
7729                    }
7730                    pending
7731                });
7732            if satisfied_by_pending {
7733                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7734            } else if let Some(generation) = satisfied_by_generation {
7735                info!(
7736                    module_id = %spec.module_id,
7737                    received_at_generation,
7738                    "restart already satisfied by generation {generation}; not restarting again"
7739                );
7740            } else if initiated {
7741                // Precedence: this restart's operator override, else the module's
7742                // configured budget (already resolved into the runtime).
7743                let drain_timeout = drain_timeout_ms
7744                    .map(Duration::from_millis)
7745                    .unwrap_or(runtime.drain_timeout);
7746                if let Err(err) = restart_child(
7747                    spec,
7748                    runtime,
7749                    registry,
7750                    process_liveness,
7751                    snapshot,
7752                    child,
7753                    drain_timeout,
7754                )
7755                .await
7756                {
7757                    warn!(
7758                        module_id = %spec.module_id,
7759                        error = %err,
7760                        "operator restart failed after initiation ack; module state carries the outcome"
7761                    );
7762                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7763                        state.state = ModuleState::Failed;
7764                        clear_current_process_facts(state);
7765                    });
7766                } else {
7767                    runtime.start_requester = requester;
7768                    runtime.pending_activity_restart = true;
7769                }
7770            }
7771            true
7772        }
7773        SupervisorCommand::Reload { requester, reply } => {
7774            let result =
7775                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7776            if result.is_ok() {
7777                runtime.start_requester = requester;
7778                runtime.pending_activity_restart = true;
7779            }
7780            if result.is_ok()
7781                && runtime
7782                    .scheduled_respawn
7783                    .lock()
7784                    .unwrap_or_else(|p| p.into_inner())
7785                    .as_ref()
7786                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7787            {
7788                *runtime
7789                    .deferred_reload_reply
7790                    .lock()
7791                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7792            } else {
7793                let _ = reply.send(result);
7794            }
7795            true
7796        }
7797        SupervisorCommand::SetEnabled {
7798            enabled,
7799            requester,
7800            reply,
7801        } => {
7802            let previous_requester = runtime.start_requester.clone();
7803            if enabled {
7804                runtime.start_requester = requester.clone();
7805            }
7806            let result = set_child_enabled(
7807                spec,
7808                runtime,
7809                registry,
7810                process_liveness,
7811                snapshot,
7812                child,
7813                enabled,
7814                &runtime.failure_context(false),
7815            )
7816            .await;
7817            if !enabled {
7818                containment::terminate(&runtime.child_roster.containment).await;
7819            }
7820            if matches!(result, Ok(true)) {
7821                runtime.pending_activity_restart = false;
7822                if enabled {
7823                    runtime.start_requester = requester;
7824                } else {
7825                    runtime.start_requester = None;
7826                    runtime.record_activity_transition(
7827                        "subc.module_disabled",
7828                        &spec.module_id,
7829                        requester,
7830                    );
7831                }
7832            } else {
7833                runtime.start_requester = previous_requester;
7834            }
7835            let _ = reply.send(result);
7836            true
7837        }
7838        SupervisorCommand::UpdateConfiguration {
7839            spec: next_spec,
7840            health,
7841            drain_timeout_ms,
7842            reply,
7843        } => {
7844            if let Some(handle) = &runtime.supervisor_handle {
7845                handle.apply_identity_configuration(&next_spec);
7846            }
7847            *spec = next_spec;
7848            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7849                state.configuration_updated_since_spawn = true;
7850            });
7851            let health_changed = runtime.health != health;
7852            runtime.health = health;
7853            // Reset the cadence and old endpoint's failure streak on a live
7854            // health-policy change rather than waiting for its old deadline.
7855            if health_changed {
7856                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7857                    state.health = ModuleHealthStatus::default();
7858                });
7859            }
7860            runtime.drain_timeout = drain_timeout_ms
7861                .map(Duration::from_millis)
7862                .unwrap_or(runtime.default_drain_timeout);
7863            *runtime
7864                .effective_drain_timeout
7865                .lock()
7866                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7867            let _ = reply.send(());
7868            true
7869        }
7870        SupervisorCommand::Swap {
7871            requester,
7872            ready_timeout,
7873            reply,
7874        } => {
7875            if spec.carry_containment
7876                || containment::current(&runtime.child_roster.containment).is_some()
7877            {
7878                let _ = reply.send(Err(SuperviseError::InvalidSpec {
7879                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
7880                }));
7881                return true;
7882            }
7883            let mut swap_runtime = runtime.clone();
7884            swap_runtime.start_requester = requester;
7885            let before = lock_snapshot(snapshot)
7886                .ok()
7887                .map(|state| state.spawn_generation);
7888            let end = swap::run_swap(
7889                spec,
7890                &swap_runtime,
7891                registry,
7892                process_liveness,
7893                snapshot,
7894                child,
7895                commands,
7896                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7897                reply,
7898            )
7899            .await;
7900            if before
7901                != lock_snapshot(snapshot)
7902                    .ok()
7903                    .map(|state| state.spawn_generation)
7904            {
7905                runtime.start_requester = swap_runtime.start_requester;
7906            }
7907            requeued.extend(end.requeue);
7908            true
7909        }
7910    }
7911}
7912
7913async fn restart_child(
7914    spec: &ModuleSpec,
7915    runtime: &SupervisorRuntimeConfig,
7916    registry: &Registry,
7917    process_liveness: &SupervisorProcessLiveness,
7918    snapshot: &SharedSnapshot,
7919    child: &mut Option<SupervisedChild>,
7920    drain_timeout: Duration,
7921) -> Result<(), SuperviseError> {
7922    // Restart cycles a running module; it must not silently start a disabled one.
7923    if !lock_snapshot(snapshot)?.enabled {
7924        return Err(SuperviseError::Disabled {
7925            module_id: spec.module_id.clone(),
7926        });
7927    }
7928    let stop_notice = begin_forwarding_drain_with_timeout(
7929        spec,
7930        runtime,
7931        registry,
7932        snapshot,
7933        None,
7934        RouteCloseReason::Restart,
7935        drain_timeout,
7936    )
7937    .await?;
7938
7939    if child.is_some() {
7940        drain_optional_child(
7941            &spec.module_id,
7942            spec.protocol,
7943            stop_notice,
7944            registry,
7945            runtime.forwarding.as_deref(),
7946            snapshot,
7947            &runtime.terminal_ring,
7948            &runtime.spawn_events,
7949            child,
7950            drain_timeout,
7951            ModuleState::Restarting,
7952            Some(true),
7953            &FailureContext::operator(),
7954        )
7955        .await?;
7956    } else {
7957        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7958            state.enabled = true;
7959            state.state = ModuleState::Restarting;
7960            clear_current_process_facts(state);
7961        })?;
7962        release_dead_registration(
7963            registry,
7964            runtime.forwarding.as_deref(),
7965            snapshot,
7966            &spec.module_id,
7967            &runtime.failure_context(false),
7968        )
7969        .await?;
7970    }
7971
7972    reset_restart_count(snapshot, &spec.module_id)?;
7973    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7974    schedule_respawn(
7975        runtime,
7976        snapshot,
7977        &spec.module_id,
7978        runtime.restart_policy.backoff,
7979        RespawnKind::Spawn,
7980        false,
7981    )
7982}
7983
7984async fn reload_child(
7985    spec: &ModuleSpec,
7986    runtime: &SupervisorRuntimeConfig,
7987    registry: &Registry,
7988    process_liveness: &SupervisorProcessLiveness,
7989    snapshot: &SharedSnapshot,
7990    child: &mut Option<SupervisedChild>,
7991) -> Result<(), SuperviseError> {
7992    // Reload cycles a running module; it must not silently start a disabled one.
7993    if !lock_snapshot(snapshot)?.enabled {
7994        return Err(SuperviseError::Disabled {
7995            module_id: spec.module_id.clone(),
7996        });
7997    }
7998    let stop_notice = begin_forwarding_drain(
7999        spec,
8000        runtime,
8001        registry,
8002        snapshot,
8003        Some(true),
8004        RouteCloseReason::Reload,
8005    )
8006    .await?;
8007
8008    if child.is_some() {
8009        drain_optional_child(
8010            &spec.module_id,
8011            spec.protocol,
8012            stop_notice,
8013            registry,
8014            runtime.forwarding.as_deref(),
8015            snapshot,
8016            &runtime.terminal_ring,
8017            &runtime.spawn_events,
8018            child,
8019            runtime.drain_timeout,
8020            ModuleState::Restarting,
8021            Some(true),
8022            &FailureContext::operator(),
8023        )
8024        .await?;
8025    } else {
8026        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8027            state.enabled = true;
8028            state.state = ModuleState::Restarting;
8029            clear_current_process_facts(state);
8030        })?;
8031        release_dead_registration(
8032            registry,
8033            runtime.forwarding.as_deref(),
8034            snapshot,
8035            &spec.module_id,
8036            &runtime.failure_context(false),
8037        )
8038        .await?;
8039    }
8040
8041    reset_restart_count(snapshot, &spec.module_id)?;
8042    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8043    schedule_respawn(
8044        runtime,
8045        snapshot,
8046        &spec.module_id,
8047        runtime.restart_policy.backoff,
8048        RespawnKind::Reload,
8049        false,
8050    )
8051}
8052
8053async fn finish_reload_child(
8054    spec: &ModuleSpec,
8055    runtime: &SupervisorRuntimeConfig,
8056    registry: &Registry,
8057    process_liveness: &SupervisorProcessLiveness,
8058    snapshot: &SharedSnapshot,
8059    child: &mut Option<SupervisedChild>,
8060) -> Result<(), SuperviseError> {
8061    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8062    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8063        Ok(next_child) => next_child,
8064        Err(err) => {
8065            return handle_reload_spawn_failure(
8066                spec,
8067                runtime,
8068                process_liveness,
8069                snapshot,
8070                child,
8071                format!("new child failed to spawn: {err}"),
8072            )
8073            .await;
8074        }
8075    };
8076    *child = Some(next_child);
8077
8078    let wait_outcome = {
8079        let active_child = child.as_mut().expect("new reload child was just stored");
8080        wait_for_registration_after_reload(
8081            registry,
8082            &spec.module_id,
8083            snapshot,
8084            active_child,
8085            REGISTRY_RELEASE_TIMEOUT,
8086        )
8087        .await?
8088    };
8089
8090    match wait_outcome {
8091        RegistrationWaitOutcome::Registered => {
8092            child
8093                .as_mut()
8094                .expect("registered reload child")
8095                .registration_deadline = None;
8096            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8097            Ok(())
8098        }
8099        RegistrationWaitOutcome::Exited(exit_report) => {
8100            if let Some(active_child) = child.as_mut() {
8101                active_child.drain_stderr(&spec.module_id).await;
8102            }
8103            // Keep the exited child's roster guard until its terminal is written.
8104            // Shutdown waits on that guard, not on the child Option used for respawn.
8105            let mut exited_child = child.take().expect("exited reload child is still stored");
8106            #[cfg(test)]
8107            if let Some(gate) = &runtime.test_reload_exit_record_gate {
8108                gate.reached.notify_one();
8109                gate.resume.notified().await;
8110            }
8111            let result = handle_reload_child_registration_failure(
8112                spec,
8113                runtime,
8114                registry,
8115                process_liveness,
8116                snapshot,
8117                child,
8118                ReloadRegistrationFailure {
8119                    exit_report: registration_failure_exit_report(exit_report),
8120                    reason: exited_child
8121                        .spawn_failure
8122                        .clone()
8123                        .unwrap_or_else(|| "new child exited before registering".to_string()),
8124                },
8125            )
8126            .await;
8127            exited_child
8128                .finish_exit(false)
8129                .await
8130                .map_err(|source| SuperviseError::Wait {
8131                    module_id: spec.module_id.clone(),
8132                    source,
8133                })?;
8134            result
8135        }
8136        RegistrationWaitOutcome::TimedOut => {
8137            let mut timed_out_child = child
8138                .take()
8139                .expect("timed-out reload child is still running");
8140            timed_out_child
8141                .start_kill()
8142                .map_err(|source| SuperviseError::Kill {
8143                    module_id: spec.module_id.clone(),
8144                    source,
8145                })?;
8146            let status = timed_out_child
8147                .wait()
8148                .await
8149                .map_err(|source| SuperviseError::Wait {
8150                    module_id: spec.module_id.clone(),
8151                    source,
8152                })?;
8153            timed_out_child.drain_stderr(&spec.module_id).await;
8154            let result = handle_reload_child_registration_failure(
8155                spec,
8156                runtime,
8157                registry,
8158                process_liveness,
8159                snapshot,
8160                child,
8161                ReloadRegistrationFailure {
8162                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8163                        snapshot,
8164                        &timed_out_child,
8165                        &status,
8166                    )),
8167                    reason: format!(
8168                        "new child did not register within {:?}",
8169                        REGISTRY_RELEASE_TIMEOUT
8170                    ),
8171                },
8172            )
8173            .await;
8174            timed_out_child
8175                .finish_exit(false)
8176                .await
8177                .map_err(|source| SuperviseError::Wait {
8178                    module_id: spec.module_id.clone(),
8179                    source,
8180                })?;
8181            result
8182        }
8183    }
8184}
8185
8186#[allow(clippy::too_many_arguments)]
8187async fn set_child_enabled(
8188    spec: &ModuleSpec,
8189    runtime: &SupervisorRuntimeConfig,
8190    registry: &Registry,
8191    process_liveness: &SupervisorProcessLiveness,
8192    snapshot: &SharedSnapshot,
8193    child: &mut Option<SupervisedChild>,
8194    enabled: bool,
8195    failure: &FailureContext,
8196) -> Result<bool, SuperviseError> {
8197    let (current_enabled, current_state, respawn_pending) = {
8198        let state = lock_snapshot(snapshot)?;
8199        (state.enabled, state.state, state.respawn_pending)
8200    };
8201    // `start` (enable on an already-enabled module) heals TERMINAL states instead
8202    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
8203    // clean (Stopped) has no live process and no other in-band recovery — the
8204    // operator's start is the explicit recovery act and resets the budget. Without
8205    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
8206    // which the 2026-07-14 aft outage proved is a trap when the failed module is
8207    // the one providing every agent's shell.
8208    let revive_terminal = enabled
8209        && current_enabled
8210        && child.is_none()
8211        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8212            || (current_state == ModuleState::Restarting && !respawn_pending));
8213    if current_enabled == enabled && !revive_terminal {
8214        return Ok(false);
8215    }
8216
8217    if enabled {
8218        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8219            state.enabled = true;
8220            state.state = ModuleState::Starting;
8221            clear_current_process_facts(state);
8222        })?;
8223        #[cfg(test)]
8224        if runtime.test_seed_stale_facts_before_enable_spawn {
8225            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8226                state.process_alive = true;
8227                state.pid = Some(41);
8228                state.spawned_at_ms = Some(42);
8229                state.spawned_from = Some(PathBuf::from("/spawned/module"));
8230                state.spawned_file_identity = Some(SpawnedFileIdentity {
8231                    device: 43,
8232                    inode: 44,
8233                    #[cfg(windows)]
8234                    windows_image: None,
8235                });
8236            })?;
8237        }
8238        release_dead_registration(
8239            registry,
8240            runtime.forwarding.as_deref(),
8241            snapshot,
8242            &spec.module_id,
8243            failure,
8244        )
8245        .await?;
8246        reset_restart_count(snapshot, &spec.module_id)?;
8247        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8248        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8249            Ok(next_child) => next_child,
8250            Err(err) => {
8251                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8252                process_liveness.untrack_if_current(&spec.module_id, snapshot);
8253                return Err(err);
8254            }
8255        };
8256        *child = Some(next_child);
8257        debug!(module_id = %spec.module_id, "supervised module enabled");
8258        Ok(true)
8259    } else {
8260        let stop_notice = begin_forwarding_drain_if_configured(
8261            spec,
8262            runtime,
8263            registry,
8264            snapshot,
8265            Some(false),
8266            RouteCloseReason::Disable,
8267        )
8268        .await?;
8269        drain_optional_child(
8270            &spec.module_id,
8271            spec.protocol,
8272            stop_notice,
8273            registry,
8274            runtime.forwarding.as_deref(),
8275            snapshot,
8276            &runtime.terminal_ring,
8277            &runtime.spawn_events,
8278            child,
8279            runtime.drain_timeout,
8280            ModuleState::Disabled,
8281            Some(false),
8282            failure,
8283        )
8284        .await?;
8285        debug!(module_id = %spec.module_id, "supervised module disabled");
8286        Ok(true)
8287    }
8288}
8289
8290#[allow(clippy::too_many_arguments)]
8291async fn on_child_exit(
8292    spec: &ModuleSpec,
8293    policy: RestartPolicy,
8294    registry: &Registry,
8295    snapshot: &SharedSnapshot,
8296    terminal_ring: &Arc<Mutex<TerminalRing>>,
8297    spawn_events: &SpawnEventFeed,
8298    roster: &ChildRoster,
8299    exit_report: ExitReport,
8300    daemon_initiated: bool,
8301) -> NextAction {
8302    // Once the daemon has begun shutting down, no exit is a crash to recover
8303    // from: the module is exiting because the daemon is going away (EOF on its
8304    // connection, or a service manager signalling the whole cgroup). Record it
8305    // as such and never schedule a respawn, which would only start a process
8306    // for the shutdown to end again.
8307    if roster.is_closed() {
8308        return on_child_exit_during_daemon_shutdown(
8309            spec,
8310            registry,
8311            snapshot,
8312            terminal_ring,
8313            spawn_events,
8314            exit_report,
8315        )
8316        .await;
8317    }
8318    // Deliberate stops the supervisor asks for (operator stop, disable,
8319    // restart, reload, swap, a health restart, a drain that runs out of budget)
8320    // takes the child out of the supervise loop and reaps it in
8321    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8322    // that reaches this point is either unrequested or a first-HELLO timeout,
8323    // which is deliberately classified as a crash even after a clean TERM exit.
8324    //
8325    // For a subc-wire module a clean exit is still a stop: those modules are
8326    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8327    // crash, and exiting 0 is a deliberate choice the module made. A
8328    // `protocol: "none"` module is a stock program we cannot change, and many
8329    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8330    // stop would leave the module down for good after any stray signal, so it
8331    // goes through the crash path instead: it spends restart budget, respawns
8332    // with the crash backoff, and ends `failed` when the budget runs out.
8333    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8334        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8335    match exit_report.kind {
8336        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8337            info!(
8338                module_id = %spec.module_id,
8339                exit_code = ?exit_report.code,
8340                exit_signal = ?exit_report.signal,
8341                "supervised module exited cleanly"
8342            );
8343            if let Err(err) = publish_after_terminal_record(
8344                &spec.module_id,
8345                || {
8346                    record_terminal(
8347                        &spec.module_id,
8348                        terminal_ring,
8349                        spawn_events,
8350                        &exit_report,
8351                        TerminalDisposition::Stopped,
8352                    )
8353                },
8354                || {
8355                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8356                        state.state = ModuleState::Stopped;
8357                        clear_current_process_facts(state);
8358                        state.last_exit = Some(exit_report.clone());
8359                    })
8360                },
8361            ) {
8362                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8363            }
8364            let registration_released = match wait_for_registration_release(
8365                registry,
8366                &spec.module_id,
8367                REGISTRY_RELEASE_TIMEOUT,
8368            )
8369            .await
8370            {
8371                Ok(()) => true,
8372                Err(err) => {
8373                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8374                    false
8375                }
8376            };
8377            NextAction::Stop {
8378                registration_released,
8379            }
8380        }
8381        ExitKind::Clean | ExitKind::Crash => {
8382            if unrequested_clean_exit_of_protocol_none {
8383                warn!(
8384                    module_id = %spec.module_id,
8385                    exit_code = ?exit_report.code,
8386                    exit_signal = ?exit_report.signal,
8387                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8388                );
8389            } else {
8390                warn!(
8391                    module_id = %spec.module_id,
8392                    exit_code = ?exit_report.code,
8393                    exit_signal = ?exit_report.signal,
8394                    "supervised module exited abnormally (crash)"
8395                );
8396            }
8397            let mut restart_schedule = None;
8398            let mut disposition = TerminalDisposition::Disabled;
8399            let mut final_state = ModuleState::Disabled;
8400            // Set only when the budget is what stopped the module, so the
8401            // terminal record says which limit was hit rather than leaving
8402            // `failed` to be read as "crashed once, badly".
8403            let mut disposition_detail = lock_snapshot(snapshot)
8404                .ok()
8405                .and_then(|mut state| state.spawn_failure.take());
8406            let now = Instant::now();
8407            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8408                if state.enabled {
8409                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8410                        // Reserving the last slot exhausts the budget for the
8411                        // replacement, not for this exit. Publish its scheduled
8412                        // recovery atomically with the reservation so connection
8413                        // cleanup sees it while terminal recording still holds
8414                        // the visible state at Running.
8415                        state.respawn_pending = true;
8416                        final_state = ModuleState::Restarting;
8417                        restart_schedule = Some(schedule);
8418                        disposition = TerminalDisposition::Restarting;
8419                    } else {
8420                        final_state = ModuleState::Failed;
8421                        disposition = TerminalDisposition::Failed;
8422                        let budget = policy.budget_exhausted_detail();
8423                        disposition_detail =
8424                            Some(disposition_detail.take().map_or_else(
8425                                || budget.clone(),
8426                                |cause| format!("{cause}; {budget}"),
8427                            ));
8428                    }
8429                } else {
8430                    disposition = TerminalDisposition::Disabled;
8431                }
8432            }) {
8433                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8434                return NextAction::Stop {
8435                    registration_released: false,
8436                };
8437            }
8438            if disposition == TerminalDisposition::Failed {
8439                // The window is in the message, not only in the fields: this line
8440                // is read in a scrollback where a bare `max_restarts=3` reads as a
8441                // lifetime cap and sends the operator looking for three crashes
8442                // that never happened together.
8443                error!(
8444                    module_id = %spec.module_id,
8445                    max_restarts = policy.max_restarts,
8446                    window_secs = policy.window.as_secs(),
8447                    "module stopped: {}",
8448                    policy.budget_exhausted_detail()
8449                );
8450            }
8451            if let Err(err) = publish_after_terminal_record(
8452                &spec.module_id,
8453                || {
8454                    record_terminal_with_detail(
8455                        &spec.module_id,
8456                        terminal_ring,
8457                        spawn_events,
8458                        &exit_report,
8459                        disposition,
8460                        disposition_detail,
8461                    )
8462                },
8463                || {
8464                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8465                        state.state = final_state;
8466                        clear_current_process_facts(state);
8467                        if final_state == ModuleState::Failed
8468                            && daemon_initiated
8469                            && !roster.is_closed()
8470                        {
8471                            state.failed_retry.schedule(&spec.module_id);
8472                        }
8473                        state.last_exit = Some(exit_report.clone());
8474                    })
8475                },
8476            ) {
8477                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8478            }
8479
8480            if let Some(schedule) = restart_schedule {
8481                NextAction::Restart {
8482                    schedule: Some(schedule),
8483                }
8484            } else {
8485                let registration_released = match wait_for_registration_release(
8486                    registry,
8487                    &spec.module_id,
8488                    REGISTRY_RELEASE_TIMEOUT,
8489                )
8490                .await
8491                {
8492                    Ok(()) => true,
8493                    Err(err) => {
8494                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8495                        false
8496                    }
8497                };
8498                NextAction::Stop {
8499                    registration_released,
8500                }
8501            }
8502        }
8503        ExitKind::DeliberateSeverance => {
8504            warn!(
8505                module_id = %spec.module_id,
8506                exit_code = ?exit_report.code,
8507                exit_signal = ?exit_report.signal,
8508                "supervised module exited after deliberate connection severance"
8509            );
8510            let should_restart = match lock_snapshot(snapshot) {
8511                Ok(state) => state.enabled,
8512                Err(err) => {
8513                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8514                    return NextAction::Stop {
8515                        registration_released: false,
8516                    };
8517                }
8518            };
8519            let final_state = if should_restart {
8520                ModuleState::Restarting
8521            } else {
8522                ModuleState::Disabled
8523            };
8524            if let Err(err) = publish_after_terminal_record(
8525                &spec.module_id,
8526                || {
8527                    record_terminal(
8528                        &spec.module_id,
8529                        terminal_ring,
8530                        spawn_events,
8531                        &exit_report,
8532                        terminal_disposition(final_state),
8533                    )
8534                },
8535                || {
8536                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8537                        state.state = final_state;
8538                        clear_current_process_facts(state);
8539                        state.last_exit = Some(exit_report.clone());
8540                        state.lifetime_restarts += 1;
8541                    })
8542                },
8543            ) {
8544                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8545                return NextAction::Stop {
8546                    registration_released: false,
8547                };
8548            }
8549
8550            if should_restart {
8551                NextAction::Restart { schedule: None }
8552            } else {
8553                let registration_released = match wait_for_registration_release(
8554                    registry,
8555                    &spec.module_id,
8556                    REGISTRY_RELEASE_TIMEOUT,
8557                )
8558                .await
8559                {
8560                    Ok(()) => true,
8561                    Err(err) => {
8562                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8563                        false
8564                    }
8565                };
8566                NextAction::Stop {
8567                    registration_released,
8568                }
8569            }
8570        }
8571    }
8572}
8573
8574async fn on_child_exit_during_daemon_shutdown(
8575    spec: &ModuleSpec,
8576    registry: &Registry,
8577    snapshot: &SharedSnapshot,
8578    terminal_ring: &Arc<Mutex<TerminalRing>>,
8579    spawn_events: &SpawnEventFeed,
8580    exit_report: ExitReport,
8581) -> NextAction {
8582    info!(
8583        module_id = %spec.module_id,
8584        exit_code = ?exit_report.code,
8585        exit_signal = ?exit_report.signal,
8586        exit_kind = ?exit_report.kind,
8587        "supervised module exited during daemon shutdown; not restarting it"
8588    );
8589    if let Err(err) = publish_after_terminal_record(
8590        &spec.module_id,
8591        || {
8592            record_terminal(
8593                &spec.module_id,
8594                terminal_ring,
8595                spawn_events,
8596                &exit_report,
8597                TerminalDisposition::DaemonShutdown,
8598            )
8599        },
8600        || {
8601            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8602                state.state = ModuleState::Stopped;
8603                clear_current_process_facts(state);
8604                state.last_exit = Some(exit_report.clone());
8605            })
8606        },
8607    ) {
8608        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8609    }
8610    let registration_released =
8611        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8612            .await
8613            .is_ok();
8614    NextAction::Stop {
8615        registration_released,
8616    }
8617}
8618
8619// Keep the status lock out of terminal I/O. Readers may observe the previous
8620// state while recording is in progress, but never a completed exit without its
8621// history. Journal errors are logged by the journal and retained in the ring;
8622// a recording panic still publishes the final state before resuming the unwind.
8623fn publish_after_terminal_record<T>(
8624    module_id: &str,
8625    record: impl FnOnce(),
8626    publish: impl FnOnce() -> T,
8627) -> T {
8628    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8629    let published = publish();
8630    if let Err(panic) = recorded {
8631        error!(
8632            module_id,
8633            "terminal recording panicked; published final state before resuming unwind"
8634        );
8635        std::panic::resume_unwind(panic);
8636    }
8637    published
8638}
8639
8640fn fail_after_wait_error(
8641    module_id: &str,
8642    snapshot: &SharedSnapshot,
8643    terminal_ring: &Arc<Mutex<TerminalRing>>,
8644    spawn_events: &SpawnEventFeed,
8645    detail: String,
8646    failure: &FailureContext,
8647) {
8648    let report = wait_error_exit_report();
8649    publish_after_terminal_record(
8650        module_id,
8651        || {
8652            record_terminal_with_detail(
8653                module_id,
8654                terminal_ring,
8655                spawn_events,
8656                &report,
8657                TerminalDisposition::Failed,
8658                Some(detail),
8659            )
8660        },
8661        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8662    );
8663}
8664
8665#[cfg(test)]
8666fn record_wait_error_terminal(
8667    module_id: &str,
8668    terminal_ring: &Arc<Mutex<TerminalRing>>,
8669    spawn_events: &SpawnEventFeed,
8670) {
8671    record_terminal(
8672        module_id,
8673        terminal_ring,
8674        spawn_events,
8675        &wait_error_exit_report(),
8676        TerminalDisposition::Failed,
8677    );
8678}
8679
8680fn record_terminal(
8681    module_id: &str,
8682    terminal_ring: &Arc<Mutex<TerminalRing>>,
8683    spawn_events: &SpawnEventFeed,
8684    exit_report: &ExitReport,
8685    disposition: TerminalDisposition,
8686) {
8687    record_terminal_with_detail(
8688        module_id,
8689        terminal_ring,
8690        spawn_events,
8691        exit_report,
8692        disposition,
8693        None,
8694    );
8695}
8696
8697/// The ring lock is held only to capture the read (see
8698/// `TerminalJournal::capture_read`), so this module's exits keep recording
8699/// while the journal files are read. Blocking: it reads files.
8700fn durable_terminal_history_of(
8701    terminal_ring: &Mutex<TerminalRing>,
8702    module_id: &str,
8703) -> subc_control::TerminalHistory {
8704    let read = terminal_ring
8705        .lock()
8706        .unwrap_or_else(|p| p.into_inner())
8707        .capture_durable_history();
8708    read.read(module_id)
8709}
8710
8711fn record_terminal_with_detail(
8712    module_id: &str,
8713    terminal_ring: &Arc<Mutex<TerminalRing>>,
8714    spawn_events: &SpawnEventFeed,
8715    exit_report: &ExitReport,
8716    disposition: TerminalDisposition,
8717    disposition_detail: Option<String>,
8718) {
8719    #[cfg(test)]
8720    terminal_history_tests::before_terminal_record(spawn_events);
8721    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8722    let record = TerminalRecord {
8723        exit_code: exit_report.code,
8724        exit_signal: exit_report.signal,
8725        at_ms: exit_report.at_ms,
8726        disposition,
8727        exit_kind: exit_report.kind.into(),
8728        disposition_detail,
8729    };
8730    terminal_ring
8731        .lock()
8732        .unwrap_or_else(|poisoned| poisoned.into_inner())
8733        .record_exit(module_id, record);
8734}
8735
8736fn untrack_if_registration_released(
8737    process_liveness: &SupervisorProcessLiveness,
8738    registry: &Registry,
8739    module_id: &str,
8740    snapshot: &SharedSnapshot,
8741) {
8742    match registry.get_module(module_id) {
8743        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8744        Ok(Some(_)) => {}
8745        Err(err) => {
8746            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8747        }
8748    }
8749}
8750
8751/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8752/// then apply the module's configured entries minus daemon-private capture keys.
8753///
8754/// Separated from `spawn_child` only so it can be asserted without spawning a
8755/// process — a duplicate of this logic in a test would pass while the real one
8756/// drifted, which is the defect class this function exists to avoid.
8757/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8758/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8759/// either and the argument would stop a stock binary from starting at all.
8760/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8761///
8762/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8763/// through [`apply_wire_spawn_args_for_role`].
8764#[cfg(test)]
8765fn apply_wire_spawn_args(
8766    command: &mut Command,
8767    spec: &ModuleSpec,
8768    connection_file_path: Option<&std::path::Path>,
8769    handle: Option<&SupervisorHandle>,
8770) -> Result<Option<NonceHandoff>, SuperviseError> {
8771    apply_wire_spawn_args_for_role(
8772        command,
8773        spec,
8774        connection_file_path,
8775        handle,
8776        SpawnRole::Plain,
8777    )
8778}
8779
8780/// The read end of a spawn's launch-nonce pipe, prepared by
8781/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8782/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8783/// server before spawn and authenticates its client after the child exists.
8784#[cfg(unix)]
8785type NonceHandoff = subc_os::LaunchNonceHandoff;
8786#[cfg(windows)]
8787type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8788#[cfg(not(any(unix, windows)))]
8789type NonceHandoff = std::convert::Infallible;
8790
8791/// Create the launch nonce that a new module process must present when it
8792/// registers or opens a consumer connection, including a replacement process
8793/// started alongside a still-serving module.
8794///
8795/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8796/// a separate candidate token so the still-serving incumbent and its consumers
8797/// keep their nonce. Both records are installed before the process exists, so
8798/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8799///
8800/// On Unix the nonce is delivered only through a pipe. It is written into
8801/// a pipe whose read end the child gets as descriptor 3, named by
8802/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8803/// process of the same user cannot read it with `ps eww`. That handoff is
8804/// returned rather than installed here, because installing it replaces
8805/// whatever the child has at descriptor 3 and so must be the last pre-exec
8806/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8807/// the Linux callback that writes the process into its cgroup. Only after that
8808/// does `handoff.install_last` register the descriptor-3 replacement, so it
8809/// cannot close a descriptor the cgroup callback still needs.
8810/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8811fn apply_wire_spawn_args_for_role(
8812    command: &mut Command,
8813    spec: &ModuleSpec,
8814    connection_file_path: Option<&std::path::Path>,
8815    handle: Option<&SupervisorHandle>,
8816    role: SpawnRole,
8817) -> Result<Option<NonceHandoff>, SuperviseError> {
8818    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8819    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8820    // included: a daemon started from a module's process tree inherits it,
8821    // and passing it on would point the child at a descriptor it does not
8822    // have.
8823    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8824    #[cfg(windows)]
8825    {
8826        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8827        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8828    }
8829    // Remove inherited or configured copies too: withholding must mean absent.
8830    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8831    if spec.protocol == ModuleProtocol::None {
8832        return Ok(None);
8833    }
8834    if let Some(connection_file_path) = connection_file_path {
8835        command.arg(SUBC_ARG).arg(connection_file_path);
8836    }
8837
8838    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8839    // route.open attestation. Reserved modules additionally use the same nonce
8840    // for HELLO id-squatting protection. A respawn rotates both records.
8841    let nonce = generate_launch_nonce()?;
8842    if let Some(handle) = handle {
8843        match role {
8844            SpawnRole::Plain => {
8845                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8846                if spec.reserved {
8847                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8848                }
8849            }
8850            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8851        }
8852    }
8853    #[cfg(unix)]
8854    let handoff = {
8855        let handoff =
8856            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8857                program: spec.program.clone(),
8858                source,
8859                cgroup_path: None,
8860            })?;
8861        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8862        Some(handoff)
8863    };
8864    #[cfg(windows)]
8865    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8866    #[cfg(not(any(unix, windows)))]
8867    let handoff = None;
8868    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8869    // until the daemon's per-spawn delivery records show that every running
8870    // Windows module has read its secret from the named pipe.
8871    #[cfg(not(unix))]
8872    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8873    Ok(handoff)
8874}
8875
8876fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8877    command.env_remove(CK_LOG_ENV);
8878    // The spawn role is the supervisor's to set, and only on a swap candidate
8879    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8880    // it, is what makes it absent on a plain spawn: the daemon's own
8881    // environment could carry it, and so could a spec built outside daemon
8882    // config (config refuses it as an `env` key). A module reading it on a
8883    // plain restart would pick the long swap budget and leave callers waiting.
8884    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8885    for (key, value) in &spec.env {
8886        // cortexkit-log currently exposes retention only as a Rust struct, not
8887        // environment names. These values are daemon-private sink metadata and
8888        // must never become a public child-process contract by being inherited.
8889        if matches!(
8890            key.as_str(),
8891            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8892        ) || key == SUBC_SPAWN_ROLE_ENV
8893        {
8894            continue;
8895        }
8896        command.env(key, value);
8897    }
8898}
8899
8900#[cfg(windows)]
8901fn prepare_windows_nonce_pipe(
8902    command: &mut Command,
8903    spec: &ModuleSpec,
8904    nonce: &str,
8905) -> Option<NonceHandoff> {
8906    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8907        Ok(handoff) => {
8908            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8909            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8910            // still register until every module reads the pipe. A reader without
8911            // this variable set to env returns the pipe error instead of using
8912            // an inherited or configured environment copy.
8913            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8914            Some(handoff)
8915        }
8916        Err(error) => {
8917            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8918            None
8919        }
8920    }
8921}
8922
8923#[cfg(windows)]
8924fn start_windows_nonce_pipe(
8925    handoff: Option<NonceHandoff>,
8926    pid: u32,
8927    spec: &ModuleSpec,
8928) -> Option<subc_os::LaunchNoncePipeDelivery> {
8929    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8930    // same maximum wait used for registration during a reload. Older modules
8931    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8932    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8933        Ok(delivery) => Some(delivery),
8934        Err(error) => {
8935            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8936            None
8937        }
8938    })
8939}
8940
8941fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8942    #[cfg(windows)]
8943    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8944        return Some(
8945            if snapshot
8946                .pipe_consumed
8947                .as_ref()
8948                .is_some_and(|consumed| consumed.get().is_some())
8949            {
8950                "pipe"
8951            } else {
8952                "env"
8953            }
8954            .to_owned(),
8955        );
8956    }
8957    let _ = snapshot;
8958    None
8959}
8960
8961/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8962/// of a blue/green swap.
8963#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8964enum SpawnRole {
8965    Plain,
8966    SwapCandidate,
8967}
8968
8969/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8970/// `apply_child_env` has already removed the variable for every spawn.
8971fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8972    if role == SpawnRole::SwapCandidate {
8973        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8974    }
8975}
8976
8977fn previous_instance_error(
8978    spec: &ModuleSpec,
8979    pid: u32,
8980    reason: crate::live_children::RetainedReason,
8981) -> SuperviseError {
8982    SuperviseError::Spawn {
8983        program: spec.program.clone(),
8984        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8985        cgroup_path: None,
8986    }
8987}
8988
8989fn carries_containment(configured: bool, has_container: bool) -> bool {
8990    configured || has_container
8991}
8992
8993fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
8994    if !carries_containment(
8995        spec.carry_containment,
8996        containment::current(&roster.containment).is_some(),
8997    ) {
8998        return None;
8999    }
9000    Some(SuperviseError::Spawn {
9001        program: spec.program.clone(),
9002        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9003        cgroup_path: None,
9004    })
9005}
9006
9007fn spawn_child(
9008    spec: &ModuleSpec,
9009    connection_file_path: Option<&std::path::Path>,
9010    handle: Option<&SupervisorHandle>,
9011    ring: &Arc<Mutex<StderrRing>>,
9012    capture_logs_dir: Option<&std::path::Path>,
9013    roster: &ChildRoster,
9014    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9015) -> Result<SupervisedChild, SuperviseError> {
9016    #[cfg(all(test, unix))]
9017    if let Some(source) = containment_tests::join_failure(spec, roster) {
9018        containment::terminate_blocking(&roster.containment);
9019        return Err(SuperviseError::Spawn {
9020            program: spec.program.clone(),
9021            source,
9022            cgroup_path: None,
9023        });
9024    }
9025    let result = spawn_child_in_slot(
9026        spec,
9027        connection_file_path,
9028        handle,
9029        ring,
9030        capture_logs_dir,
9031        roster,
9032        #[cfg(target_os = "linux")]
9033        cgroup_placement,
9034        SpawnRole::Plain,
9035        false,
9036    );
9037    if result.is_err() {
9038        if roster.admission_pending(&spec.module_id) {
9039            containment::terminate_deferred(&roster.containment);
9040        } else {
9041            containment::terminate_blocking(&roster.containment);
9042        }
9043    }
9044    result
9045}
9046
9047/// Spawn one process of `spec` into a slot.
9048///
9049/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
9050/// A swap candidate needs a different cgroup from the process it is replacing,
9051/// which is still alive: in the same cgroup the two would be one kill domain,
9052/// and killing a failed candidate could take the incumbent with it.
9053///
9054/// The stderr capture file is `<module_id>.stderr.log` for every process of
9055/// the module, whichever slot it is in, because that is the one file
9056/// `ck module logs` reads. During a swap's overlap both processes append to it;
9057/// the daemon writes whole lines, so the two interleave by line, which is also
9058/// the merged view an operator wants while a swap runs.
9059#[allow(clippy::too_many_arguments)]
9060fn spawn_child_in_slot(
9061    spec: &ModuleSpec,
9062    connection_file_path: Option<&std::path::Path>,
9063    handle: Option<&SupervisorHandle>,
9064    ring: &Arc<Mutex<StderrRing>>,
9065    capture_logs_dir: Option<&std::path::Path>,
9066    roster: &ChildRoster,
9067    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9068    role: SpawnRole,
9069    alternate_slot: bool,
9070) -> Result<SupervisedChild, SuperviseError> {
9071    if let Some(error) = recovery_block_error(spec, roster) {
9072        return Err(error);
9073    }
9074    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9075        return Err(previous_instance_error(spec, pid, reason));
9076    }
9077    if roster.is_closed() {
9078        return Err(SuperviseError::Spawn {
9079            program: spec.program.clone(),
9080            source: io::Error::other("the daemon is shutting down; not starting a new process"),
9081            cgroup_path: None,
9082        });
9083    }
9084    let carried = containment::current(&roster.containment);
9085    let carrying = carries_containment(spec.carry_containment, carried.is_some());
9086    if role != SpawnRole::Plain && carrying {
9087        return Err(SuperviseError::InvalidSpec {
9088            reason: "a carried container cannot overlap a swap candidate".to_owned(),
9089        });
9090    }
9091    #[cfg(unix)]
9092    let launch_identity = if carrying && carried.is_none() {
9093        Some(
9094            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9095                program: spec.program.clone(),
9096                source,
9097                cgroup_path: None,
9098            })?,
9099        )
9100    } else {
9101        None
9102    };
9103    #[cfg(target_os = "linux")]
9104    let cgroup_placement = match &carried {
9105        Some(container) => container.cgroup().map(|(placement, _)| placement),
9106        None => cgroup_placement,
9107    };
9108    #[cfg(unix)]
9109    let group_target = if let Some(container) = carried
9110        .as_ref()
9111        .filter(|container| container.group().is_some())
9112    {
9113        containment::GroupTarget::Pinned(Arc::clone(container))
9114    } else {
9115        #[cfg(target_os = "linux")]
9116        let cgroup_container = carrying && cgroup_placement.is_some();
9117        #[cfg(not(target_os = "linux"))]
9118        let cgroup_container = false;
9119        if cgroup_container {
9120            containment::GroupTarget::NoGroup
9121        } else {
9122            containment::GroupTarget::OwnLeader
9123        }
9124    };
9125    #[cfg(target_os = "linux")]
9126    let cgroup_name = {
9127        // Slot names alone are not kill domains: a retired incumbent may still
9128        // be draining when a later enable/restart spawns into the same slot.
9129        // Decimal entropy keeps the suffix unambiguous; Placement performs
9130        // the module-id escaping and constructs the filesystem path.
9131        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9132            name.to_owned()
9133        } else if cgroup_placement.is_none() {
9134            swap::cgroup_name(&spec.module_id, alternate_slot)
9135        } else {
9136            let nonce = generate_launch_nonce()?;
9137            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9138            // Leave room for byte escaping and the suffix under NAME_MAX. The
9139            // label is only for humans; the nonce identifies the kill domain.
9140            let mut end = spec.module_id.len().min(64);
9141            while !spec.module_id.is_char_boundary(end) {
9142                end -= 1;
9143            }
9144            format!(
9145                "{}_{suffix}",
9146                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9147            )
9148        }
9149    };
9150    #[cfg(not(target_os = "linux"))]
9151    let _ = alternate_slot;
9152    #[cfg(target_os = "macos")]
9153    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9154    #[cfg(windows)]
9155    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9156        program: spec.program.clone(),
9157        source,
9158        cgroup_path: None,
9159    })?;
9160    #[cfg(windows)]
9161    let mut command = Command::new(image_capture.path());
9162    #[cfg(not(any(target_os = "macos", windows)))]
9163    let mut command = Command::new(&spec.program);
9164    command.args(&spec.args);
9165    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
9166    // that is the whole of the intent, so remove that one key rather than the
9167    // environment.
9168    //
9169    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
9170    // and took the POSIX environment with it. Modules spawned that way had no
9171    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
9172    // logging:
9173    //
9174    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
9175    //     both unset it fell back to the temp dir alone and `ck` could not find
9176    //     a daemon running on the same machine from inside any module's process
9177    //     tree — reporting a path the file has never lived at, which reads as
9178    //     "the daemon did not write its file".
9179    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
9180    //     the RELATIVE `.local/share`, so a module deriving its own store path
9181    //     resolved it against its own CWD. That is the store-fragmentation
9182    //     defect the daemon already refuses in config (`parse_doc` rejects a
9183    //     relative `storage.data_home`) arriving by derivation instead.
9184    //   * anything a module spawns inherited it: git without ~/.gitconfig,
9185    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
9186    //     quietly rather than erroring.
9187    //
9188    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
9189    // offered one candidate under /tmp while the file sat in /run/user/1000.
9190    //
9191    // A configured module is unaffected either way: `module_spec()` puts the
9192    // resolved CK_LOG into `spec.env`, which is applied below and therefore
9193    // wins over anything ambient.
9194    apply_child_env(&mut command, spec);
9195    apply_spawn_role(&mut command, role);
9196    let nonce_handoff =
9197        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9198
9199    #[cfg(target_os = "linux")]
9200    let cgroup_path = cgroup_placement
9201        .map(|placement| {
9202            if let Some(container) = &carried {
9203                let path = container
9204                    .cgroup_path()
9205                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9206                placement.existing_module_path(path)?.ok_or_else(|| {
9207                    io::Error::new(
9208                        io::ErrorKind::NotFound,
9209                        "carried cgroup disappeared before replacement",
9210                    )
9211                })
9212            } else {
9213                if carrying {
9214                    containment::prepare_cgroup_path(placement, &cgroup_name)
9215                } else {
9216                    placement.module_path(&cgroup_name)
9217                }
9218            }
9219        })
9220        .transpose()
9221        .map_err(|source| SuperviseError::Cgroup {
9222            module_id: spec.module_id.clone(),
9223            source,
9224        })?;
9225    #[cfg(not(target_os = "linux"))]
9226    let cgroup_path: Option<PathBuf> = None;
9227    #[cfg(unix)]
9228    let mut admission_gate = if carrying {
9229        Some(
9230            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9231                SuperviseError::Spawn {
9232                    program: spec.program.clone(),
9233                    source,
9234                    cgroup_path: cgroup_path.clone(),
9235                }
9236            })?,
9237        )
9238    } else {
9239        None
9240    };
9241    #[cfg(target_os = "macos")]
9242    exec_ack.install(command.as_std_mut());
9243    #[cfg(target_os = "linux")]
9244    if let Some(path) = &cgroup_path {
9245        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9246            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9247                remove_module_cgroup(placement, &cgroup_name);
9248            }
9249            return Err(error);
9250        }
9251    }
9252
9253    let output_sink = if let Some(logs_dir) = capture_logs_dir {
9254        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9255        match ChildOutputSink::open(&path, capture_retention(spec)) {
9256            Ok(sink) => sink,
9257            Err(error) => {
9258                warn!(
9259                    module_id = %spec.module_id,
9260                    path = %path.display(),
9261                    error = %error,
9262                    "could not open child output capture file; forwarding to stderr"
9263                );
9264                ChildOutputSink::Stderr
9265            }
9266        }
9267    } else {
9268        ChildOutputSink::Stderr
9269    };
9270
9271    command.stdout(Stdio::piped());
9272    command.stderr(Stdio::piped());
9273    command.kill_on_drop(!carrying);
9274    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
9275    // before exec). In the daemon's group, a service manager that kills the
9276    // job's process group when the daemon exits (launchd's default) killed
9277    // every module at the same moment its control connection closed, so no
9278    // module ever ran its EOF teardown on a daemon stop. Outside that group a
9279    // module is reached only by the daemon: the EOF it sees when its
9280    // connection closes, and the bounded stop in `child_roster` for anything
9281    // still running after that. On Linux this composes with the cgroup
9282    // placement above: that is a pre_exec write to cgroup.procs, std performs
9283    // setpgid in the child before running pre_exec callbacks, and the two
9284    // change independent process attributes.
9285    //
9286    // stdin is /dev/null because a process outside the terminal's foreground
9287    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
9288    // by hand would otherwise hand down. Under a service manager stdin is
9289    // already /dev/null.
9290    #[cfg(unix)]
9291    command.process_group(
9292        carried
9293            .as_ref()
9294            .map(|container| container.verified_join_group())
9295            .transpose()
9296            .map_err(|source| SuperviseError::Spawn {
9297                program: spec.program.clone(),
9298                source,
9299                cgroup_path: cgroup_path.clone(),
9300            })?
9301            .flatten()
9302            .unwrap_or(0) as i32,
9303    );
9304    command.stdin(Stdio::null());
9305    #[cfg(unix)]
9306    if let Some(gate) = &mut admission_gate {
9307        gate.install_placement(&mut command);
9308    }
9309    // The LAST pre-exec step, after the cgroup placement above: installing the
9310    // nonce at descriptor 3 replaces whatever the child had there, which could
9311    // be the descriptor an earlier step writes through.
9312    #[cfg(unix)]
9313    if let Some(handoff) = nonce_handoff {
9314        handoff.install_last(command.as_std_mut());
9315    }
9316    #[cfg(not(any(unix, windows)))]
9317    let _ = nonce_handoff;
9318
9319    // Containment, step 1 of 3 (issue #109): create the child suspended so it
9320    // cannot run a single instruction -- and therefore cannot spawn a
9321    // grandchild -- before it is in the job. See `contain_spawned_child` for the
9322    // other two steps and why the window matters.
9323    #[cfg(windows)]
9324    {
9325        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9326            program: spec.program.clone(),
9327            source,
9328            cgroup_path: None,
9329        })?;
9330        subc_jobobject::suspend_on_create_async(&mut command);
9331    }
9332    // Timestamp before the spawn call: on Unix the child can serve before that
9333    // call even returns. This conservative bound never labels an older child
9334    // as a replacement for a request received during process creation.
9335    let spawned_at = Instant::now();
9336    #[cfg(all(test, unix))]
9337    containment_tests::record_spawn_attempt(spec, &mut command);
9338    #[cfg(unix)]
9339    let spawned = match admission_gate {
9340        Some(gate) => gate
9341            .spawn(
9342                command,
9343                spec,
9344                roster,
9345                launch_identity,
9346                #[cfg(target_os = "linux")]
9347                cgroup_placement
9348                    .zip(cgroup_path.as_ref())
9349                    .map(|(placement, path)| {
9350                        (placement.clone(), cgroup_name.clone(), path.clone())
9351                    }),
9352            )
9353            .map(|(child, admitted)| (child, Some(admitted))),
9354        None => command.spawn().map(|child| (child, None)),
9355    };
9356    #[cfg(not(unix))]
9357    let spawned = command.spawn();
9358    let spawned = match spawned {
9359        Ok(child) => child,
9360        Err(source) => {
9361            if carried
9362                .as_ref()
9363                .and_then(|container| container.group())
9364                .is_some()
9365            {
9366                containment::terminate_blocking(&roster.containment);
9367            }
9368            #[cfg(target_os = "linux")]
9369            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9370                remove_module_cgroup(placement, &cgroup_name);
9371            }
9372            return Err(SuperviseError::Spawn {
9373                program: spec.program.clone(),
9374                source,
9375                cgroup_path,
9376            });
9377        }
9378    };
9379    #[cfg(unix)]
9380    let (mut child, mut admitted) = spawned;
9381    #[cfg(unix)]
9382    let group_target = admitted
9383        .as_ref()
9384        .map(|attempt| attempt.group_target())
9385        .unwrap_or(group_target);
9386    #[cfg(not(unix))]
9387    let mut child = spawned;
9388    #[cfg(test)]
9389    spawn_record_tests::after_create(&child, spec);
9390    // The parent must close its writer now: the acknowledgement pipe reports EOF
9391    // only when every writer is gone, and the child's copy closes when the
9392    // trampoline replaces itself with the module. Command holds only an integer
9393    // in its pre_exec callback, not another writer.
9394    #[cfg(target_os = "macos")]
9395    drop(exec_ack);
9396
9397    // Bind while the child is still suspended, before job assignment resumes it.
9398    // Keeping the executable handle open across CreateProcessW denies rename
9399    // and deletion, so the file cannot be exchanged during spawn. Bind that
9400    // file ID to the process handle returned by spawn, never a reopened PID.
9401    #[cfg(windows)]
9402    let windows_image = match image_capture.bind_tokio(&child) {
9403        Ok(image) => Arc::new(image),
9404        Err(source) => {
9405            kill_and_reap_rejected_child(&mut child);
9406            return Err(SuperviseError::Spawn {
9407                program: spec.program.clone(),
9408                source,
9409                cgroup_path: None,
9410            });
9411        }
9412    };
9413    // Assign the suspended child to its module job; resume only after its
9414    // process identity has been durably recorded for orphan recovery.
9415    #[cfg(windows)]
9416    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9417    let spawned_at_ms = unix_ms_now();
9418    #[cfg(not(windows))]
9419    let spawned_from = spec.program.clone();
9420    #[cfg(windows)]
9421    let spawned_from = windows_image.path().to_path_buf();
9422    #[cfg(not(windows))]
9423    let spawned_file_identity = spawned_file_identity(&spawned_from);
9424    #[cfg(windows)]
9425    let spawned_file_identity = Some(SpawnedFileIdentity {
9426        device: 0,
9427        inode: 0,
9428        windows_image: Some(Arc::clone(&windows_image)),
9429    });
9430    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9431        program: spec.program.clone(),
9432        source: io::Error::other("spawned child exposed no live pid"),
9433        cgroup_path: cgroup_path.clone(),
9434    })?;
9435    #[cfg(windows)]
9436    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9437    #[cfg(not(windows))]
9438    let process_start_time = admitted
9439        .as_ref()
9440        .map(|admitted| admitted.birth)
9441        .or_else(|| crate::provenance::process_start_time(pid));
9442    #[cfg(windows)]
9443    let process_start_time = Some(windows_image.start_time());
9444    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9445    #[cfg(all(test, target_os = "macos"))]
9446    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9447    // Record the committed kernel image, not the configured pathname: PATH
9448    // lookup and shebang interpretation can select a different executable.
9449    // Linux's posix_spawn can return before that image is installed, so the
9450    // observer also waits for the kernel's first-exec flag to clear.
9451    #[cfg(not(windows))]
9452    let recorded_image = observe_spawned_image(pid);
9453    // spawn() confirms only the first exec, into the trampoline. Never persist
9454    // the trampoline image; the asynchronous acknowledgement publishes the
9455    // module image once the trampoline has replaced itself with the module.
9456    #[cfg(target_os = "macos")]
9457    let recorded_image = if privacy_exec.is_some() {
9458        None
9459    } else {
9460        recorded_image
9461    };
9462    #[cfg(target_os = "linux")]
9463    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9464    #[cfg(not(target_os = "linux"))]
9465    let recorded_cgroup_name = None;
9466    #[cfg(unix)]
9467    let preadmitted = admitted
9468        .as_mut()
9469        .and_then(|admitted| admitted.leader.take());
9470    #[cfg(not(unix))]
9471    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9472    let roster_guard = if let Some(guard) = preadmitted {
9473        Ok(guard)
9474    } else {
9475        roster.admit(
9476            spec.module_id.clone(),
9477            pid,
9478            spec.protocol,
9479            process_start_time,
9480            crate::child_roster::RecordedIdentity {
9481                container: None,
9482                #[cfg(unix)]
9483                carried: None,
9484                #[cfg(not(windows))]
9485                start_time: recorded_image.map(|image| image.start_time),
9486                #[cfg(windows)]
9487                start_time: process_start_time,
9488                #[cfg(not(windows))]
9489                executable: recorded_image
9490                    .and_then(|image| image.executable)
9491                    .map(crate::live_children::ExecutableIdentity::from),
9492                #[cfg(windows)]
9493                executable: Some(crate::live_children::ExecutableIdentity::from(
9494                    windows_image.identity(),
9495                )),
9496                cgroup_name: recorded_cgroup_name,
9497                #[cfg(target_os = "linux")]
9498                cgroup_placement: cgroup_placement.cloned(),
9499            },
9500        )
9501    };
9502    let roster_guard = match roster_guard {
9503        Ok(guard) => guard,
9504        Err(failure) => {
9505            return failure.with_cleanup(|source| {
9506                #[cfg(windows)]
9507                if let Some(job) = &job {
9508                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9509                }
9510                #[cfg(target_os = "linux")]
9511                kill_module_cgroup(cgroup_placement, &cgroup_name);
9512                #[cfg(unix)]
9513                let rejected_group = group_target.clone();
9514                kill_and_reap_rejected_child(
9515                    &mut child,
9516                    #[cfg(unix)]
9517                    rejected_group,
9518                );
9519                #[cfg(target_os = "linux")]
9520                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9521                    remove_module_cgroup(placement, &cgroup_name);
9522                }
9523                Err(SuperviseError::Spawn {
9524                    program: spec.program.clone(),
9525                    source: io::Error::other(format!(
9526                        "could not durably record module child: {source}"
9527                    )),
9528                    cgroup_path,
9529                })
9530            });
9531        }
9532    };
9533    #[cfg(unix)]
9534    if admitted.is_some() {
9535        if let Some(image) = recorded_image {
9536            if let Err(source) = roster_guard.confirm_executable(image) {
9537                kill_and_reap_rejected_child(&mut child, group_target.clone());
9538                return Err(SuperviseError::Spawn {
9539                    program: spec.program.clone(),
9540                    source,
9541                    cgroup_path,
9542                });
9543            }
9544        }
9545    }
9546    // The pre-spawn check can pass just before shutdown begins. Ownership is
9547    // published at the start of admission, even while a record write is queued.
9548    // If shutdown already saw an empty roster before that publication, this
9549    // check sees the roster closed and ends the child before starting a module.
9550    if roster.is_closed() {
9551        if carrying {
9552            #[cfg(unix)]
9553            let rejected_group = admitted
9554                .as_ref()
9555                .map(|admitted| admitted.rejection_target())
9556                .unwrap_or_else(|| group_target.clone());
9557            #[cfg(windows)]
9558            if let Some(job) = &job {
9559                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9560            }
9561            kill_and_reap_rejected_child(
9562                &mut child,
9563                #[cfg(unix)]
9564                rejected_group,
9565            );
9566            #[cfg(all(test, unix))]
9567            let _ =
9568                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9569            drop(roster_guard);
9570            return Err(SuperviseError::Spawn {
9571                program: spec.program.clone(),
9572                source: io::Error::other(
9573                    "the daemon began shutting down during carried admission; ended the container",
9574                ),
9575                cgroup_path,
9576            });
9577        }
9578        // This child was never admitted, so there is no module protocol shutdown to wait for.
9579        #[cfg(target_os = "linux")]
9580        kill_module_cgroup(cgroup_placement, &cgroup_name);
9581        if let Err(error) = child.start_kill() {
9582            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9583        }
9584        #[cfg(target_os = "linux")]
9585        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9586            // This spawn was never admitted, so shutdown has no roster entry
9587            // to await. Do not detach its cleanup: the runtime could exit
9588            // before that task reaps the rejected child and removes its group.
9589            while matches!(child.try_wait(), Ok(None)) {
9590                std::thread::yield_now();
9591            }
9592            if matches!(
9593                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9594                subc_cgroup::KillOutcome::Killed
9595            ) {
9596                if let Ok(path) = placement.module_path(&cgroup_name) {
9597                    while std::fs::read_to_string(path.join("cgroup.events"))
9598                        .ok()
9599                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9600                    {
9601                        std::thread::yield_now();
9602                    }
9603                }
9604            }
9605            remove_module_cgroup(placement, &cgroup_name);
9606        }
9607        drop(roster_guard);
9608        return Err(SuperviseError::Spawn {
9609            program: spec.program.clone(),
9610            source: io::Error::other(
9611                "the daemon began shutting down while this process was starting; ended it",
9612            ),
9613            cgroup_path,
9614        });
9615    }
9616
9617    let stdout_pump = match child.stdout.take() {
9618        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9619        None => {
9620            warn!(
9621                module_id = %spec.module_id,
9622                "spawned child exposed no stdout pipe; file capture will be incomplete"
9623            );
9624            None
9625        }
9626    };
9627    let stderr_pump = match child.stderr.take() {
9628        Some(stderr) => {
9629            let generation = ring
9630                .lock()
9631                .unwrap_or_else(|poisoned| poisoned.into_inner())
9632                .begin_process();
9633            Some(StderrPump {
9634                task: tokio::spawn(pump_stderr_to(
9635                    stderr,
9636                    Arc::clone(ring),
9637                    generation,
9638                    output_sink,
9639                )),
9640                generation,
9641            })
9642        }
9643        None => {
9644            // Spawning succeeded but the pipe did not materialise. Recording it as
9645            // uncaptured keeps the tail honest: the alternative is an empty tail
9646            // that reads as a module which printed nothing.
9647            ring.lock()
9648                .unwrap_or_else(|poisoned| poisoned.into_inner())
9649                .mark_not_captured("stderr pipe was not available on spawn");
9650            warn!(
9651                module_id = %spec.module_id,
9652                "spawned child exposed no stderr pipe; tail will be unavailable"
9653            );
9654            None
9655        }
9656    };
9657
9658    let exec = Arc::new(crate::activity::ExecOutcome::new());
9659    #[cfg(target_os = "macos")]
9660    let pending_exec = privacy_exec.is_some();
9661    #[cfg(not(target_os = "macos"))]
9662    let pending_exec = false;
9663    if !pending_exec {
9664        exec.finish(crate::activity::ExecState::CommittedExec);
9665    }
9666    #[cfg(windows)]
9667    let subtree_target = job
9668        .as_ref()
9669        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9670        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9671    #[cfg(target_os = "linux")]
9672    let subtree_target = cgroup_path
9673        .map(crate::subtree_resources::Target::Cgroup)
9674        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9675    #[cfg(not(any(target_os = "linux", windows)))]
9676    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9677    let mut supervised = SupervisedChild {
9678        subtree_target,
9679        child: containment::ProcessChild::plain(child, pid),
9680        exec: crate::activity::ExecOwner(exec),
9681        #[cfg(windows)]
9682        nonce_delivery,
9683        protocol: spec.protocol,
9684        #[cfg(target_os = "linux")]
9685        module_id: cgroup_name,
9686        #[cfg(target_os = "linux")]
9687        cgroup_placement: cgroup_placement.cloned(),
9688        #[cfg(windows)]
9689        job,
9690        stdout_pump,
9691        stderr_pump,
9692        stderr_ring: Arc::clone(ring),
9693        spawned_at,
9694        registration_deadline: None,
9695        registration_watch: None,
9696        spawned_at_ms,
9697        spawned_from,
9698        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9699        spawned_file_identity,
9700        process_start_time,
9701        process_identity,
9702        pid,
9703        roster_guard: Some(roster_guard),
9704        #[cfg(target_os = "macos")]
9705        privacy_exec,
9706        #[cfg(target_os = "macos")]
9707        report_ready: Arc::new(OnceLock::new()),
9708        spawn_failure: None,
9709    };
9710    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9711        supervised.child.reject(
9712            #[cfg(unix)]
9713            group_target,
9714        );
9715        #[cfg(all(test, unix))]
9716        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9717        return Err(error);
9718    }
9719    #[cfg(unix)]
9720    if let Some(admitted) = admitted.take() {
9721        admitted.disarm();
9722    }
9723    #[cfg(windows)]
9724    if let Err(error) = resume_suspended_child(pid, spec) {
9725        containment::terminate_blocking(&roster.containment);
9726        let _ = supervised.child.start_kill();
9727        return Err(error);
9728    }
9729    Ok(supervised)
9730}
9731
9732#[cfg(test)]
9733mod spawn_record_tests {
9734    use super::*;
9735
9736    thread_local! {
9737        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9738    }
9739
9740    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9741        LAST_PID.set(child.id());
9742        #[cfg(windows)]
9743        if let Some((_, address)) = _spec
9744            .env
9745            .iter()
9746            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9747        {
9748            use std::io::{Read, Write};
9749            let mut stream = std::net::TcpStream::connect(address).unwrap();
9750            stream
9751                .write_all(&child.id().unwrap().to_ne_bytes())
9752                .unwrap();
9753            let mut release = [0];
9754            stream.read_exact(&mut release).unwrap();
9755        }
9756    }
9757
9758    #[cfg(windows)]
9759    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9760        if spec
9761            .env
9762            .iter()
9763            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9764        {
9765            Err(io::Error::other("injected module job assignment failure"))
9766        } else {
9767            Ok(())
9768        }
9769    }
9770
9771    fn spec() -> ModuleSpec {
9772        ModuleSpec {
9773            module_id: "spawn-record-worker".into(),
9774            #[cfg(unix)]
9775            program: "/bin/sleep".into(),
9776            #[cfg(windows)]
9777            program: std::env::current_exe().unwrap(),
9778            #[cfg(unix)]
9779            args: vec!["60".into()],
9780            #[cfg(windows)]
9781            args: vec![
9782                "--exact".into(),
9783                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9784                "--nocapture".into(),
9785            ],
9786            env: vec![],
9787            reserved: false,
9788            reserved_prefixes: vec![],
9789            protocol: ModuleProtocol::None,
9790            overlap: Default::default(),
9791            carry_containment: false,
9792        }
9793    }
9794
9795    #[tokio::test]
9796    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9797        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9798        let supervisor =
9799            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9800                .with_live_children_record(home.join("live-children.json"));
9801        crate::live_children::fail_next_file_sync();
9802        let result = supervisor.spawn(spec());
9803        let failed = result.is_err();
9804        if let Ok(module) = result {
9805            module.stop().await.unwrap();
9806        }
9807        assert!(failed, "record failure must fail spawn");
9808        let pid = LAST_PID.get().expect("the real OS child was created");
9809        assert!(
9810            subc_os::Process::open(pid).unwrap().is_none(),
9811            "child {pid} survived rejected admission or was not reaped"
9812        );
9813    }
9814
9815    #[cfg(target_os = "macos")]
9816    #[test]
9817    #[ignore = "manual on-device latency measurement"]
9818    fn durable_record_write_latency_50() {
9819        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9820        let path = home.join("live-children.json");
9821        let children = vec![crate::live_children::LiveChild {
9822            module_id: "latency-worker".into(),
9823            pid: std::process::id(),
9824            protocol: ModuleProtocol::None,
9825            start_time: Some(1),
9826            executable: None,
9827            cgroup_name: None,
9828            container: None,
9829        }];
9830        let mut samples = Vec::new();
9831        for _ in 0..50 {
9832            let start = std::time::Instant::now();
9833            crate::live_children::write_record(&path, &children).unwrap();
9834            samples.push(start.elapsed());
9835        }
9836        samples.sort();
9837        eprintln!(
9838            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9839            (samples[24] + samples[25]) / 2,
9840            samples[47]
9841        );
9842    }
9843
9844    #[cfg(windows)]
9845    #[test]
9846    fn windows_store_touch_fixture() {
9847        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9848            std::fs::write(path, b"module ran").unwrap();
9849        }
9850    }
9851
9852    #[cfg(windows)]
9853    #[tokio::test]
9854    async fn windows_daemon_fixture() {
9855        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9856            return;
9857        };
9858        let mut spec = spec();
9859        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9860        let supervisor =
9861            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9862        let _module = supervisor.spawn(spec).unwrap();
9863    }
9864
9865    #[cfg(windows)]
9866    #[test]
9867    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9868        use std::io::Read;
9869        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9870        // An outer job models a scheduler/service host. The daemon's root and
9871        // root job must nest without requiring breakaway from this job.
9872        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9873        command
9874            .args([
9875                "--exact",
9876                "supervise::spawn_record_tests::windows_daemon_fixture",
9877                "--nocapture",
9878            ])
9879            .env(
9880                "SUBC_TEST_DAEMON_BARRIER",
9881                listener.local_addr().unwrap().to_string(),
9882            );
9883        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9884        listener.set_nonblocking(true).unwrap();
9885        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9886        let mut stream = loop {
9887            if let Ok((stream, _)) = listener.accept() {
9888                break stream;
9889            }
9890            assert!(
9891                daemon.child.try_wait().unwrap().is_none(),
9892                "daemon failed before rendezvous"
9893            );
9894            assert!(
9895                std::time::Instant::now() < deadline,
9896                "daemon did not reach CreateProcess failpoint"
9897            );
9898            std::thread::yield_now();
9899        };
9900        stream
9901            .set_read_timeout(Some(Duration::from_secs(30)))
9902            .unwrap();
9903        let mut pid = [0; 4];
9904        stream.read_exact(&mut pid).unwrap();
9905        let pid = u32::from_ne_bytes(pid);
9906        assert!(subc_jobobject::process_exists(pid));
9907        // Terminate only the daemon, not its outer job: otherwise the host job
9908        // would also kill a child whose daemon never joined its root job.
9909        daemon.child.kill().unwrap();
9910        daemon.child.wait().unwrap();
9911        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9912        if !exited {
9913            daemon.job.terminate().unwrap();
9914        }
9915        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9916    }
9917
9918    #[cfg(windows)]
9919    #[tokio::test]
9920    async fn windows_containment_failure_never_resumes() {
9921        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9922        let sentinel = home.join("store-touched");
9923        let mut spec = spec();
9924        spec.env
9925            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9926        spec.env.push((
9927            "SUBC_TEST_STORE_TOUCH".into(),
9928            sentinel.display().to_string(),
9929        ));
9930        let supervisor =
9931            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9932        let result = supervisor.spawn(spec);
9933        // A version that resumes despite assignment failure must have time to
9934        // execute its sentinel before dropping the returned module kills it.
9935        if let Ok(module) = result {
9936            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9937                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9938                    tokio::task::yield_now().await;
9939                }
9940            })
9941            .await;
9942            let _ = module.stop().await;
9943            panic!(
9944                "assignment failure resumed a module; store touched={}",
9945                sentinel.exists()
9946            );
9947        }
9948        assert!(
9949            !sentinel.exists(),
9950            "refused module executed store-touch sentinel"
9951        );
9952        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9953    }
9954
9955    #[cfg(windows)]
9956    #[tokio::test]
9957    async fn windows_nested_daemon_fixture() {
9958        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9959            return;
9960        };
9961        let mut spec = spec();
9962        spec.env.push((
9963            "SUBC_TEST_STORE_TOUCH".into(),
9964            sentinel.to_str().unwrap().into(),
9965        ));
9966        let supervisor =
9967            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9968        let module = supervisor
9969            .spawn(spec)
9970            .expect("per-module job must nest below daemon and host jobs");
9971        tokio::time::timeout(Duration::from_secs(10), async {
9972            while !PathBuf::from(&sentinel).exists() {
9973                tokio::task::yield_now().await;
9974            }
9975        })
9976        .await
9977        .unwrap();
9978        module.stop().await.unwrap();
9979    }
9980
9981    #[cfg(windows)]
9982    #[test]
9983    fn windows_module_job_nests_under_host_job() {
9984        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9985        let sentinel = home.join("nested-store-touched");
9986        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9987        command
9988            .args([
9989                "--exact",
9990                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9991                "--nocapture",
9992            ])
9993            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9994        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9995        assert!(daemon.child.wait().unwrap().success());
9996        assert!(sentinel.exists(), "nested protocol:none module never ran");
9997    }
9998}
9999
10000#[cfg(target_os = "linux")]
10001pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10002    use subc_cgroup::KillOutcome;
10003    match subc_cgroup::kill_module(placement, module_id) {
10004        KillOutcome::Killed => {}
10005        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10006            debug!(
10007                module_id,
10008                "cgroup tree kill unavailable; using direct-child kill"
10009            );
10010        }
10011        KillOutcome::IoError { path, error } => {
10012            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10013        }
10014    }
10015}
10016
10017/// Assign a freshly spawned Windows child without ever resuming on failure.
10018///
10019/// Steps 2 and 3 of the suspended-create contract: the job is created and the
10020/// child assigned **while it is still suspended** (step 1 is
10021/// `suspend_on_create_async` at the spawn site). The caller resumes only after
10022/// the child's roster admission is durable. Failed containment kills and reaps
10023/// the suspended child instead of running an unprotected module.
10024#[cfg(windows)]
10025fn contain_spawned_child(
10026    child: &mut Child,
10027    spec: &ModuleSpec,
10028    carried: Option<&Arc<containment::CarriedContainment>>,
10029) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10030    let assignment = || {
10031        let job = match carried {
10032            Some(container) => container.job(),
10033            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10034        };
10035        #[cfg(test)]
10036        spawn_record_tests::assignment_failpoint(spec)?;
10037        job.lock()
10038            .unwrap_or_else(|p| p.into_inner())
10039            .assign(child)?;
10040        Ok::<_, io::Error>(job)
10041    };
10042    match assignment() {
10043        Ok(job) => Ok(Some(job)),
10044        Err(source) => {
10045            kill_and_reap_rejected_child(child);
10046            Err(SuperviseError::Spawn {
10047                program: spec.program.clone(),
10048                source: io::Error::other(format!(
10049                    "Windows module job containment failed: {source}"
10050                )),
10051                cgroup_path: None,
10052            })
10053        }
10054    }
10055}
10056
10057/// Rejection is synchronous: do not return a spawn error while the child can
10058/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
10059fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10060    #[cfg(unix)]
10061    group.signal_rejected(child);
10062    let _ = child.start_kill();
10063    while matches!(child.try_wait(), Ok(None)) {
10064        std::thread::yield_now();
10065    }
10066}
10067
10068/// Resume a suspended child, killing it if it cannot be started.
10069///
10070/// A suspended process holds a pid and does nothing, so there is no useful
10071/// state to return: the caller gets an error and the spawn fails.
10072#[cfg(windows)]
10073fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10074    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10075        // Kill it here rather than leaving a suspended process for the caller
10076        // to notice; `kill_on_drop` would eventually do this, but the module
10077        // would have been reported as running in between.
10078        let _ = std::process::Command::new("taskkill.exe")
10079            .args(["/PID", &pid.to_string(), "/T", "/F"])
10080            .stdin(Stdio::null())
10081            .stdout(Stdio::null())
10082            .stderr(Stdio::null())
10083            .status();
10084        return Err(SuperviseError::Spawn {
10085            program: spec.program.clone(),
10086            source,
10087            cgroup_path: None,
10088        });
10089    }
10090    Ok(())
10091}
10092
10093#[cfg(target_os = "linux")]
10094fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10095    match placement.remove_module(module_id) {
10096        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10097        Err(error) => warn!(
10098            module_id,
10099            error = %error,
10100            "could not remove module cgroup after process exit; continuing teardown"
10101        ),
10102    }
10103}
10104
10105#[cfg(target_os = "linux")]
10106const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10107#[cfg(target_os = "linux")]
10108const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10109
10110#[cfg(target_os = "linux")]
10111async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10112    let path = placement.module_path(module_id).ok();
10113    if path.as_ref().is_some_and(|path| {
10114        std::fs::read_to_string(path.join("cgroup.events"))
10115            .ok()
10116            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10117    }) {
10118        remove_module_cgroup(placement, module_id);
10119        return;
10120    }
10121
10122    // Freeze before enumerating: a frozen task cannot exit and release its PID
10123    // for reuse by a process outside this module while SIGTERM is being sent.
10124    if terminate_reaped_cgroup(placement, module_id).await {
10125        if let Some(path) = &path {
10126            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10127            while cgroup_is_populated(path) && Instant::now() < deadline {
10128                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10129            }
10130        }
10131    }
10132
10133    // Reaping the direct child is not proof its descendants exited. Kill any
10134    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
10135    // cgroup empty before rmdir.
10136    if matches!(
10137        subc_cgroup::kill_module(Some(placement), module_id),
10138        subc_cgroup::KillOutcome::Killed
10139    ) {
10140        if let Some(path) = &path {
10141            while cgroup_is_populated(path) {
10142                sleep(Duration::from_millis(1)).await;
10143            }
10144        }
10145    }
10146    remove_module_cgroup(placement, module_id);
10147}
10148
10149#[cfg(target_os = "linux")]
10150fn cgroup_is_populated(path: &std::path::Path) -> bool {
10151    std::fs::read_to_string(path.join("cgroup.events"))
10152        .ok()
10153        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10154}
10155
10156#[cfg(target_os = "linux")]
10157async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10158    let frozen =
10159        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10160            .await
10161        {
10162            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10163            outcome => {
10164                tracing::debug!(
10165                    module_id,
10166                    ?outcome,
10167                    "cgroup freeze failed; skipping SIGTERM grace"
10168                );
10169                return false;
10170            }
10171        };
10172    let listed = match frozen.processes() {
10173        subc_cgroup::ProcsOutcome::Listed(pids) => {
10174            for pid in pids.into_iter().filter(|pid| *pid > 1) {
10175                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10176                    if let Err(error) =
10177                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10178                    {
10179                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10180                    }
10181                }
10182            }
10183            true
10184        }
10185        outcome => {
10186            tracing::debug!(
10187                module_id,
10188                ?outcome,
10189                "cgroup process listing failed; skipping SIGTERM grace"
10190            );
10191            false
10192        }
10193    };
10194    match frozen.thaw() {
10195        subc_cgroup::ThawOutcome::Thawed => listed,
10196        outcome => {
10197            tracing::debug!(
10198                module_id,
10199                ?outcome,
10200                "cgroup thaw failed; proceeding to cgroup.kill"
10201            );
10202            false
10203        }
10204    }
10205}
10206
10207#[cfg(target_os = "linux")]
10208fn apply_cgroup_placement(
10209    command: &mut Command,
10210    spec: &ModuleSpec,
10211    path: &std::path::Path,
10212) -> Result<(), SuperviseError> {
10213    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10214        module_id: spec.module_id.clone(),
10215        source,
10216    })
10217}
10218
10219fn capture_retention(spec: &ModuleSpec) -> Retention {
10220    let defaults = Retention::default();
10221    let value = |name: &str| {
10222        spec.env
10223            .iter()
10224            .rev()
10225            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10226    };
10227    Retention {
10228        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10229            .and_then(|value| value.parse().ok())
10230            .unwrap_or(defaults.max_file_mb),
10231        keep: value(CAPTURE_KEEP_ENV)
10232            .and_then(|value| value.parse().ok())
10233            .unwrap_or(defaults.keep),
10234        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10235            .and_then(|value| value.parse().ok())
10236            .unwrap_or(defaults.max_age_days),
10237    }
10238}
10239
10240/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
10241/// module's registration to the exact process the supervisor spawned.
10242fn generate_launch_nonce() -> Result<String, SuperviseError> {
10243    let mut bytes = [0u8; 32];
10244    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10245        reason: source.to_string(),
10246    })?;
10247    let mut hex = String::with_capacity(64);
10248    for b in bytes {
10249        use std::fmt::Write;
10250        let _ = write!(hex, "{b:02x}");
10251    }
10252    Ok(hex)
10253}
10254
10255/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
10256/// signal about how many leading bytes matched.
10257fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10258    if a.len() != b.len() {
10259        return false;
10260    }
10261    let mut diff = 0u8;
10262    for (x, y) in a.iter().zip(b.iter()) {
10263        diff |= x ^ y;
10264    }
10265    diff == 0
10266}
10267
10268/// The kernel's image after an acknowledged exec, shared by ordinary launches
10269/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
10270/// not identities inferred from a configured pathname.
10271#[cfg(not(windows))]
10272fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10273    let process = subc_os::Process::open(pid).ok().flatten()?;
10274    #[cfg(target_os = "linux")]
10275    return sample_after_linux_exec(
10276        || {
10277            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10278            let committed = linux_exec_committed(&stat);
10279            #[cfg(test)]
10280            spawn_image_tests::report_exec_state(committed);
10281            committed
10282        },
10283        || process.observe(),
10284    );
10285    #[cfg(not(target_os = "linux"))]
10286    process.observe()
10287}
10288
10289/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
10290/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
10291/// installation. Until it clears, /proc/pid/exe can still name the daemon.
10292#[cfg(target_os = "linux")]
10293fn linux_exec_committed(stat: &str) -> Option<bool> {
10294    const PF_FORKNOEXEC: u64 = 0x40;
10295    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10296    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10297}
10298
10299#[cfg(target_os = "linux")]
10300fn sample_after_linux_exec(
10301    mut exec_committed: impl FnMut() -> Option<bool>,
10302    observe: impl FnOnce() -> Option<subc_os::Observation>,
10303) -> Option<subc_os::Observation> {
10304    let deadline = std::time::Instant::now() + Duration::from_secs(1);
10305    while !exec_committed()? {
10306        // A failed or indefinitely stalled exec must not publish the inherited
10307        // image as authority to signal a future orphan. Missing identity keeps
10308        // the existing conservative orphan-sweep behavior.
10309        if std::time::Instant::now() >= deadline {
10310            return None;
10311        }
10312        std::thread::sleep(Duration::from_micros(50));
10313    }
10314    observe()
10315}
10316
10317#[cfg(all(test, target_os = "linux"))]
10318mod spawn_image_tests {
10319    use super::*;
10320    use std::cell::{Cell, RefCell};
10321
10322    enum ExecEvent {
10323        Uncommitted,
10324        Image(Option<subc_os::Observation>),
10325    }
10326
10327    thread_local! {
10328        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10329    }
10330
10331    pub(super) fn report_exec_state(committed: Option<bool>) {
10332        if committed == Some(false) {
10333            EXEC_EVENTS.with(|events| {
10334                if let Some(events) = events.borrow_mut().take() {
10335                    let _ = events.send(ExecEvent::Uncommitted);
10336                }
10337            });
10338        }
10339    }
10340
10341    #[test]
10342    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10343        use std::io::{Read, Write};
10344        use std::os::fd::AsRawFd;
10345
10346        let descriptor = std::fs::File::open("/dev/null").unwrap();
10347        let mut command = std::process::Command::new("/bin/sleep");
10348        command.arg("30");
10349        let (mut ready, mut release) =
10350            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10351                .unwrap();
10352        let spawn = std::thread::spawn(move || command.spawn());
10353        let mut message = [0u8; 8];
10354        ready.read_exact(&mut message).unwrap();
10355        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10356        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10357        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10358        let (events, received) = std::sync::mpsc::channel();
10359        let observer = std::thread::spawn(move || {
10360            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10361            let image = observe_spawned_image(pid);
10362            let _ = events.send(ExecEvent::Image(image));
10363        });
10364        // Release only once the production observer has seen the uncommitted
10365        // exec, or has incorrectly returned the inherited image. Either way,
10366        // release and reap before asserting so a regression cannot leak a child.
10367        let first = received.recv_timeout(Duration::from_secs(5));
10368        release.write_all(&[1]).unwrap();
10369        let mut child = spawn.join().unwrap().unwrap();
10370        let after = match first {
10371            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10372                Ok(ExecEvent::Image(image)) => image,
10373                _ => None,
10374            },
10375            Ok(ExecEvent::Image(image)) => image,
10376            Err(_) => None,
10377        };
10378        observer.join().unwrap();
10379        child.kill().unwrap();
10380        child.wait().unwrap();
10381
10382        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10383        assert_eq!(
10384            inherited.unwrap().executable,
10385            subc_os::file_identity(&std::env::current_exe().unwrap())
10386        );
10387        assert_eq!(
10388            after.unwrap().executable,
10389            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10390        );
10391    }
10392
10393    #[test]
10394    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10395        let committed = Cell::new(false);
10396        let inherited = subc_os::Observation {
10397            start_time: 23,
10398            executable: Some(subc_os::FileIdentity {
10399                device: 29,
10400                inode: 31,
10401            }),
10402        };
10403        let module = subc_os::Observation {
10404            start_time: 23,
10405            executable: Some(subc_os::FileIdentity {
10406                device: 29,
10407                inode: 37,
10408            }),
10409        };
10410        let polls = Cell::new(0);
10411        let image = sample_after_linux_exec(
10412            || {
10413                polls.set(polls.get() + 1);
10414                // Hold the inherited image through the first poll, as in the
10415                // posix_spawn parent-wake window, then commit the module image.
10416                committed.set(polls.get() == 2);
10417                Some(committed.get())
10418            },
10419            || Some(if committed.get() { module } else { inherited }),
10420        );
10421        assert_eq!(image, Some(module));
10422        assert_eq!(polls.get(), 2);
10423    }
10424
10425    #[test]
10426    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10427        assert_eq!(
10428            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10429            Some(false)
10430        );
10431        assert_eq!(
10432            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10433            Some(true)
10434        );
10435        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10436        assert_eq!(linux_exec_committed("17 (a) R"), None);
10437    }
10438
10439    #[test]
10440    fn unreadable_exec_state_never_samples_an_inherited_image() {
10441        assert_eq!(
10442            sample_after_linux_exec(
10443                || None,
10444                || panic!("must not sample before exec confirmation")
10445            ),
10446            None
10447        );
10448    }
10449
10450    #[test]
10451    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10452        assert_eq!(
10453            sample_after_linux_exec(
10454                || Some(false),
10455                || panic!("must not sample after an exec timeout")
10456            ),
10457            None
10458        );
10459    }
10460}
10461
10462/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
10463/// different executable after the no-delete capture was taken.
10464#[cfg(windows)]
10465fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10466    let mut program = spec.program.clone();
10467    if program.is_absolute() || program.components().count() > 1 {
10468        // An explicit path can name a PE image with no extension. Try that
10469        // exact file first, as Command does, before falling back to .exe.
10470        match subc_os::ExecutableCapture::open(&program) {
10471            Ok(capture) => return Ok(capture),
10472            Err(error)
10473                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10474            Err(error) => return Err(error),
10475        }
10476        program.set_extension("exe");
10477        return subc_os::ExecutableCapture::open(&program);
10478    }
10479    if program.extension().is_none() {
10480        program.set_extension("exe");
10481    }
10482    let custom_path = spec
10483        .env
10484        .iter()
10485        .rev()
10486        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10487    let mut directories = Vec::new();
10488    // These are Windows' default executable search locations. A configured
10489    // PATH still overrides the inherited PATH, not the system directories.
10490    if let Some(parent) = std::env::current_exe()?.parent() {
10491        directories.push(parent.to_path_buf());
10492    }
10493    directories.push(std::env::current_dir()?);
10494    if let Some(windows) = std::env::var_os("SystemRoot") {
10495        let windows = PathBuf::from(windows);
10496        directories.push(windows.join("System32"));
10497        directories.push(windows.clone());
10498    }
10499    let path = custom_path
10500        .map(|(_, value)| std::ffi::OsString::from(value))
10501        .or_else(|| std::env::var_os("PATH"));
10502    if let Some(path) = path {
10503        directories.extend(std::env::split_paths(&path));
10504    }
10505    for directory in directories {
10506        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10507            Ok(capture) => return Ok(capture),
10508            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10509            Err(error) => return Err(error),
10510        }
10511    }
10512    Err(io::Error::new(
10513        io::ErrorKind::NotFound,
10514        "resolved executable not found",
10515    ))
10516}
10517
10518fn spawn_and_mark_running(
10519    spec: &ModuleSpec,
10520    runtime: &SupervisorRuntimeConfig,
10521    snapshot: &SharedSnapshot,
10522) -> Result<SupervisedChild, SuperviseError> {
10523    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10524    let registration_watch = watch_registration(&runtime.registry, spec);
10525    #[cfg(unix)]
10526    let joining_group = containment::current(&runtime.child_roster.containment)
10527        .is_some_and(|container| container.group().is_some());
10528    let spawn = || {
10529        spawn_child(
10530            spec,
10531            runtime.connection_file_path.as_deref(),
10532            runtime.supervisor_handle.as_ref(),
10533            &runtime.stderr_ring,
10534            runtime.capture_logs_dir.as_deref(),
10535            &runtime.child_roster,
10536            #[cfg(target_os = "linux")]
10537            runtime.cgroup_placement.as_ref(),
10538        )
10539    };
10540    let result = spawn();
10541    #[cfg(unix)]
10542    let result = if result.is_err()
10543        && joining_group
10544        && !runtime.child_roster.is_closed()
10545        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10546        && runtime
10547            .child_roster
10548            .previous_instance(&spec.module_id)
10549            .is_none()
10550    {
10551        // The failed join already destroyed and verified the old container.
10552        // Try one fresh group as part of this same restart, without spending
10553        // another crash-budget entry. Cgroups and Windows jobs do not retry.
10554        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10555        spawn()
10556    } else {
10557        result
10558    };
10559    let child = result?;
10560    let mut child = child;
10561    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10562    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10563        containment::terminate_blocking(&runtime.child_roster.containment);
10564        return Err(error);
10565    }
10566    if let Some(spawn) = activity_spawn {
10567        spawn.complete(snapshot);
10568    }
10569    Ok(child)
10570}
10571
10572struct RegistrationWatch {
10573    changes: watch::Receiver<u64>,
10574    baseline: u64,
10575}
10576
10577// Capture before spawning: a fast child's accepted HELLO can disappear from the
10578// active registry before the monitor first runs. A pre-existing registration
10579// must not satisfy a new process's deadline either.
10580fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10581    if spec.protocol != ModuleProtocol::Subc {
10582        return None;
10583    }
10584    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10585    let baseline = *changes.borrow();
10586    Some(RegistrationWatch { changes, baseline })
10587}
10588
10589fn arm_registration_deadline(
10590    child: &mut SupervisedChild,
10591    budget: Duration,
10592    registration_watch: Option<RegistrationWatch>,
10593) {
10594    child.registration_deadline =
10595        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10596    child.registration_watch = registration_watch;
10597}
10598
10599async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10600    if let Some(container) = child.child.container() {
10601        container.stop_members().await;
10602        child.kill_leader()?;
10603        child.wait().await?;
10604        return Ok(());
10605    }
10606    #[cfg(unix)]
10607    {
10608        // Children lead their own process groups. Signal the whole group so a
10609        // helper stuck before HELLO cannot survive its supervised parent.
10610        let group = unreaped_registration_group(child)?;
10611        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10612        // Process-group safety rule: only signal a group while its leader is
10613        // our unreaped child. Even a zombie leader reserves its pid/group id;
10614        // reaping it before KILL would let that id name an unrelated group.
10615        // Sleep without waiting so TERM-ignoring helpers get the full grace.
10616        sleep(crate::child_roster::TERM_TO_KILL).await;
10617        let group = unreaped_registration_group(child)?;
10618        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10619    }
10620    // Windows terminates the containing Job Object as well as the direct
10621    // child. Linux also kills the child's cgroup when placement is enabled.
10622    child.start_kill()?;
10623    child.wait().await?;
10624    Ok(())
10625}
10626
10627#[cfg(unix)]
10628fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10629    if let Some(container) = child.child.container() {
10630        return container
10631            .group()
10632            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10633            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10634    }
10635    // The group id comes only from the pid recorded at spawn, when the child
10636    // created its own group. Never admit init's group or a negative group id.
10637    let group = i32::try_from(child.pid)
10638        .ok()
10639        .filter(|pid| *pid > 1)
10640        .and_then(rustix::process::Pid::from_raw)
10641        .ok_or_else(|| {
10642            io::Error::new(
10643                io::ErrorKind::InvalidInput,
10644                "invalid supervised process group",
10645            )
10646        })?;
10647    if child.child.id() != Some(child.pid) {
10648        return Err(io::Error::new(
10649            io::ErrorKind::InvalidInput,
10650            "refusing process-group signal after supervised child was reaped",
10651        ));
10652    }
10653    Ok(group)
10654}
10655
10656enum RegistrationWaitOutcome {
10657    Registered,
10658    Exited(ExitReport),
10659    TimedOut,
10660}
10661
10662struct ReloadRegistrationFailure {
10663    exit_report: ExitReport,
10664    reason: String,
10665}
10666
10667#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10668enum BusyGaugeObservation {
10669    Quiescent,
10670    Busy,
10671    Omitted,
10672}
10673
10674/// What one health answer said about one declared busy gauge.
10675#[derive(Debug, Clone, PartialEq, Eq)]
10676enum GaugeVerdict {
10677    /// The gauge read 0: this gauge is not what holds the drain.
10678    Zero,
10679    /// The gauge read a positive count.
10680    NonZero(u64),
10681    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10682    /// a negative number). It counts as busy, because the daemon cannot tell
10683    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10684    NotACount(String),
10685    /// The health answer had no metrics object, or one without this gauge.
10686    Missing,
10687    /// The probe produced no health answer to read; carries the probe error's
10688    /// evidence label (`no-answer` for a probe that timed out).
10689    ProbeFailed(&'static str),
10690}
10691
10692impl GaugeVerdict {
10693    fn reads_quiet(&self) -> bool {
10694        matches!(self, Self::Zero)
10695    }
10696}
10697
10698impl fmt::Display for GaugeVerdict {
10699    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10700        match self {
10701            Self::Zero => f.write_str("zero"),
10702            Self::NonZero(value) => write!(f, "non_zero({value})"),
10703            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10704            Self::Missing => f.write_str("missing_from_health_body"),
10705            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10706        }
10707    }
10708}
10709
10710/// The longest JSON text kept for a gauge value that is not a count, so a
10711/// module reporting a large object as a gauge cannot flood the drain log.
10712const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10713
10714/// Read each declared gauge out of one health answer's metrics.
10715fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10716    let metrics = metrics.and_then(Value::as_object);
10717    gauges
10718        .iter()
10719        .map(
10720            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10721                None => GaugeVerdict::Missing,
10722                Some(value) => match value.as_u64() {
10723                    Some(0) => GaugeVerdict::Zero,
10724                    Some(count) => GaugeVerdict::NonZero(count),
10725                    None => {
10726                        let mut text = value.to_string();
10727                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10728                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10729                            while !text.is_char_boundary(end) {
10730                                end -= 1;
10731                            }
10732                            text.truncate(end);
10733                            text.push_str("...");
10734                        }
10735                        GaugeVerdict::NotACount(text)
10736                    }
10737                },
10738            },
10739        )
10740        .collect()
10741}
10742
10743/// The drain's overall reading of one health answer: omitted when any
10744/// declared gauge is missing, quiescent only when every gauge reads 0.
10745fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10746    if verdicts
10747        .iter()
10748        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10749    {
10750        BusyGaugeObservation::Omitted
10751    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10752        BusyGaugeObservation::Quiescent
10753    } else {
10754        BusyGaugeObservation::Busy
10755    }
10756}
10757
10758/// One declared gauge's history over a drain: the latest verdict on it, and
10759/// the latest value actually read for it (which a later failed probe or
10760/// missing gauge does not erase). Times are offsets from the start of the
10761/// drain's wait.
10762#[derive(Debug, Clone, Default, PartialEq, Eq)]
10763struct GaugeTrace {
10764    latest: Option<(GaugeVerdict, Duration)>,
10765    last_value: Option<(GaugeVerdict, Duration)>,
10766}
10767
10768/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10769/// its ceiling can say in its timeout line which declared gauge held it and
10770/// why. Without it, a gauge that read busy, a gauge the module never
10771/// reported, and a probe that never got an answer all looked the same: a
10772/// drain that ended at its ceiling with nothing in flight.
10773#[derive(Debug, Clone, Default, PartialEq, Eq)]
10774struct DrainGaugeTrace {
10775    /// Health probes the drain sent.
10776    probes: u32,
10777    /// Each declared gauge, in declaration order.
10778    gauges: Vec<(String, GaugeTrace)>,
10779}
10780
10781impl DrainGaugeTrace {
10782    fn new(gauges: &[String]) -> Self {
10783        Self {
10784            probes: 0,
10785            gauges: gauges
10786                .iter()
10787                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10788                .collect(),
10789        }
10790    }
10791
10792    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10793    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10794        self.probes = self.probes.saturating_add(1);
10795        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10796            trace.latest = Some((verdict.clone(), at));
10797            if matches!(
10798                verdict,
10799                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10800            ) {
10801                trace.last_value = Some((verdict.clone(), at));
10802            }
10803        }
10804    }
10805}
10806
10807/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10808/// declared gauge, separated by ` | `. A value is shown as its verdict
10809/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10810/// and whether that counted as busy.
10811impl fmt::Display for DrainGaugeTrace {
10812    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10813        if self.gauges.is_empty() {
10814            return f.write_str("none declared");
10815        }
10816        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10817            if index > 0 {
10818                f.write_str(" | ")?;
10819            }
10820            write!(f, "{name}: ")?;
10821            match &trace.latest {
10822                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10823                None => f.write_str("never probed")?,
10824            }
10825            match &trace.last_value {
10826                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10827                None => f.write_str("; no value ever read")?,
10828            }
10829        }
10830        Ok(())
10831    }
10832}
10833
10834pub(crate) fn declared_busy_gauges(
10835    registry: &Registry,
10836    module_id: &str,
10837) -> Result<Vec<String>, SuperviseError> {
10838    busy_gauges_of(
10839        registry
10840            .get_module(module_id)
10841            .map_err(SuperviseError::Registry)?,
10842    )
10843}
10844
10845/// [`declared_busy_gauges`] for the registration a connection holds, in any
10846/// slot: after cutover the incumbent is no longer the id's active
10847/// registration, and its own manifest is the one that names its gauges.
10848fn declared_busy_gauges_for_connection(
10849    registry: &Registry,
10850    connection_id: ConnectionId,
10851) -> Result<Vec<String>, SuperviseError> {
10852    busy_gauges_of(
10853        registry
10854            .get_module_by_connection(connection_id)
10855            .map_err(SuperviseError::Registry)?,
10856    )
10857}
10858
10859fn busy_gauges_of(
10860    registration: Option<crate::registry::ModuleRegistration>,
10861) -> Result<Vec<String>, SuperviseError> {
10862    let Some(registration) = registration else {
10863        return Ok(Vec::new());
10864    };
10865    let Some(self_signals) = registration.manifest.self_signals else {
10866        return Ok(Vec::new());
10867    };
10868
10869    let mut gauges = Vec::new();
10870    for declaration in self_signals {
10871        if declaration.kind != SelfSignalKind::Busy {
10872            continue;
10873        }
10874        match declaration.anchored_to {
10875            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10876                gauges.extend(declared)
10877            }
10878            _ => {
10879                // An invalid Busy anchor is fail-safe: the empty name cannot be
10880                // present in a conforming health report, so this drain stays busy.
10881                gauges.push(String::new());
10882            }
10883        }
10884    }
10885    Ok(gauges)
10886}
10887
10888/// Wait for `endpoint` to have nothing in flight and, when the module declares
10889/// busy gauges, for a health probe to report them quiet. The probe is addressed
10890/// by `scope`: a swap's superseded incumbent must be asked about its own
10891/// gauges, and by module id the probe would reach the promoted candidate.
10892/// While the gauges read busy, the probe repeats every
10893/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10894/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10895/// timeout line.
10896#[allow(clippy::too_many_arguments)]
10897async fn wait_for_forwarding_quiescence(
10898    forwarding: &ForwardingTable,
10899    module_id: &str,
10900    runtime: &SupervisorRuntimeConfig,
10901    endpoint: crate::ModuleEndpointId,
10902    deadline: Instant,
10903    busy_gauges: &[String],
10904    scope: DrainScope,
10905    trace: &mut DrainGaugeTrace,
10906) -> Result<bool, SuperviseError> {
10907    let wait_started = Instant::now();
10908    let mut gauges_quiescent = busy_gauges.is_empty();
10909    let mut next_probe_at = wait_started;
10910    let mut omission_reported = false;
10911
10912    loop {
10913        let now = Instant::now();
10914        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10915            let report = match scope {
10916                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10917                DrainScope::Endpoint(endpoint) => {
10918                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10919                }
10920            };
10921            let verdicts = match &report {
10922                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10923                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10924            };
10925            trace.record(&verdicts, wait_started.elapsed());
10926            gauges_quiescent = match report {
10927                Ok(_) => match busy_gauge_observation(&verdicts) {
10928                    BusyGaugeObservation::Quiescent => true,
10929                    BusyGaugeObservation::Busy => false,
10930                    BusyGaugeObservation::Omitted => {
10931                        // Once per drain: a module that never reports a gauge
10932                        // it declared would otherwise say so on every probe.
10933                        if !omission_reported {
10934                            forwarding
10935                                .counters()
10936                                .increment_drains_with_undeclared_gauge();
10937                            let missing = busy_gauges
10938                                .iter()
10939                                .zip(&verdicts)
10940                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10941                                .map(|(gauge, _)| gauge.as_str())
10942                                .collect::<Vec<_>>()
10943                                .join(",");
10944                            warn!(
10945                                module_id,
10946                                missing_gauges = %missing,
10947                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10948                            );
10949                            omission_reported = true;
10950                        }
10951                        false
10952                    }
10953                },
10954                Err(err) => {
10955                    warn!(
10956                        module_id,
10957                        error = %err,
10958                        "drain health.check did not produce declared busy gauges; treating module as busy"
10959                    );
10960                    false
10961                }
10962            };
10963            // Counted from the probe's completion: the probe above was awaited,
10964            // so no second probe can start while one is still outstanding.
10965            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10966        }
10967
10968        let in_flight = forwarding
10969            .endpoint_in_flight_count(endpoint)
10970            .map_err(SuperviseError::Forwarding)?;
10971        if in_flight == 0 && gauges_quiescent {
10972            return Ok(true);
10973        }
10974
10975        let now = Instant::now();
10976        if now >= deadline {
10977            return Ok(false);
10978        }
10979        let mut wait = deadline
10980            .saturating_duration_since(now)
10981            .min(REGISTRY_RELEASE_POLL);
10982        if !busy_gauges.is_empty() {
10983            wait = wait.min(next_probe_at.saturating_duration_since(now));
10984        }
10985        sleep(wait).await;
10986    }
10987}
10988
10989/// The wait between drain busy-gauge probes: the larger of
10990/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10991/// drain's own in-flight poll interval.
10992fn drain_busy_reprobe_interval() -> Duration {
10993    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10994}
10995
10996/// Logs why a drain gave up: the drain waited its full timeout without the
10997/// module going quiet, and teardown is being forced.
10998///
10999/// The line names two kinds of evidence, because a bare "timed out" hides
11000/// causes that need different fixes:
11001/// - `holdouts`: the requests and routes still open when the timeout hit,
11002///   and on which connections. One stuck call is a module bug; a held stream
11003///   on every session is a module that should end its streams when it
11004///   receives `module.draining`.
11005/// - `gauges`: for each busy gauge the module declares, its latest reading
11006///   and how many probes ran. This separates a module that kept reporting
11007///   itself busy, one whose health answer never contained the gauge, and one
11008///   that never answered the probes, which otherwise all look like a timeout
11009///   with nothing in flight.
11010fn warn_drain_timed_out(
11011    module_id: &str,
11012    drain_timeout: Duration,
11013    reason: RouteCloseReason,
11014    holdouts: &crate::forwarding::DrainHoldouts,
11015    gauges: &DrainGaugeTrace,
11016) {
11017    warn!(
11018        module_id = %module_id,
11019        waited = ?drain_timeout,
11020        ?reason,
11021        held_requests = holdouts.requests,
11022        held_routes = holdouts.routes,
11023        total_routes = holdouts.total_routes,
11024        top_connections = ?holdouts.top_connections,
11025        // `module_channel:corr`, so the module can find each held request
11026        // in its own log; capped, so `held_requests` is the full count.
11027        held = %holdouts
11028            .held
11029            .iter()
11030            .map(|(channel, corr)| format!("{channel}:{corr}"))
11031            .collect::<Vec<_>>()
11032            .join(","),
11033        gauge_probes = gauges.probes,
11034        busy_gauges = %gauges,
11035        "route drain timed out before request quiescence; forcing teardown"
11036    );
11037}
11038
11039/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
11040///
11041/// `Ok` is always honest and passed straight through -- the wait actually measured
11042/// in-flight state. `Err` means the wait produced no measurement at all (the
11043/// forwarding table's lock was poisoned), so `false` is reported as the one honest
11044/// constant: the drain did not complete. Never recomputed from route state, never a
11045/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
11046fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11047    match wait_result {
11048        Ok(drained) => *drained,
11049        Err(_) => false,
11050    }
11051}
11052
11053fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11054    for released in released_routes {
11055        let frame = match Frame::build_with_version(
11056            released.negotiated_ver,
11057            FrameType::Goodbye,
11058            control_flags(),
11059            released.channel,
11060            released.epoch,
11061            0,
11062            Vec::new(),
11063        ) {
11064            Ok(frame) => frame,
11065            Err(err) => {
11066                warn!(
11067                    route_channel = released.channel,
11068                    error = %err,
11069                    "failed to build supervisor drain route GOODBYE frame"
11070                );
11071                continue;
11072            }
11073        };
11074        if !released.close_on_delivery_failure() {
11075            crate::forwarding::send_module_route_goodbye(
11076                &forwarding.counters(),
11077                &released.sink,
11078                frame,
11079                released.module_id.as_deref(),
11080                "supervisor drain",
11081            );
11082            continue;
11083        }
11084        if let Err(err) = released.sink.try_send(frame) {
11085            warn!(
11086                target_connection_id = released.connection_id.get(),
11087                route_channel = released.channel,
11088                error = %err,
11089                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11090            );
11091            let _ = forwarding.escalate_client_delivery_failure(
11092                released.connection_id,
11093                released.channel,
11094                released.epoch,
11095                CloseReason::new(
11096                    "route_goodbye_delivery_failed",
11097                    format!(
11098                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11099                        released.channel
11100                    ),
11101                ),
11102                crate::forwarding::UndeliveredFrame {
11103                    module_id: released.module_id.as_deref(),
11104                    sink: &released.sink,
11105                },
11106            );
11107        }
11108    }
11109}
11110
11111async fn send_module_draining(
11112    module_id: &str,
11113    reason: RouteCloseReason,
11114    deadline_ms: u64,
11115    target: &ModuleDrainTarget,
11116    _wait_for_flush: bool,
11117) -> bool {
11118    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11119        reason,
11120        deadline_ms,
11121    }) {
11122        Ok(body) => body,
11123        Err(err) => {
11124            warn!(
11125                module_id,
11126                error = %err,
11127                "failed to encode module draining command"
11128            );
11129            return false;
11130        }
11131    };
11132    let frame = match Frame::build_with_version(
11133        target.negotiated_ver,
11134        FrameType::Push,
11135        control_flags(),
11136        0,
11137        0,
11138        0,
11139        body,
11140    ) {
11141        Ok(frame) => frame,
11142        Err(err) => {
11143            warn!(
11144                module_id,
11145                error = %err,
11146                "failed to build module draining command frame"
11147            );
11148            return false;
11149        }
11150    };
11151    #[cfg(unix)]
11152    if _wait_for_flush {
11153        return target.sink.send_flushed(frame).await.is_ok();
11154    }
11155    if let Err(err) = target.sink.try_send(frame) {
11156        warn!(
11157            module_id,
11158            target_connection_id = target.endpoint.connection_id.get(),
11159            error = %err,
11160            "module draining command was not delivered to peer"
11161        );
11162        return false;
11163    }
11164    true
11165}
11166
11167/// The channel-0 GOODBYE that tells a module its stop is planned.
11168fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11169    match Frame::build_with_version(
11170        negotiated_ver,
11171        FrameType::Goodbye,
11172        control_flags(),
11173        0,
11174        0,
11175        0,
11176        Vec::new(),
11177    ) {
11178        Ok(frame) => Some(frame),
11179        Err(err) => {
11180            warn!(
11181                module_id,
11182                error = %err,
11183                "failed to build module GOODBYE frame"
11184            );
11185            None
11186        }
11187    }
11188}
11189
11190/// Send every registered module connection its module GOODBYE at daemon
11191/// shutdown, then request that connection's close.
11192///
11193/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
11194/// before EOF, so the GOODBYE must reach the socket before the close. A close
11195/// request does not wait for the connection's queued frames: its writer gets a
11196/// bounded grace after the close, is aborted if it overruns it, and the daemon
11197/// process may exit before that grace ends. So with `wait_for_flush`, each
11198/// connection is closed only after its writer has acknowledged writing the
11199/// GOODBYE, or once a short shared budget runs out, so one module that is not
11200/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
11201/// are only queued, for a shutdown the operator has told to stop waiting.
11202/// A connection that is already gone is skipped.
11203#[cfg(unix)]
11204async fn send_module_goodbyes_for_daemon_shutdown(
11205    forwarding: &Arc<ForwardingTable>,
11206    reason: &CloseReason,
11207    wait_for_flush: bool,
11208) {
11209    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11210    let targets = match forwarding.module_connections() {
11211        Ok(targets) => targets,
11212        Err(err) => {
11213            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11214            return;
11215        }
11216    };
11217    let deadline = Instant::now() + GOODBYE_BUDGET;
11218    let mut sends = tokio::task::JoinSet::new();
11219    for target in targets {
11220        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11221            continue;
11222        };
11223        if !wait_for_flush {
11224            if let Err(err) = target.sink.try_send(frame) {
11225                debug!(
11226                    module_id = %target.module_id,
11227                    error = %err,
11228                    "shutdown module GOODBYE was not queued"
11229                );
11230            }
11231            continue;
11232        }
11233        let forwarding = Arc::clone(forwarding);
11234        let reason = reason.clone();
11235        sends.spawn(async move {
11236            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11237                Ok(Ok(())) => {}
11238                Ok(Err(err)) => debug!(
11239                    module_id = %target.module_id,
11240                    error = %err,
11241                    "module connection closed before its shutdown GOODBYE was written"
11242                ),
11243                Err(_) => warn!(
11244                    module_id = %target.module_id,
11245                    budget = ?GOODBYE_BUDGET,
11246                    "shutdown module GOODBYE was not written within its budget; closing anyway"
11247                ),
11248            }
11249            forwarding.request_connection_close(target.endpoint.connection_id, reason);
11250        });
11251    }
11252    // Every task ends by the shared deadline, so this wait is bounded too.
11253    while sends.join_next().await.is_some() {}
11254}
11255
11256fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11257    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11258        return;
11259    };
11260    if let Err(err) = target.sink.try_send(frame) {
11261        warn!(
11262            module_id,
11263            target_connection_id = target.endpoint.connection_id.get(),
11264            error = %err,
11265            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11266        );
11267        forwarding.request_connection_close(
11268            target.endpoint.connection_id,
11269            CloseReason::new(
11270                "module_goodbye_delivery_failed",
11271                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11272            ),
11273        );
11274    }
11275}
11276
11277#[derive(Clone, Copy)]
11278struct ForwardingDrainContext<'a> {
11279    spec: &'a ModuleSpec,
11280    runtime: &'a SupervisorRuntimeConfig,
11281    registry: &'a Registry,
11282    scope: DrainScope,
11283}
11284
11285/// Which process a forwarding drain addresses.
11286#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11287enum DrainScope {
11288    /// Whatever endpoint is active for the module id: every plain stop,
11289    /// restart and reload. Also moves the module's state to `Draining`.
11290    Active,
11291    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
11292    /// module id would resolve to the promoted candidate and leave neither
11293    /// process routable. The module's state is left alone, since the promoted
11294    /// candidate is what it describes and that process is running.
11295    Endpoint(crate::ModuleEndpointId),
11296}
11297
11298/// Whether a child being drained has already been asked to stop by the time
11299/// its drain wait starts.
11300///
11301/// The drain wait is the same budget whatever this says. What it decides is
11302/// whether the supervisor must ask by signal before that wait begins: a child
11303/// that nobody asked will sit out the whole budget and then be SIGKILLed,
11304/// healthy or not.
11305#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11306enum StopNotice {
11307    /// The module was sent `module.draining` and a module GOODBYE over its own
11308    /// registered connection, and stops itself.
11309    SentOverConnection,
11310    /// The forwarding drain found no registered connection for the module: a
11311    /// subc child spawned moments ago that has not sent HELLO yet, or a
11312    /// `protocol: "none"` child, which never registers.
11313    NoConnection,
11314    /// This path sends nothing over the module's connection: the supervisor has
11315    /// no forwarding table, or the caller stops the child without a forwarding
11316    /// drain.
11317    NotSent,
11318}
11319
11320async fn begin_forwarding_drain(
11321    spec: &ModuleSpec,
11322    runtime: &SupervisorRuntimeConfig,
11323    registry: &Registry,
11324    snapshot: &SharedSnapshot,
11325    enabled: Option<bool>,
11326    reason: RouteCloseReason,
11327) -> Result<StopNotice, SuperviseError> {
11328    let Some(forwarding) = runtime.forwarding.as_ref() else {
11329        return Err(SuperviseError::ReloadUnavailable {
11330            module_id: spec.module_id.clone(),
11331            reason: "supervisor was not configured with a forwarding table".to_string(),
11332        });
11333    };
11334
11335    begin_forwarding_drain_with(
11336        forwarding,
11337        ForwardingDrainContext {
11338            spec,
11339            runtime,
11340            registry,
11341            scope: DrainScope::Active,
11342        },
11343        snapshot,
11344        enabled,
11345        reason,
11346        runtime.drain_timeout,
11347    )
11348    .await
11349}
11350
11351async fn begin_forwarding_drain_if_configured(
11352    spec: &ModuleSpec,
11353    runtime: &SupervisorRuntimeConfig,
11354    registry: &Registry,
11355    snapshot: &SharedSnapshot,
11356    enabled: Option<bool>,
11357    reason: RouteCloseReason,
11358) -> Result<StopNotice, SuperviseError> {
11359    begin_forwarding_drain_with_timeout(
11360        spec,
11361        runtime,
11362        registry,
11363        snapshot,
11364        enabled,
11365        reason,
11366        runtime.drain_timeout,
11367    )
11368    .await
11369}
11370
11371/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
11372/// budget, for paths where the operator overrides the module's configured one
11373/// (`supervisor.restart{drain_timeout_ms}`).
11374async fn begin_forwarding_drain_with_timeout(
11375    spec: &ModuleSpec,
11376    runtime: &SupervisorRuntimeConfig,
11377    registry: &Registry,
11378    snapshot: &SharedSnapshot,
11379    enabled: Option<bool>,
11380    reason: RouteCloseReason,
11381    drain_timeout: Duration,
11382) -> Result<StopNotice, SuperviseError> {
11383    let Some(forwarding) = runtime.forwarding.as_ref() else {
11384        return Ok(StopNotice::NotSent);
11385    };
11386
11387    begin_forwarding_drain_with(
11388        forwarding,
11389        ForwardingDrainContext {
11390            spec,
11391            runtime,
11392            registry,
11393            scope: DrainScope::Active,
11394        },
11395        snapshot,
11396        enabled,
11397        reason,
11398        drain_timeout,
11399    )
11400    .await
11401}
11402
11403async fn begin_forwarding_drain_with(
11404    forwarding: &ForwardingTable,
11405    context: ForwardingDrainContext<'_>,
11406    snapshot: &SharedSnapshot,
11407    enabled: Option<bool>,
11408    reason: RouteCloseReason,
11409    drain_timeout: Duration,
11410) -> Result<StopNotice, SuperviseError> {
11411    let ForwardingDrainContext {
11412        spec,
11413        runtime,
11414        registry,
11415        scope,
11416    } = context;
11417    debug_assert_ne!(reason, RouteCloseReason::Crash);
11418    let terminal = matches!(reason, RouteCloseReason::Disable);
11419    let drain_started_at = Instant::now();
11420    let drain_deadline = drain_started_at + drain_timeout;
11421    let deadline_ms =
11422        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11423    let busy_gauges = match scope {
11424        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11425        DrainScope::Endpoint(endpoint) => {
11426            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11427        }
11428    };
11429
11430    // Admission gate first: route.open/commit and route REQUEST admission are closed
11431    // before the first quiescence check, so the outstanding count can only fall.
11432    let gate_started = Instant::now();
11433    let drain_target = match scope {
11434        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11435        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11436    }
11437    .map_err(SuperviseError::Forwarding)?;
11438    // The instant admission closed, and how long taking the forwarding write
11439    // lock to close it took. The timeout line reports only the quiescence
11440    // wait, so without this a drain that started late looked like one that
11441    // started on time.
11442    info!(
11443        module_id = %spec.module_id,
11444        ?reason,
11445        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11446        connected = drain_target.is_some(),
11447        "module drain began; route admission closed"
11448    );
11449    if scope == DrainScope::Active {
11450        update_snapshot(snapshot, Some(&spec.module_id), |state| {
11451            state.state = ModuleState::Draining;
11452            state.draining_to_replace =
11453                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11454            if let Some(enabled) = enabled {
11455                state.enabled = enabled;
11456            }
11457        })?;
11458    }
11459
11460    let Some(target) = drain_target.as_ref() else {
11461        // Nothing was sent: the module has no registered connection to carry
11462        // `module.draining` or a GOODBYE. The caller must not assume the child
11463        // was asked to stop.
11464        return Ok(StopNotice::NoConnection);
11465    };
11466    {
11467        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11468        #[cfg(feature = "test-support")]
11469        forwarding.wait_before_route_closing_for_test().await;
11470        let routes = forwarding
11471            .endpoint_routes(target.endpoint)
11472            .map_err(SuperviseError::Forwarding)?;
11473        let routes_notified = routes.len();
11474        crate::control::send_route_control_pushes(
11475            forwarding,
11476            routes.clone(),
11477            ClientControlPush::RouteClosing {
11478                module_id: spec.module_id.clone(),
11479                channels: Vec::new(),
11480                reason,
11481            },
11482        );
11483        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11484
11485        // `route.closing` was just sent above: from here on every return path,
11486        // including an early one, MUST send `route.closed` before propagating
11487        // anything else. A client holds `closing` as a promise that a verdict is
11488        // coming; leaving early without `closed` strands it waiting forever, since
11489        // `closing` carries no timeout of its own.
11490        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11491        let wait_result = wait_for_forwarding_quiescence(
11492            forwarding,
11493            &spec.module_id,
11494            runtime,
11495            target.endpoint,
11496            drain_deadline,
11497            &busy_gauges,
11498            scope,
11499            &mut gauge_trace,
11500        )
11501        .await;
11502        let drained = drained_after_quiescence_wait(&wait_result);
11503        if let Err(err) = &wait_result {
11504            error!(
11505                module_id = %spec.module_id,
11506                ?reason,
11507                error = %err,
11508                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11509            );
11510        } else if !drained {
11511            // Read before teardown releases the routes.
11512            let holdouts = forwarding
11513                .endpoint_drain_holdouts(target.endpoint)
11514                .unwrap_or_default();
11515            warn_drain_timed_out(
11516                &spec.module_id,
11517                drain_timeout,
11518                reason,
11519                &holdouts,
11520                &gauge_trace,
11521            );
11522        }
11523        crate::control::send_route_control_pushes(
11524            forwarding,
11525            routes,
11526            ClientControlPush::RouteClosed {
11527                module_id: spec.module_id.clone(),
11528                channels: Vec::new(),
11529                reason,
11530                drained,
11531                abandoned: target.abandoned_bindings.len() as u32,
11532                excluded_subscriptions: target.excluded_subscriptions,
11533                terminal: Some(terminal),
11534            },
11535        );
11536        wait_result?;
11537
11538        // `route.closed` has now been sent unconditionally above. From here the
11539        // remaining steps are cleanup (route + module GOODBYE) rather than a
11540        // promise the client is waiting on, but a lock-poisoned
11541        // `release_module_endpoint_routes` would otherwise skip the module
11542        // GOODBYE silently too -- send it before propagating the error.
11543        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11544            Ok(routes) => routes,
11545            Err(err) => {
11546                warn!(
11547                    module_id = %spec.module_id,
11548                    ?reason,
11549                    error = %err,
11550                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11551                );
11552                send_module_goodbye(&spec.module_id, forwarding, target);
11553                return Err(SuperviseError::Forwarding(err));
11554            }
11555        };
11556        let route_goodbye_count = released_routes.len();
11557        send_route_goodbyes(forwarding, released_routes);
11558        send_module_goodbye(&spec.module_id, forwarding, target);
11559
11560        // The drain's happy path was previously silent: every emission above is
11561        // best-effort with only its failure arm logged, so "were consumers told"
11562        // was unprovable from the daemon log (surfaced by a 30-minute consumer
11563        // hang where the open question was exactly whether teardown notice went
11564        // out). One summary line makes that class decidable in one grep.
11565        info!(
11566            module_id = %spec.module_id,
11567            ?reason,
11568            routes_notified,
11569            route_goodbyes = route_goodbye_count,
11570            abandoned_reservations = target.abandoned_bindings.len(),
11571            excluded_subscriptions = target.excluded_subscriptions,
11572            drained,
11573            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11574        );
11575    }
11576
11577    Ok(StopNotice::SentOverConnection)
11578}
11579
11580/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
11581/// the only slot a plain (non-swap) spawn can register into.
11582async fn wait_for_registration_after_reload(
11583    registry: &Registry,
11584    module_id: &str,
11585    snapshot: &SharedSnapshot,
11586    child: &mut SupervisedChild,
11587    wait: Duration,
11588) -> Result<RegistrationWaitOutcome, SuperviseError> {
11589    wait_for_slot_registration(
11590        registry,
11591        crate::registry::RegistrationSlot::Active(module_id),
11592        module_id,
11593        snapshot,
11594        child,
11595        wait,
11596    )
11597    .await
11598}
11599
11600/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
11601///
11602/// Keyed on the slot rather than the bare module id because during a swap the
11603/// id's active slot is already held by the incumbent: an id-keyed wait would
11604/// report the incumbent's registration as the candidate's and a candidate that
11605/// never registers would look registered. A swap candidate waits on
11606/// `crate::registry::RegistrationSlot::Candidate`.
11607async fn wait_for_slot_registration(
11608    registry: &Registry,
11609    slot: crate::registry::RegistrationSlot<'_>,
11610    module_id: &str,
11611    snapshot: &SharedSnapshot,
11612    child: &mut SupervisedChild,
11613    wait: Duration,
11614) -> Result<RegistrationWaitOutcome, SuperviseError> {
11615    let deadline = Instant::now() + wait;
11616    loop {
11617        if registry
11618            .registration(slot)
11619            .map_err(SuperviseError::Registry)?
11620            .is_some()
11621        {
11622            return Ok(RegistrationWaitOutcome::Registered);
11623        }
11624
11625        let now = Instant::now();
11626        if now >= deadline {
11627            return Ok(RegistrationWaitOutcome::TimedOut);
11628        }
11629        let remaining = deadline.saturating_duration_since(now);
11630        let poll = remaining.min(REGISTRY_RELEASE_POLL);
11631
11632        tokio::select! {
11633            wait_result = child.wait() => {
11634                let status = wait_result.map_err(|source| SuperviseError::Wait {
11635                    module_id: module_id.to_string(),
11636                    source,
11637                })?;
11638                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11639                    snapshot,
11640                    child,
11641                    &status,
11642                )));
11643            }
11644            _ = sleep(poll) => {}
11645        }
11646    }
11647}
11648
11649fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11650    // A process stopped for missing its first HELLO did not provide service,
11651    // even if its TERM handler exits zero. Reload also uses this classification
11652    // when a replacement exits before registering.
11653    if exit_report.kind != ExitKind::DeliberateSeverance {
11654        exit_report.kind = ExitKind::Crash;
11655    }
11656    exit_report
11657}
11658
11659async fn handle_reload_child_registration_failure(
11660    spec: &ModuleSpec,
11661    runtime: &SupervisorRuntimeConfig,
11662    registry: &Registry,
11663    process_liveness: &SupervisorProcessLiveness,
11664    snapshot: &SharedSnapshot,
11665    _child: &mut Option<SupervisedChild>,
11666    failure: ReloadRegistrationFailure,
11667) -> Result<(), SuperviseError> {
11668    let ReloadRegistrationFailure {
11669        exit_report,
11670        reason,
11671    } = failure;
11672    match on_child_exit(
11673        spec,
11674        runtime.restart_policy,
11675        registry,
11676        snapshot,
11677        &runtime.terminal_ring,
11678        &runtime.spawn_events,
11679        &runtime.child_roster,
11680        exit_report,
11681        runtime.recovery_is_daemon,
11682    )
11683    .await
11684    {
11685        NextAction::Stop {
11686            registration_released,
11687        } => {
11688            if registration_released {
11689                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11690            }
11691        }
11692        NextAction::Restart { schedule } => {
11693            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11694                schedule.delay
11695            });
11696            if let Some(schedule) = schedule {
11697                log_crash_respawn(&spec.module_id, schedule);
11698            }
11699            schedule_respawn(
11700                runtime,
11701                snapshot,
11702                &spec.module_id,
11703                delay,
11704                RespawnKind::Spawn,
11705                true,
11706            )?;
11707        }
11708    }
11709    Err(SuperviseError::ReloadFailed {
11710        module_id: spec.module_id.clone(),
11711        reason,
11712    })
11713}
11714
11715async fn handle_reload_spawn_failure(
11716    spec: &ModuleSpec,
11717    runtime: &SupervisorRuntimeConfig,
11718    process_liveness: &SupervisorProcessLiveness,
11719    snapshot: &SharedSnapshot,
11720    _child: &mut Option<SupervisedChild>,
11721    reason: String,
11722) -> Result<(), SuperviseError> {
11723    let now = Instant::now();
11724    let mut schedule = None;
11725    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11726        clear_current_process_facts(state);
11727        if state.enabled {
11728            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11729            state.state = if schedule.is_some() {
11730                ModuleState::Restarting
11731            } else {
11732                if runtime
11733                    .failure_context(runtime.recovery_is_daemon)
11734                    .may_retry()
11735                {
11736                    state.failed_retry.schedule(&spec.module_id);
11737                }
11738                ModuleState::Failed
11739            };
11740        } else {
11741            state.state = ModuleState::Disabled;
11742        }
11743    })?;
11744    if let Some(schedule) = schedule {
11745        schedule_respawn(
11746            runtime,
11747            snapshot,
11748            &spec.module_id,
11749            schedule.delay,
11750            RespawnKind::Spawn,
11751            true,
11752        )?;
11753    } else {
11754        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11755    }
11756    Err(SuperviseError::ReloadFailed {
11757        module_id: spec.module_id.clone(),
11758        reason,
11759    })
11760}
11761
11762fn control_flags() -> Flags {
11763    Flags::new(false, Priority::Passive, false)
11764}
11765
11766#[allow(clippy::too_many_arguments)]
11767async fn drain_optional_child(
11768    module_id: &str,
11769    protocol: ModuleProtocol,
11770    stop_notice: StopNotice,
11771    registry: &Registry,
11772    forwarding: Option<&ForwardingTable>,
11773    snapshot: &SharedSnapshot,
11774    terminal_ring: &Arc<Mutex<TerminalRing>>,
11775    spawn_events: &SpawnEventFeed,
11776    child: &mut Option<SupervisedChild>,
11777    drain_timeout: Duration,
11778    final_state: ModuleState,
11779    enabled: Option<bool>,
11780    failure: &FailureContext,
11781) -> Result<(), SuperviseError> {
11782    if let Some(child) = child.take() {
11783        drain_child_to_state(
11784            module_id,
11785            protocol,
11786            stop_notice,
11787            registry,
11788            forwarding,
11789            snapshot,
11790            terminal_ring,
11791            spawn_events,
11792            child,
11793            drain_timeout,
11794            final_state,
11795            enabled,
11796            failure,
11797        )
11798        .await
11799    } else {
11800        update_snapshot(snapshot, Some(module_id), |state| {
11801            state.state = final_state;
11802            if final_state == ModuleState::Failed {
11803                state.failed_retry.failed_at = Some(Instant::now());
11804            }
11805            if let Some(enabled) = enabled {
11806                state.enabled = enabled;
11807            }
11808            clear_current_process_facts(state);
11809        })?;
11810        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11811    }
11812}
11813
11814#[allow(clippy::too_many_arguments)]
11815async fn drain_child_to_state(
11816    module_id: &str,
11817    _protocol: ModuleProtocol,
11818    stop_notice: StopNotice,
11819    registry: &Registry,
11820    forwarding: Option<&ForwardingTable>,
11821    snapshot: &SharedSnapshot,
11822    terminal_ring: &Arc<Mutex<TerminalRing>>,
11823    spawn_events: &SpawnEventFeed,
11824    mut child: SupervisedChild,
11825    drain_timeout: Duration,
11826    final_state: ModuleState,
11827    enabled: Option<bool>,
11828    failure: &FailureContext,
11829) -> Result<(), SuperviseError> {
11830    let protocol = child.protocol;
11831    update_snapshot(snapshot, Some(module_id), |state| {
11832        state.state = ModuleState::Draining;
11833        state.draining_to_replace = final_state == ModuleState::Restarting;
11834        if let Some(enabled) = enabled {
11835            state.enabled = enabled;
11836        }
11837    })?;
11838
11839    // The wait below is the same budget in every case; what differs is
11840    // whether anything has ASKED the child to stop before it starts. Only a
11841    // forwarding drain that reached the module's registered connection has
11842    // (`module.draining`, then a module GOODBYE). Every other child was told
11843    // nothing: a `protocol: "none"` module, which never registers; a subc
11844    // module spawned moments ago that has not sent HELLO yet; or a stop that
11845    // runs no forwarding drain. Without a signal the budget is only a delay
11846    // in front of SIGKILL -- and the not-yet-registered child is the worst
11847    // case, because it registers into a module that is already draining,
11848    // is never told, and is killed while healthy.
11849    if stop_notice != StopNotice::SentOverConnection {
11850        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11851            info!(
11852                module_id,
11853                pid = child.pid,
11854                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11855                "module has no connection yet; requesting stop by signal"
11856            );
11857        }
11858        request_graceful_stop(module_id, &child);
11859    }
11860
11861    let exit_report = match timeout(drain_timeout, child.wait()).await {
11862        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11863        Ok(Err(source)) => {
11864            fail_after_wait_error(
11865                module_id,
11866                snapshot,
11867                terminal_ring,
11868                spawn_events,
11869                format!("failed to wait during drain: {source}"),
11870                failure,
11871            );
11872            return Err(SuperviseError::Wait {
11873                module_id: module_id.to_string(),
11874                source,
11875            });
11876        }
11877        Err(_) => {
11878            // Mirror the sibling arm above: state is already `Draining`, and an
11879            // error propagated from here would strand it there -- a state
11880            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11881            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11882            // `Failed` before `?` keeps the module operator-visible and
11883            // revivable. Trigger is an ESRCH race (process exits between the
11884            // drain timeout firing and the kill) or a post-kill wait failure
11885            // (issue #34).
11886            //
11887            // Logged because the kill is otherwise visible only as signal 9 in
11888            // the terminal ring, and the budget it follows can be long enough
11889            // that consumers see a stretch of refusals with no stated cause.
11890            warn!(
11891                module_id,
11892                pid = child.pid,
11893                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11894                reason = ?final_state,
11895                ?stop_notice,
11896                "drain budget expired before the module exited; killing it"
11897            );
11898            let kill =
11899                if final_state == ModuleState::Restarting && child.child.container().is_some() {
11900                    child.kill_leader()
11901                } else {
11902                    child.start_kill()
11903                };
11904            kill.map_err(|source| {
11905                fail_after_wait_error(
11906                    module_id,
11907                    snapshot,
11908                    terminal_ring,
11909                    spawn_events,
11910                    format!("failed to kill during drain: {source}"),
11911                    failure,
11912                );
11913                SuperviseError::Kill {
11914                    module_id: module_id.to_string(),
11915                    source,
11916                }
11917            })?;
11918            let status = child.wait().await.map_err(|source| {
11919                fail_after_wait_error(
11920                    module_id,
11921                    snapshot,
11922                    terminal_ring,
11923                    spawn_events,
11924                    format!("failed to wait after drain kill: {source}"),
11925                    failure,
11926                );
11927                SuperviseError::Wait {
11928                    module_id: module_id.to_string(),
11929                    source,
11930                }
11931            })?;
11932            classify_reaped_child_exit(snapshot, &child, &status)
11933        }
11934    };
11935
11936    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11937    publish_after_terminal_record(
11938        module_id,
11939        || {
11940            record_terminal_with_detail(
11941                module_id,
11942                terminal_ring,
11943                spawn_events,
11944                &exit_report,
11945                terminal_disposition(final_state),
11946                detail,
11947            )
11948        },
11949        || {
11950            update_snapshot(snapshot, Some(module_id), |state| {
11951                state.state = final_state;
11952                if final_state == ModuleState::Failed {
11953                    state.failed_retry.failed_at = Some(Instant::now());
11954                }
11955                if let Some(enabled) = enabled {
11956                    state.enabled = enabled;
11957                }
11958                clear_current_process_facts(state);
11959                state.last_exit = Some(exit_report.clone());
11960                if exit_report.kind == ExitKind::DeliberateSeverance {
11961                    state.lifetime_restarts += 1;
11962                }
11963            })
11964        },
11965    )?;
11966    child.drain_stderr(module_id).await;
11967
11968    let released =
11969        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11970    child
11971        .finish_exit(
11972            released.is_ok()
11973                && final_state == ModuleState::Restarting
11974                && child.spawn_failure.is_none(),
11975        )
11976        .await
11977        .map_err(|source| SuperviseError::Wait {
11978            module_id: module_id.to_owned(),
11979            source,
11980        })?;
11981    released
11982}
11983
11984/// Ask a child that nothing else has asked to stop, by signal.
11985///
11986/// A registered subc module is asked over its own connection: the drain sends
11987/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11988/// module GOODBYE, and the module stops itself. A module that speaks no subc
11989/// wire receives none of that, and neither does a subc module that has not
11990/// registered yet, so for them the drain budget would be pure delay in front of
11991/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11992/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11993/// into a recovery on the next start.
11994///
11995/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
11996/// rule rather than an optimisation: that module's graceful stop is already
11997/// running by the time its child is drained, and a signal would race it.
11998///
11999/// Best-effort by construction. A child that has already exited is the ordinary
12000/// case rather than an error (the kill lands on a reaped or exiting pid), so a
12001/// failure is logged at debug and the wait-then-kill below still decides the
12002/// outcome.
12003#[cfg(unix)]
12004fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12005    let Some(pid) = child
12006        .id()
12007        .and_then(|pid| i32::try_from(pid).ok())
12008        .and_then(rustix::process::Pid::from_raw)
12009    else {
12010        debug!(
12011            module_id,
12012            "no pid to signal for teardown; falling through to the drain wait"
12013        );
12014        return;
12015    };
12016    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12017        Ok(()) => debug!(
12018            module_id,
12019            "sent SIGTERM to a module nothing else asked to stop"
12020        ),
12021        Err(err) => debug!(
12022            module_id,
12023            error = %err,
12024            "SIGTERM to module failed; the drain wait and kill still apply"
12025        ),
12026    }
12027}
12028
12029/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
12030/// Windows does offer need cooperation this supervisor cannot assume: a console
12031/// control event requires sharing a console with the child, and `WM_CLOSE`
12032/// requires the child to pump a message loop. A supervised server process does
12033/// neither, so there is nothing to send and teardown is the wait followed by the
12034/// kill. Emulating a signal here would mean inventing a stop protocol, which is
12035/// the thing `protocol: "none"` exists to avoid.
12036#[cfg(not(unix))]
12037fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12038    debug!(
12039        module_id,
12040        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12041    );
12042}
12043
12044fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12045    match final_state {
12046        ModuleState::Stopped => TerminalDisposition::Stopped,
12047        ModuleState::Disabled => TerminalDisposition::Disabled,
12048        ModuleState::Restarting => TerminalDisposition::Restarting,
12049        ModuleState::Failed => TerminalDisposition::Failed,
12050        ModuleState::Starting
12051        | ModuleState::Running
12052        | ModuleState::Unresponsive
12053        | ModuleState::Draining => {
12054            unreachable!("terminal exits only finish in terminal or restarting states")
12055        }
12056    }
12057}
12058
12059/// Release a reaped child's registration before allowing another spawn.
12060///
12061/// EOF is not a process-lifetime signal: an inherited socket can stay open
12062/// indefinitely, and serial frame dispatch can be waiting on egress instead of
12063/// reading EOF. After the normal release grace, request connection close (which
12064/// cancels both reads and dispatch), then allow one more release grace for the
12065/// connection guard's forwarding cleanup. Never evict a different connection.
12066async fn release_dead_registration(
12067    registry: &Registry,
12068    forwarding: Option<&ForwardingTable>,
12069    snapshot: &SharedSnapshot,
12070    module_id: &str,
12071    failure: &FailureContext,
12072) -> Result<(), SuperviseError> {
12073    let result = async {
12074        let registration = registry
12075            .get_module(module_id)
12076            .map_err(SuperviseError::Registry)?;
12077        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12078            Ok(()) => return Ok(()),
12079            Err(SuperviseError::RegistrationStillActive { .. }) => {}
12080            Err(err) => return Err(err),
12081        }
12082        let pid = lock_snapshot(snapshot)?.reaped_pid;
12083        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12084            warn!(
12085                module_id,
12086                pid,
12087                connection_id = registration.connection_id.get(),
12088                "reaped module registration outlived release grace; closing dead connection"
12089            );
12090            forwarding.request_connection_close(
12091                registration.connection_id,
12092                CloseReason::new(
12093                    "supervised_process_reaped",
12094                    format!("module '{module_id}' pid {pid} exited"),
12095                ),
12096            );
12097            wait_for_slot_registration_release(
12098                registry,
12099                crate::registry::RegistrationSlot::Connection(registration.connection_id),
12100                REGISTRY_RELEASE_TIMEOUT,
12101            )
12102            .await?;
12103        }
12104        wait_for_registration_release(registry, module_id, Duration::ZERO).await
12105    }
12106    .await;
12107    if let Err(err) = &result {
12108        fail_snapshot(snapshot, Some(module_id), None, failure);
12109        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12110    }
12111    result
12112}
12113
12114/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
12115/// plain stop or restart waits for before it spawns a replacement.
12116async fn wait_for_registration_release(
12117    registry: &Registry,
12118    module_id: &str,
12119    wait: Duration,
12120) -> Result<(), SuperviseError> {
12121    wait_for_slot_registration_release(
12122        registry,
12123        crate::registry::RegistrationSlot::Active(module_id),
12124        wait,
12125    )
12126    .await
12127}
12128
12129/// Wait for the registration in `slot` to go away.
12130///
12131/// Keyed on the slot rather than the bare module id because a successful swap
12132/// never empties the id's active slot (the promoted candidate is in it), so an
12133/// id-keyed wait for the incumbent's release would always time out. Draining a
12134/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
12135/// incumbent's connection instead.
12136async fn wait_for_slot_registration_release(
12137    registry: &Registry,
12138    slot: crate::registry::RegistrationSlot<'_>,
12139    wait: Duration,
12140) -> Result<(), SuperviseError> {
12141    let deadline = Instant::now() + wait;
12142    let mut release_events = registration_release_events().subscribe();
12143    let still_active = |registration: &crate::registry::ModuleRegistration| {
12144        SuperviseError::RegistrationStillActive {
12145            module_id: registration.manifest.module_id.clone(),
12146            waited: wait,
12147        }
12148    };
12149    loop {
12150        let _observed_generation = *release_events.borrow_and_update();
12151        let Some(registration) = registry
12152            .registration(slot)
12153            .map_err(SuperviseError::Registry)?
12154        else {
12155            return Ok(());
12156        };
12157
12158        let now = Instant::now();
12159        if now >= deadline {
12160            return Err(still_active(&registration));
12161        }
12162
12163        let remaining = deadline.saturating_duration_since(now);
12164        match timeout(remaining, release_events.changed()).await {
12165            Ok(Ok(())) | Ok(Err(_)) => {}
12166            Err(_) => return Err(still_active(&registration)),
12167        }
12168    }
12169}
12170
12171#[cfg(test)]
12172mod slot_registration_wait_tests {
12173    use super::*;
12174    use crate::registry::{ConnectionId, RegistrationSlot};
12175    use subc_protocol::manifest::ModuleManifest;
12176
12177    #[tokio::test]
12178    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12179        let registry = Arc::new(Registry::default());
12180        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
12181        let runtime = supervisor.runtime_config();
12182        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12183        let spec = ModuleSpec {
12184            module_id: "enable-stale-registration".to_string(),
12185            program: PathBuf::from("/missing/enable-retry-test"),
12186            args: Vec::new(),
12187            env: Vec::new(),
12188            reserved: false,
12189            reserved_prefixes: Vec::new(),
12190            protocol: ModuleProtocol::Subc,
12191            overlap: Default::default(),
12192            carry_containment: false,
12193        };
12194        let connection = ConnectionId::new(90);
12195        registry
12196            .register_with_control_ops(
12197                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12198                1,
12199                connection,
12200                Vec::new(),
12201            )
12202            .unwrap();
12203        let mut child = None;
12204        let err = set_child_enabled(
12205            &spec,
12206            &runtime,
12207            &registry,
12208            &supervisor.process_liveness,
12209            &snapshot,
12210            &mut child,
12211            true,
12212            &runtime.failure_context(false),
12213        )
12214        .await
12215        .unwrap_err();
12216        assert!(matches!(
12217            err,
12218            SuperviseError::RegistrationStillActive { .. }
12219        ));
12220        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12221        assert!(child.is_none());
12222        registry.deregister_connection(connection).unwrap();
12223        let err = set_child_enabled(
12224            &spec,
12225            &runtime,
12226            &registry,
12227            &supervisor.process_liveness,
12228            &snapshot,
12229            &mut child,
12230            true,
12231            &runtime.failure_context(false),
12232        )
12233        .await
12234        .unwrap_err();
12235        assert!(
12236            matches!(err, SuperviseError::Spawn { .. }),
12237            "second enable must attempt a spawn: {err}"
12238        );
12239        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12240    }
12241
12242    const INCUMBENT: u64 = 1;
12243    const CANDIDATE: u64 = 2;
12244
12245    fn swapped_registry() -> Arc<Registry> {
12246        let registry = Arc::new(Registry::default());
12247        let manifest = ModuleManifest::builder("m", "0.1.0").build();
12248        registry
12249            .register_with_control_ops(
12250                manifest.clone(),
12251                1,
12252                ConnectionId::new(INCUMBENT),
12253                Vec::new(),
12254            )
12255            .unwrap();
12256        registry
12257            .register_candidate_with_control_ops(
12258                manifest,
12259                1,
12260                ConnectionId::new(CANDIDATE),
12261                Vec::new(),
12262            )
12263            .unwrap();
12264        registry
12265    }
12266
12267    /// After a promotion the id's active slot is held by the new process, so an
12268    /// id-keyed wait for the incumbent's release can never succeed; the
12269    /// connection-keyed wait completes as soon as the incumbent deregisters.
12270    #[tokio::test]
12271    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12272        let registry = swapped_registry();
12273        registry.promote_candidate("m").unwrap().unwrap();
12274
12275        assert!(matches!(
12276            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
12277            Err(SuperviseError::RegistrationStillActive { .. })
12278        ));
12279
12280        // Still held while the incumbent's connection has not deregistered.
12281        assert!(matches!(
12282            wait_for_slot_registration_release(
12283                &registry,
12284                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12285                Duration::from_millis(50),
12286            )
12287            .await,
12288            Err(SuperviseError::RegistrationStillActive { .. })
12289        ));
12290
12291        let releaser = Arc::clone(&registry);
12292        let release = tokio::spawn(async move {
12293            sleep(Duration::from_millis(20)).await;
12294            releaser
12295                .deregister_connection(ConnectionId::new(INCUMBENT))
12296                .unwrap();
12297            notify_registration_release();
12298        });
12299        wait_for_slot_registration_release(
12300            &registry,
12301            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12302            Duration::from_secs(5),
12303        )
12304        .await
12305        .expect("the incumbent's own registration is released");
12306        release.await.unwrap();
12307        assert!(registry.get_module("m").unwrap().is_some());
12308    }
12309
12310    /// The candidate slot is waited on separately from the active slot: the
12311    /// incumbent's registration neither holds up nor stands in for it.
12312    #[tokio::test]
12313    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12314        let registry = swapped_registry();
12315        assert!(matches!(
12316            wait_for_slot_registration_release(
12317                &registry,
12318                RegistrationSlot::Candidate("m"),
12319                Duration::from_millis(50),
12320            )
12321            .await,
12322            Err(SuperviseError::RegistrationStillActive { .. })
12323        ));
12324        registry
12325            .deregister_connection(ConnectionId::new(CANDIDATE))
12326            .unwrap();
12327        wait_for_slot_registration_release(
12328            &registry,
12329            RegistrationSlot::Candidate("m"),
12330            Duration::from_millis(50),
12331        )
12332        .await
12333        .expect("a candidate slot with no candidate is released");
12334        assert!(registry
12335            .registration(RegistrationSlot::Active("m"))
12336            .unwrap()
12337            .is_some());
12338    }
12339}
12340
12341fn classify_exit(status: &ExitStatus) -> ExitReport {
12342    ExitReport {
12343        kind: if status.success() {
12344            ExitKind::Clean
12345        } else {
12346            ExitKind::Crash
12347        },
12348        code: status.code(),
12349        signal: exit_signal(status),
12350        at_ms: unix_ms_now(),
12351    }
12352}
12353
12354/// The terminal record for a module whose `wait()` call itself errored (e.g. the
12355/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
12356/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
12357/// disposition still must be `Failed` so the terminal ring is not silently missing
12358/// an entry, matching what `fail_snapshot` records for this same arm.
12359fn wait_error_exit_report() -> ExitReport {
12360    ExitReport {
12361        kind: ExitKind::Crash,
12362        code: None,
12363        signal: None,
12364        at_ms: unix_ms_now(),
12365    }
12366}
12367
12368#[cfg(unix)]
12369fn exit_signal(status: &ExitStatus) -> Option<i32> {
12370    use std::os::unix::process::ExitStatusExt;
12371
12372    status.signal()
12373}
12374
12375#[cfg(not(unix))]
12376fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12377    None
12378}
12379
12380/// Give an operator-touched module its full crash budget back.
12381///
12382/// Named for the counter it used to zero; it now empties the in-window ring,
12383/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
12384/// ledger of what happened survives every operator action.
12385fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12386    update_snapshot(snapshot, Some(module_id), |state| {
12387        state.clear_crash_restarts();
12388    })
12389}
12390
12391fn set_running(
12392    snapshot: &SharedSnapshot,
12393    child: &SupervisedChild,
12394    module_id: &str,
12395    spawn_events: &SpawnEventFeed,
12396) -> Result<(), SuperviseError> {
12397    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12398        module_id: Some(module_id.to_string()),
12399    })?;
12400    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12401    if std::mem::take(&mut state.coalesced_restart_pending) {
12402        let generation = state.spawn_generation;
12403        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12404    }
12405    state.drain_disposition_detail = None;
12406    state.spawn_failure = None;
12407    state.waiting_for_previous_pid = None;
12408    state.waiting_for_previous_detail = None;
12409    // Every caller of this is a plain spawn, which always uses the primary key;
12410    // a promoted swap candidate sets the flag itself after this returns.
12411    state.in_alternate_slot = false;
12412    state.configuration_updated_since_spawn = false;
12413    state.spawned_protocol = Some(child.protocol);
12414    state.state = ModuleState::Running;
12415    state.enabled = true;
12416    state.process_alive = true;
12417    state.pid = child.id();
12418    state.subtree_target = child.subtree_target.clone();
12419    // Probe samples belong to one process. Keep module-level escalation and
12420    // late-answer history, but never charge a new child for its predecessor's misses.
12421    state.health.status = SupervisorHealthStatus::Unknown;
12422    state.health.consecutive_failures = 0;
12423    state.health.detail = None;
12424    state.health.metrics = None;
12425    state.health.last_probe_ms = None;
12426    state.failed_retry.healthy_since = None;
12427    state.failed_retry.failed_at = None;
12428    state.failed_retry.running_since = Some(child.spawned_at);
12429    #[cfg(windows)]
12430    {
12431        state.pipe_consumed = child
12432            .nonce_delivery
12433            .as_ref()
12434            .map(|delivery| delivery.consumed());
12435    }
12436    #[cfg(target_os = "macos")]
12437    {
12438        state.report_ready = Some(Arc::clone(&child.report_ready));
12439    }
12440    state.spawned_at_ms = Some(child.spawned_at_ms);
12441    state.exec_outcome = Some(Arc::clone(&child.exec.0));
12442    state.spawned_from = Some(child.spawned_from.clone());
12443    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12444    state.spawned_file_identity = child.spawned_file_identity.clone();
12445    state.process_start_time = child.process_start_time;
12446    Ok(())
12447}
12448
12449fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12450    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12451    state.failed_retry.healthy_since = None;
12452    state.failed_retry.running_since = None;
12453    state.waiting_for_previous_pid = None;
12454    state.waiting_for_previous_detail = None;
12455    state.process_alive = false;
12456    state.spawned_protocol = None;
12457    state.pid = None;
12458    #[cfg(windows)]
12459    {
12460        state.pipe_consumed = None;
12461    }
12462    #[cfg(target_os = "macos")]
12463    {
12464        state.report_ready = None;
12465    }
12466    state.spawned_at_ms = None;
12467    state.exec_outcome = None;
12468    state.spawned_from = None;
12469    state.launch_spec_digest = None;
12470    state.spawned_file_identity = None;
12471    state.process_start_time = None;
12472    state.deliberate_severance = None;
12473}
12474
12475#[cfg(test)]
12476fn record_deliberate_severance(
12477    snapshot: &SharedSnapshot,
12478    identity: ProcessIdentity,
12479) -> Result<(), SuperviseError> {
12480    update_snapshot(snapshot, None, |state| {
12481        state.deliberate_severance = Some(identity);
12482    })
12483}
12484
12485fn apply_deliberate_severance_marker(
12486    snapshot: &SharedSnapshot,
12487    exited_identity: Option<ProcessIdentity>,
12488    mut exit_report: ExitReport,
12489) -> ExitReport {
12490    let marker = lock_snapshot(snapshot)
12491        .ok()
12492        .and_then(|mut state| state.deliberate_severance.take());
12493    if marker.is_some() && marker == exited_identity {
12494        exit_report.kind = ExitKind::DeliberateSeverance;
12495    }
12496    exit_report
12497}
12498
12499fn classify_reaped_child_exit(
12500    snapshot: &SharedSnapshot,
12501    child: &SupervisedChild,
12502    status: &ExitStatus,
12503) -> ExitReport {
12504    let _ = update_snapshot(snapshot, None, |state| {
12505        state.reaped_pid = Some(child.pid);
12506        state.spawn_failure = child.spawn_failure.clone();
12507    });
12508    if child.spawn_failure.as_deref() == Some("registration_timeout") {
12509        registration_failure_exit_report(classify_exit(status))
12510    } else {
12511        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12512    }
12513}
12514
12515fn fail_snapshot(
12516    snapshot: &SharedSnapshot,
12517    module_id: Option<&str>,
12518    last_exit: Option<ExitReport>,
12519    failure: &FailureContext,
12520) {
12521    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12522        state.state = ModuleState::Failed;
12523        clear_current_process_facts(state);
12524        if failure.may_retry() && state.enabled {
12525            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12526        } else if failure.shutdown.is_set() {
12527            state.failed_retry.cancel();
12528        }
12529        if let Some(last_exit) = last_exit {
12530            state.last_exit = Some(last_exit);
12531        }
12532    }) {
12533        error!(error = %err, "failed to mark supervisor state failed");
12534    }
12535}
12536
12537fn update_snapshot(
12538    snapshot: &SharedSnapshot,
12539    module_id: Option<&str>,
12540    update: impl FnOnce(&mut SupervisorSnapshot),
12541) -> Result<(), SuperviseError> {
12542    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12543        module_id: module_id.map(ToOwned::to_owned),
12544    })?;
12545    update(&mut state);
12546    Ok(())
12547}
12548
12549const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12550
12551fn lock_snapshot_for_control<'a>(
12552    snapshot: &'a SharedSnapshot,
12553    module_id: &str,
12554    caller: &'static str,
12555) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12556    let started_at = Instant::now();
12557    let guard = lock_snapshot(snapshot)?;
12558    let waited = started_at.elapsed();
12559    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12560        warn!(
12561            module_id = %module_id,
12562            waited_ms = waited.as_millis() as u64,
12563            caller = %caller,
12564            "slow snapshot lock"
12565        );
12566    }
12567    Ok(guard)
12568}
12569
12570fn lock_snapshot(
12571    snapshot: &SharedSnapshot,
12572) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12573    snapshot
12574        .lock()
12575        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12576}
12577
12578#[cfg(test)]
12579mod terminal_history_tests {
12580    use std::{
12581        path::PathBuf,
12582        sync::Arc,
12583        time::{Duration, Instant},
12584    };
12585
12586    use tokio::{sync::mpsc, time::sleep};
12587
12588    use super::{
12589        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12590        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12591        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12592        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12593        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12594        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12595        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12596        SupervisorHealthStatus, SupervisorSnapshot,
12597    };
12598    // The supervisor's clock, distinct from the `std::time::Instant` these tests
12599    // use for their own wall-clock deadlines: crash-restart instants must be on
12600    // the same clock the production code stamps them with, which is tokio's (and
12601    // is what `start_paused` tests can move).
12602    use super::Instant as ClockInstant;
12603    use crate::{
12604        registry::Registry,
12605        terminal_ring::{TerminalRing, TerminalRingConfig},
12606    };
12607    use std::sync::Mutex;
12608    use subc_control::TerminalDisposition;
12609
12610    /// See the twin in `control.rs` for why this derives the path from
12611    /// `current_exe()` and why the existence check is here: `--lib` alone does
12612    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
12613    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
12614    pub(super) fn fake_aft_stub_path() -> PathBuf {
12615        let mut path = std::env::current_exe().expect("current_exe available in tests");
12616        path.pop();
12617        path.pop();
12618        path.push(if cfg!(windows) {
12619            "fake-aft-stub.exe"
12620        } else {
12621            "fake-aft-stub"
12622        });
12623        assert!(
12624            path.exists(),
12625            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12626             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12627            path.display()
12628        );
12629        path
12630    }
12631
12632    #[test]
12633    fn reserved_never_spawned_refuses_every_hello() {
12634        // The canary hole: a reserved id whose module has never spawned had NO
12635        // gate entry and admitted anyone -- the reservation protected the nonce
12636        // holder, not the NAME. Now the entry is present with no legitimate
12637        // holder and refuses all comers.
12638        let supervisor = SupervisorHandle::default();
12639        supervisor.apply_identity_configuration(&ModuleSpec {
12640            module_id: "never-spawned".to_string(),
12641            program: PathBuf::from("/usr/bin/false"),
12642            args: Vec::new(),
12643            env: Vec::new(),
12644            reserved: true,
12645            reserved_prefixes: Vec::new(),
12646            protocol: ModuleProtocol::Subc,
12647            overlap: Default::default(),
12648            carry_containment: false,
12649        });
12650        assert!(
12651            supervisor
12652                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12653                .is_err(),
12654            "forged nonce must refuse on a reserved never-spawned id"
12655        );
12656        assert!(
12657            supervisor
12658                .reserved_hello_verification("never-spawned", None)
12659                .is_err(),
12660            "absent nonce must refuse on a reserved never-spawned id"
12661        );
12662        // And a real spawn nonce minted later admits exactly that nonce.
12663        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12664        supervisor.apply_identity_configuration(&ModuleSpec {
12665            module_id: "never-spawned".to_string(),
12666            program: PathBuf::from("/usr/bin/false"),
12667            args: Vec::new(),
12668            env: Vec::new(),
12669            reserved: true,
12670            reserved_prefixes: Vec::new(),
12671            protocol: ModuleProtocol::Subc,
12672            overlap: Default::default(),
12673            carry_containment: false,
12674        });
12675        assert!(supervisor
12676            .reserved_hello_verification("never-spawned", Some("minted"))
12677            .is_ok());
12678        assert!(supervisor
12679            .reserved_hello_verification("never-spawned", Some("forged"))
12680            .is_err());
12681    }
12682
12683    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12684    /// happened, which is what "spent budget" looks like to every reader.
12685    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12686        let now = ClockInstant::now();
12687        for _ in 0..count {
12688            state.crash_restarts.push_back(now);
12689        }
12690    }
12691
12692    /// Age the oldest recorded restart out of `window`, standing in for the hours
12693    /// that would otherwise have to pass. Injecting the instant is the point: a
12694    /// test that slept a real window would take ten minutes and still prove less.
12695    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12696        let aged = state
12697            .crash_restarts
12698            .front()
12699            .expect("a crash restart must be recorded before it can be aged")
12700            .checked_sub(window + Duration::from_secs(1))
12701            .expect("the test clock is far enough from its origin to age an instant");
12702        state.crash_restarts[0] = aged;
12703    }
12704
12705    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12706        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12707        seed_crash_restarts(&mut state, count);
12708        state
12709    }
12710
12711    #[test]
12712    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12713        let policy = RestartPolicy::new(3, Duration::ZERO);
12714        let now = ClockInstant::now();
12715        assert!(daemon_will_restart(
12716            &mut snapshot_with_restarts(true, 2),
12717            &policy,
12718            now
12719        ));
12720        assert!(!daemon_will_restart(
12721            &mut snapshot_with_restarts(true, 3),
12722            &policy,
12723            now
12724        ));
12725        assert!(!daemon_will_restart(
12726            &mut snapshot_with_restarts(false, 0),
12727            &policy,
12728            now
12729        ));
12730    }
12731
12732    #[test]
12733    fn crash_restart_backoff_escalates_with_in_window_count() {
12734        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12735            .with_max_backoff(Duration::from_secs(30));
12736        let now = ClockInstant::now();
12737        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12738        let schedules = (0..4)
12739            .map(|_| {
12740                state
12741                    .next_crash_restart(&policy, now)
12742                    .expect("the test policy allows four crash restarts")
12743            })
12744            .collect::<Vec<_>>();
12745
12746        assert_eq!(
12747            schedules
12748                .iter()
12749                .map(|schedule| schedule.restart_in_window)
12750                .collect::<Vec<_>>(),
12751            vec![0, 1, 2, 3]
12752        );
12753        assert_eq!(
12754            schedules
12755                .iter()
12756                .map(|schedule| schedule.delay)
12757                .collect::<Vec<_>>(),
12758            vec![
12759                Duration::from_millis(100),
12760                Duration::from_secs(1),
12761                Duration::from_secs(10),
12762                Duration::from_secs(30),
12763            ]
12764        );
12765    }
12766
12767    #[test]
12768    fn crash_restart_backoff_resets_after_ring_clear() {
12769        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12770        let now = ClockInstant::now();
12771        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12772        assert_eq!(
12773            state.next_crash_restart(&policy, now).unwrap().delay,
12774            Duration::from_millis(100)
12775        );
12776        assert_eq!(
12777            state.next_crash_restart(&policy, now).unwrap().delay,
12778            Duration::from_secs(1)
12779        );
12780
12781        state.clear_crash_restarts();
12782        let schedule = state
12783            .next_crash_restart(&policy, now)
12784            .expect("a cleared ring must allow another restart");
12785        assert_eq!(schedule.restart_in_window, 0);
12786        assert_eq!(schedule.delay, Duration::from_millis(100));
12787    }
12788
12789    #[test]
12790    fn crash_restart_backoff_ignores_aged_restarts() {
12791        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12792        let now = ClockInstant::now();
12793        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12794        state
12795            .next_crash_restart(&policy, now)
12796            .expect("the first restart is allowed");
12797        state
12798            .next_crash_restart(&policy, now)
12799            .expect("the second restart is allowed");
12800        state.crash_restarts[0] = now
12801            .checked_sub(policy.window + Duration::from_secs(1))
12802            .expect("the fake clock can age a restart past the window");
12803
12804        let schedule = state
12805            .next_crash_restart(&policy, now)
12806            .expect("an aged restart must release its slot");
12807        assert_eq!(schedule.restart_in_window, 1);
12808        assert_eq!(schedule.delay, Duration::from_secs(1));
12809        assert_eq!(state.crash_restarts.len(), 2);
12810    }
12811
12812    /// The budget is a rate: the same three spent restarts refuse a respawn
12813    /// while they are recent and allow one once they have aged past the window.
12814    /// Nothing about the module changed in between, which is the whole point.
12815    #[test]
12816    fn a_budget_spent_before_the_window_no_longer_refuses() {
12817        let policy = RestartPolicy::new(3, Duration::ZERO);
12818        let mut state = snapshot_with_restarts(true, 3);
12819        let now = ClockInstant::now();
12820        assert!(!daemon_will_restart(&mut state, &policy, now));
12821
12822        assert!(daemon_will_restart(
12823            &mut state,
12824            &policy,
12825            now + policy.window + Duration::from_secs(1)
12826        ));
12827        assert!(
12828            state.crash_restarts.is_empty(),
12829            "reading the budget must drop the instants that left the window"
12830        );
12831    }
12832
12833    fn module_with_recovery_snapshot(
12834        state: ModuleState,
12835        enabled: bool,
12836        restart_count: u32,
12837    ) -> SupervisedModule {
12838        let registry = Arc::new(Registry::default());
12839        let supervisor =
12840            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12841        let runtime = supervisor.runtime_config();
12842        let spec = ModuleSpec {
12843            module_id: "recovery-snapshot".to_string(),
12844            program: fake_aft_stub_path(),
12845            args: Vec::new(),
12846            env: Vec::new(),
12847            reserved: false,
12848            reserved_prefixes: Vec::new(),
12849            protocol: ModuleProtocol::Subc,
12850            overlap: Default::default(),
12851            carry_containment: false,
12852        };
12853        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12854        seed_crash_restarts(&mut snapshot, restart_count);
12855        // These tests read synthetic snapshots. A real child and monitor would
12856        // race those reads by replacing the requested state during startup.
12857        let (commands, _rx) = mpsc::channel(4);
12858        SupervisedModule {
12859            inner: Arc::new(SupervisedModuleInner {
12860                module_id: spec.module_id.clone(),
12861                registry,
12862                snapshot: Arc::new(Mutex::new(snapshot)),
12863                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12864                    spec,
12865                    health: runtime.health,
12866                })),
12867                stderr_ring: runtime.stderr_ring,
12868                terminal_ring: runtime.terminal_ring,
12869                commands,
12870                monitor: Mutex::new(None),
12871                restart_policy: runtime.restart_policy,
12872                effective_drain_timeout: runtime.effective_drain_timeout,
12873                provenance_probe: supervisor.provenance_probe.clone(),
12874                retained: runtime.child_roster.retained_view(),
12875            }),
12876        }
12877    }
12878
12879    #[cfg(target_os = "linux")]
12880    #[tokio::test]
12881    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12882        let supervisor =
12883            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12884                .with_cgroup_placement(None);
12885        let result = supervisor.spawn(ModuleSpec {
12886            module_id: "no-cgroup-placement".to_string(),
12887            program: fake_aft_stub_path(),
12888            args: Vec::new(),
12889            env: Vec::new(),
12890            reserved: false,
12891            reserved_prefixes: Vec::new(),
12892            protocol: ModuleProtocol::Subc,
12893            overlap: Default::default(),
12894            carry_containment: false,
12895        });
12896
12897        assert!(
12898            result.is_ok(),
12899            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12900        );
12901    }
12902
12903    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12904    async fn undecided_snapshot_uses_shared_restart_predicate() {
12905        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12906            .will_recover_after_connection_loss()
12907            .unwrap());
12908        assert!(
12909            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12910                .will_recover_after_connection_loss()
12911                .unwrap()
12912        );
12913    }
12914
12915    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12916    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12917        assert!(
12918            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12919                .will_recover_after_connection_loss()
12920                .unwrap()
12921        );
12922    }
12923
12924    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12926        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12927            .will_recover_after_connection_loss()
12928            .unwrap());
12929        assert!(
12930            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12931                .will_recover_after_connection_loss()
12932                .unwrap()
12933        );
12934    }
12935
12936    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12937    async fn warming_snapshot_is_limited_to_startup_phases() {
12938        for state in [
12939            ModuleState::Starting,
12940            ModuleState::Running,
12941            ModuleState::Restarting,
12942        ] {
12943            assert!(
12944                module_with_recovery_snapshot(state, true, 0)
12945                    .is_warming()
12946                    .unwrap(),
12947                "{state:?} should be warming"
12948            );
12949        }
12950        for state in [
12951            ModuleState::Unresponsive,
12952            ModuleState::Draining,
12953            ModuleState::Stopped,
12954            ModuleState::Failed,
12955            ModuleState::Disabled,
12956        ] {
12957            assert!(
12958                !module_with_recovery_snapshot(state, true, 0)
12959                    .is_warming()
12960                    .unwrap(),
12961                "{state:?} should not be warming"
12962            );
12963        }
12964    }
12965
12966    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12967    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12968        let registry = Arc::new(Registry::default());
12969        let supervisor =
12970            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12971        let module = supervisor
12972            .spawn(ModuleSpec {
12973                module_id: "terminal-history".to_string(),
12974                program: fake_aft_stub_path(),
12975                args: Vec::new(),
12976                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12977                reserved: false,
12978                reserved_prefixes: Vec::new(),
12979                protocol: ModuleProtocol::Subc,
12980                overlap: Default::default(),
12981                carry_containment: false,
12982            })
12983            .unwrap();
12984
12985        let deadline = Instant::now() + Duration::from_secs(5);
12986        loop {
12987            let history = module.terminal_history();
12988            if history.entries.len() == 2 {
12989                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12990                assert_eq!(history.dropped, 0);
12991                assert_eq!(
12992                    history
12993                        .entries
12994                        .iter()
12995                        .map(|entry| entry.exit_code)
12996                        .collect::<Vec<_>>(),
12997                    vec![Some(23), Some(23)]
12998                );
12999                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13000                return;
13001            }
13002            assert!(
13003                Instant::now() < deadline,
13004                "module did not retain two terminal exits: {history:?}"
13005            );
13006            sleep(Duration::from_millis(10)).await;
13007        }
13008    }
13009
13010    /// A disable issued while a crash respawn is still backing off must preempt
13011    /// that respawn: the operator's stop wins, the disable must not queue behind
13012    /// the backoff, and the module must never come back up afterwards.
13013    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13014    async fn disable_during_crash_backoff_cancels_pending_respawn() {
13015        let backoff = Duration::from_secs(2);
13016        let supervisor = Supervisor::new_for_test(
13017            Arc::new(Registry::default()),
13018            RestartPolicy::new(10, backoff),
13019        );
13020        let module = supervisor
13021            .spawn(ModuleSpec {
13022                module_id: "disable-during-backoff".to_string(),
13023                program: fake_aft_stub_path(),
13024                args: Vec::new(),
13025                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13026                reserved: false,
13027                reserved_prefixes: Vec::new(),
13028                protocol: ModuleProtocol::Subc,
13029                overlap: Default::default(),
13030                carry_containment: false,
13031            })
13032            .unwrap();
13033
13034        // Wait for the first crash to put the module into its backoff window.
13035        let deadline = Instant::now() + Duration::from_secs(5);
13036        loop {
13037            if module.status().unwrap().state == ModuleState::Restarting {
13038                break;
13039            }
13040            assert!(
13041                Instant::now() < deadline,
13042                "module never entered the crash backoff"
13043            );
13044            sleep(Duration::from_millis(10)).await;
13045        }
13046
13047        let started = Instant::now();
13048        module.set_enabled(false).await.unwrap();
13049        let waited = started.elapsed();
13050
13051        assert!(
13052            waited < backoff / 2,
13053            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13054        );
13055        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13056
13057        // Outlast the backoff: the respawn it was counting down to must never run.
13058        sleep(backoff + Duration::from_millis(500)).await;
13059        let status = module.status().unwrap();
13060        assert_eq!(status.state, ModuleState::Disabled);
13061        assert_eq!(
13062            status.spawn_generation, 1,
13063            "module respawned after the operator disabled it"
13064        );
13065    }
13066
13067    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
13068    /// the shape of nats-server, the program this rule exists for.
13069    #[cfg(unix)]
13070    fn protocol_none_sigterm_exits_clean_spec(
13071        module_id: &str,
13072        dir: &std::path::Path,
13073    ) -> (ModuleSpec, PathBuf, PathBuf) {
13074        let ready = dir.join("ready");
13075        let marker = dir.join("sigterm");
13076        let spec = ModuleSpec {
13077            module_id: module_id.to_string(),
13078            program: fake_aft_stub_path(),
13079            args: Vec::new(),
13080            env: vec![
13081                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13082                (
13083                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13084                    marker.display().to_string(),
13085                ),
13086                (
13087                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13088                    ready.display().to_string(),
13089                ),
13090            ],
13091            reserved: false,
13092            reserved_prefixes: Vec::new(),
13093            protocol: ModuleProtocol::None,
13094            overlap: Default::default(),
13095            carry_containment: false,
13096        };
13097        (spec, ready, marker)
13098    }
13099
13100    /// Wait for a file the child writes, so a signal is never sent before the
13101    /// child's SIGTERM handler is installed (the default disposition would
13102    /// kill it by signal and the exit would not be clean).
13103    #[cfg(unix)]
13104    async fn wait_for_file(path: &std::path::Path) {
13105        let deadline = Instant::now() + Duration::from_secs(10);
13106        while !path.exists() {
13107            assert!(
13108                Instant::now() < deadline,
13109                "{} never appeared",
13110                path.display()
13111            );
13112            sleep(Duration::from_millis(10)).await;
13113        }
13114    }
13115
13116    /// A protocol-none module that exits 0 because something OUTSIDE the
13117    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
13118    /// the crash-path disposition rather than `stopped`.
13119    #[cfg(unix)]
13120    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13121    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13122        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13123        let (spec, ready, marker) =
13124            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13125        let supervisor = Supervisor::new_for_test(
13126            Arc::new(Registry::default()),
13127            RestartPolicy::new(3, Duration::ZERO),
13128        );
13129        let module = supervisor.spawn(spec).unwrap();
13130        wait_for_file(&ready).await;
13131        // The ready file proves the child installed its SIGTERM handler, not
13132        // that the supervisor has processed the privacy trampoline's exec
13133        // acknowledgement. On macOS status withholds the pid until then.
13134        let deadline = Instant::now() + Duration::from_secs(10);
13135        let first_pid = loop {
13136            let status = module.status().unwrap();
13137            if status.state == ModuleState::Running {
13138                if let Some(pid) = status.pid {
13139                    break pid;
13140                }
13141            }
13142            assert!(
13143                Instant::now() < deadline,
13144                "a running module must report its pid after exec confirmation: {status:?}"
13145            );
13146            sleep(Duration::from_millis(10)).await;
13147        };
13148
13149        rustix::process::kill_process(
13150            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13151            rustix::process::Signal::TERM,
13152        )
13153        .unwrap();
13154
13155        let deadline = Instant::now() + Duration::from_secs(10);
13156        let respawned = loop {
13157            let status = module.status().unwrap();
13158            if status.state == ModuleState::Running
13159                && status.pid.is_some_and(|pid| pid != first_pid)
13160            {
13161                break status;
13162            }
13163            assert!(
13164                Instant::now() < deadline,
13165                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13166            );
13167            sleep(Duration::from_millis(10)).await;
13168        };
13169        assert_eq!(respawned.spawn_generation, 2);
13170        assert!(
13171            marker.exists(),
13172            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13173        );
13174
13175        let history = module.terminal_history();
13176        assert_eq!(history.entries.len(), 1, "{history:?}");
13177        let entry = &history.entries[0];
13178        assert_eq!(entry.exit_code, Some(0));
13179        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13180        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13181
13182        module.stop().await.unwrap();
13183    }
13184
13185    /// Repeated unrequested clean exits of a protocol-none module spend the
13186    /// restart budget exactly as crashes do, and the module ends `failed` with
13187    /// the budget named.
13188    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13189    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13190        let supervisor = Supervisor::new_for_test(
13191            Arc::new(Registry::default()),
13192            RestartPolicy::new(1, Duration::ZERO),
13193        );
13194        let module = supervisor
13195            .spawn(ModuleSpec {
13196                module_id: "none-clean-exit-budget".to_string(),
13197                program: fake_aft_stub_path(),
13198                args: Vec::new(),
13199                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13200                reserved: false,
13201                reserved_prefixes: Vec::new(),
13202                protocol: ModuleProtocol::None,
13203                overlap: Default::default(),
13204                carry_containment: false,
13205            })
13206            .unwrap();
13207
13208        // Failed follows the terminal write; this deadline only bounds a hang,
13209        // not an assumed duration for the two launches or their exit recording.
13210        let deadline = Instant::now() + Duration::from_secs(10);
13211        loop {
13212            let status = module.status().unwrap();
13213            if status.state == ModuleState::Failed {
13214                break;
13215            }
13216            assert!(
13217                Instant::now() < deadline,
13218                "module never exhausted its budget: {status:?} {:?}",
13219                module.terminal_history()
13220            );
13221            sleep(Duration::from_millis(10)).await;
13222        }
13223        let history = module.terminal_history();
13224        assert_eq!(
13225            history
13226                .entries
13227                .iter()
13228                .map(|entry| (entry.exit_code, entry.disposition.clone()))
13229                .collect::<Vec<_>>(),
13230            vec![
13231                (Some(0), TerminalDisposition::Restarting),
13232                (Some(0), TerminalDisposition::Failed),
13233            ]
13234        );
13235        let detail = history.entries[1]
13236            .disposition_detail
13237            .as_deref()
13238            .expect("a budget failure names the budget");
13239        assert!(detail.contains("max_restarts=1"), "{detail}");
13240        assert_eq!(module.status().unwrap().spawn_generation, 2);
13241    }
13242
13243    #[test]
13244    fn restart_budget_failure_is_published_after_its_terminal_record() {
13245        use std::io::{Read, Write};
13246        let supervisor = Supervisor::new_for_test(
13247            Arc::new(Registry::default()),
13248            RestartPolicy::new(0, Duration::ZERO),
13249        );
13250        let runtime = supervisor.runtime_config();
13251        let spec = ModuleSpec {
13252            protocol: ModuleProtocol::None,
13253            ..windowed_crash_spec("budget-publication-order")
13254        };
13255        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13256            ModuleState::Running,
13257            true,
13258        )));
13259        lock_snapshot(&snapshot).unwrap().process_alive = true;
13260        let reaped_snapshot = snapshot.clone();
13261        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13262        let history = runtime.terminal_ring.clone();
13263        let reap = std::thread::spawn(move || {
13264            tokio::runtime::Builder::new_current_thread()
13265                .enable_all()
13266                .build()
13267                .unwrap()
13268                .block_on(on_child_exit(
13269                    &spec,
13270                    runtime.restart_policy,
13271                    &supervisor.registry,
13272                    &reaped_snapshot,
13273                    &runtime.terminal_ring,
13274                    &runtime.spawn_events,
13275                    &runtime.child_roster,
13276                    ExitReport {
13277                        kind: ExitKind::Clean,
13278                        code: Some(0),
13279                        signal: None,
13280                        at_ms: 1,
13281                    },
13282                    true,
13283                ))
13284        });
13285        let mut reached = [0];
13286        barrier.read_exact(&mut reached).unwrap();
13287        assert_eq!(&reached, b"R");
13288        let (before_state, before_alive) = {
13289            let state = lock_snapshot(&snapshot).unwrap();
13290            (state.state, state.process_alive)
13291        };
13292        let before_history = history.lock().unwrap().snapshot();
13293        barrier.write_all(b"X").unwrap();
13294        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13295        assert!(before_history.entries.is_empty());
13296        assert_ne!(
13297            before_state,
13298            ModuleState::Failed,
13299            "Failed was visible before its terminal record could be written"
13300        );
13301        assert!(before_alive);
13302        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13303        let history = history.lock().unwrap().snapshot();
13304        assert_eq!(history.entries.len(), 1);
13305        assert_eq!(history.entries[0].exit_code, Some(0));
13306        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13307    }
13308
13309    #[test]
13310    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13311        use std::io::{Read, Write};
13312
13313        let supervisor = Supervisor::new_for_test(
13314            Arc::new(Registry::default()),
13315            RestartPolicy::new(3, Duration::ZERO),
13316        );
13317        let mut runtime = supervisor.runtime_config();
13318        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13319        let snapshot = module.inner.snapshot.clone();
13320        lock_snapshot(&snapshot).unwrap().process_alive = true;
13321        runtime.terminal_ring = module.inner.terminal_ring.clone();
13322        let spec = windowed_crash_spec(module.module_id());
13323        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13324        let reaped_snapshot = snapshot.clone();
13325        let reap = std::thread::spawn(move || {
13326            tokio::runtime::Builder::new_current_thread()
13327                .enable_all()
13328                .build()
13329                .unwrap()
13330                .block_on(on_child_exit(
13331                    &spec,
13332                    runtime.restart_policy,
13333                    &supervisor.registry,
13334                    &reaped_snapshot,
13335                    &runtime.terminal_ring,
13336                    &runtime.spawn_events,
13337                    &runtime.child_roster,
13338                    ExitReport {
13339                        kind: ExitKind::Crash,
13340                        code: Some(2),
13341                        signal: None,
13342                        at_ms: 1,
13343                    },
13344                    true,
13345                ))
13346        });
13347
13348        // Hold the crash handler between its budget decision and publication.
13349        // Connection cleanup must still see the recovery that this crash chose,
13350        // even when that recovery consumes the last available restart slot.
13351        let mut reached = [0];
13352        barrier.read_exact(&mut reached).unwrap();
13353        assert_eq!(&reached, b"R");
13354        let before_status = module.status().unwrap();
13355        let before_history = module.terminal_history();
13356        let before_recovers = module.will_recover_after_connection_loss().unwrap();
13357        barrier.write_all(b"X").unwrap();
13358        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13359
13360        assert_eq!(before_status.state, ModuleState::Running);
13361        assert!(before_status.process_alive);
13362        assert_eq!(before_status.restart_count, 3);
13363        assert!(before_history.entries.is_empty());
13364        assert!(
13365            before_recovers,
13366            "connection cleanup would report a terminal close for a scheduled crash restart"
13367        );
13368        let restarting = module.status().unwrap();
13369        assert_eq!(restarting.state, ModuleState::Restarting);
13370        assert_eq!(restarting.restart_count, 3);
13371        assert!(module.will_recover_after_connection_loss().unwrap());
13372        let history = module.terminal_history();
13373        assert_eq!(history.entries.len(), 1);
13374        assert_eq!(
13375            history.entries[0].disposition,
13376            TerminalDisposition::Restarting
13377        );
13378
13379        // Once the replacement is running, its next crash must be judged
13380        // against the exhausted budget rather than the previous recovery.
13381        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13382            state.state = ModuleState::Running;
13383            state.respawn_pending = false;
13384        })
13385        .unwrap();
13386        assert!(!module.will_recover_after_connection_loss().unwrap());
13387        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13388            state.enabled = false;
13389            state.respawn_pending = true;
13390        })
13391        .unwrap();
13392        assert!(!module.will_recover_after_connection_loss().unwrap());
13393    }
13394
13395    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13396        use std::net::{TcpListener, TcpStream};
13397        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13398        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13399        let (barrier, _) = listener.accept().unwrap();
13400        for stream in [&writer, &barrier] {
13401            stream
13402                .set_read_timeout(Some(Duration::from_secs(10)))
13403                .unwrap();
13404            stream
13405                .set_write_timeout(Some(Duration::from_secs(10)))
13406                .unwrap();
13407        }
13408        *events.1.lock().unwrap() = Some(writer);
13409        barrier
13410    }
13411
13412    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13413        use std::io::{Read, Write};
13414        let barrier = events.1.lock().unwrap().take();
13415        if let Some(mut barrier) = barrier {
13416            barrier.write_all(b"R").unwrap();
13417            let mut release = [0];
13418            barrier.read_exact(&mut release).unwrap();
13419            assert_eq!(&release, b"X");
13420        }
13421    }
13422
13423    #[test]
13424    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13425        use std::io::{Read, Write};
13426
13427        let supervisor = Supervisor::new_for_test(
13428            Arc::new(Registry::default()),
13429            RestartPolicy::new(0, Duration::ZERO),
13430        );
13431        let mut runtime = supervisor.runtime_config();
13432        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13433        let snapshot = module.inner.snapshot.clone();
13434        runtime.terminal_ring = module.inner.terminal_ring.clone();
13435        let spec = ModuleSpec {
13436            module_id: module.module_id().into(),
13437            program: fake_aft_stub_path(),
13438            protocol: ModuleProtocol::None,
13439            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13440            ..windowed_crash_spec(module.module_id())
13441        };
13442        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13443        let reap = std::thread::spawn(move || {
13444            tokio::runtime::Builder::new_current_thread()
13445                .enable_all()
13446                .build()
13447                .unwrap()
13448                .block_on(async {
13449                    let mut child =
13450                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13451                    health_restart_child(
13452                        &spec,
13453                        &runtime,
13454                        &supervisor.registry,
13455                        &supervisor.process_liveness,
13456                        &snapshot,
13457                        &mut child,
13458                        SupervisorHealthStatus::Failing,
13459                        Some("health probe did not answer"),
13460                        1,
13461                    )
13462                    .await
13463                    .unwrap();
13464                    assert!(child.is_none());
13465                });
13466        });
13467        // The loopback barrier acknowledges entry into terminal recording by
13468        // health_restart_child's drain, then waits for the test's release byte.
13469        // Status and history are observed while that writer is held, not after
13470        // an elapsed-time guess about when the health action has completed.
13471        let mut reached = [0];
13472        barrier.read_exact(&mut reached).unwrap();
13473        assert_eq!(&reached, b"R");
13474        let before_status = module.status().unwrap();
13475        let before_history = module.terminal_history();
13476        barrier.write_all(b"X").unwrap();
13477        reap.join().unwrap();
13478        assert!(before_history.entries.is_empty());
13479        assert_ne!(
13480            before_status.state,
13481            ModuleState::Failed,
13482            "Failed was visible before its terminal record could be written"
13483        );
13484        assert!(
13485            before_status.process_alive,
13486            "process death was visible before its terminal record could be written"
13487        );
13488        let failed = module.status().unwrap();
13489        assert_eq!(failed.state, ModuleState::Failed);
13490        assert!(!failed.process_alive);
13491        let history = module.terminal_history();
13492        assert_eq!(history.entries.len(), 1);
13493        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13494        assert_eq!(
13495            history.entries[0].disposition_detail.as_deref(),
13496            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13497        );
13498    }
13499
13500    #[test]
13501    fn observed_exits_publish_process_death_after_terminal_history() {
13502        use std::io::{Read, Write};
13503        for (kind, enabled, shutdown, wait_error, expected_state) in [
13504            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13505            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13506            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13507            (
13508                ExitKind::DeliberateSeverance,
13509                true,
13510                false,
13511                false,
13512                ModuleState::Restarting,
13513            ),
13514            #[cfg(unix)]
13515            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13516            (ExitKind::Crash, true, false, true, ModuleState::Failed),
13517        ] {
13518            let supervisor = Supervisor::new_for_test(
13519                Arc::new(Registry::default()),
13520                RestartPolicy::new(1, Duration::ZERO),
13521            );
13522            let runtime = supervisor.runtime_config();
13523            #[cfg(unix)]
13524            if shutdown {
13525                runtime.child_roster.close();
13526            }
13527            #[cfg(not(unix))]
13528            let _ = shutdown;
13529            let spec = windowed_crash_spec("exit-publication-order");
13530            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13531                ModuleState::Running,
13532                enabled,
13533            )));
13534            lock_snapshot(&snapshot).unwrap().process_alive = true;
13535            let reaped_snapshot = snapshot.clone();
13536            let history = runtime.terminal_ring.clone();
13537            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13538            let reap = std::thread::spawn(move || {
13539                if wait_error {
13540                    super::fail_after_wait_error(
13541                        &spec.module_id,
13542                        &reaped_snapshot,
13543                        &runtime.terminal_ring,
13544                        &runtime.spawn_events,
13545                        "wait failed".into(),
13546                        &runtime.failure_context(true),
13547                    );
13548                } else {
13549                    tokio::runtime::Builder::new_current_thread()
13550                        .enable_all()
13551                        .build()
13552                        .unwrap()
13553                        .block_on(on_child_exit(
13554                            &spec,
13555                            runtime.restart_policy,
13556                            &supervisor.registry,
13557                            &reaped_snapshot,
13558                            &runtime.terminal_ring,
13559                            &runtime.spawn_events,
13560                            &runtime.child_roster,
13561                            ExitReport {
13562                                kind,
13563                                code: Some(0),
13564                                signal: None,
13565                                at_ms: 1,
13566                            },
13567                            true,
13568                        ));
13569                }
13570            });
13571            let mut reached = [0];
13572            barrier.read_exact(&mut reached).unwrap();
13573            assert_eq!(&reached, b"R");
13574            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13575            let before_history = history.lock().unwrap().snapshot();
13576            barrier.write_all(b"X").unwrap();
13577            reap.join().unwrap();
13578            assert!(
13579                before_alive,
13580                "{kind:?}: death was published before recording"
13581            );
13582            assert!(before_history.entries.is_empty());
13583            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13584            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13585            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13586        }
13587    }
13588
13589    #[tokio::test]
13590    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13591        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13592        let path = dir.join("terminals.jsonl");
13593        std::fs::create_dir(&path).unwrap();
13594        let supervisor = Supervisor::new_for_test(
13595            Arc::new(Registry::default()),
13596            RestartPolicy::new(0, Duration::ZERO),
13597        )
13598        .with_terminal_journal(path, "health-budget-journal-failure".into());
13599        let runtime = supervisor.runtime_config();
13600        let spec = ModuleSpec {
13601            program: fake_aft_stub_path(),
13602            protocol: ModuleProtocol::None,
13603            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13604            ..windowed_crash_spec("health-budget-journal-failure")
13605        };
13606        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13607        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13608        health_restart_child(
13609            &spec,
13610            &runtime,
13611            &supervisor.registry,
13612            &supervisor.process_liveness,
13613            &snapshot,
13614            &mut child,
13615            SupervisorHealthStatus::Failing,
13616            None,
13617            1,
13618        )
13619        .await
13620        .unwrap();
13621        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13622        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13623        let history = runtime
13624            .terminal_ring
13625            .lock()
13626            .unwrap()
13627            .durable_history(&spec.module_id);
13628        assert!(history.journal_write_failures > 0);
13629        assert_eq!(history.entries.len(), 1);
13630        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13631    }
13632
13633    #[test]
13634    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13635        let supervisor = Supervisor::new_for_test(
13636            Arc::new(Registry::default()),
13637            RestartPolicy::new(0, Duration::ZERO),
13638        );
13639        let runtime = supervisor.runtime_config();
13640        let spec = ModuleSpec {
13641            program: fake_aft_stub_path(),
13642            protocol: ModuleProtocol::None,
13643            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13644            ..windowed_crash_spec("health-budget-recording-panic")
13645        };
13646        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13647        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13648            tokio::runtime::Builder::new_current_thread()
13649                .enable_all()
13650                .build()
13651                .unwrap()
13652                .block_on(async {
13653                    let mut child =
13654                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13655                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13656                    health_restart_child(
13657                        &spec,
13658                        &runtime,
13659                        &supervisor.registry,
13660                        &supervisor.process_liveness,
13661                        &snapshot,
13662                        &mut child,
13663                        SupervisorHealthStatus::Failing,
13664                        None,
13665                        1,
13666                    )
13667                    .await
13668                    .unwrap();
13669                });
13670        }));
13671        let panic = result.expect_err("recording must still unwind");
13672        assert_eq!(
13673            panic.downcast_ref::<String>().map(String::as_str),
13674            Some("spawn event sequence exhausted")
13675        );
13676        let state = lock_snapshot(&snapshot).unwrap();
13677        assert_eq!(state.state, ModuleState::Failed);
13678        assert!(!state.process_alive);
13679        assert!(state.last_exit.is_some());
13680    }
13681
13682    #[tokio::test]
13683    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13684        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13685        let path = dir.join("terminals.jsonl");
13686        std::fs::create_dir(&path).unwrap();
13687        let supervisor = Supervisor::new_for_test(
13688            Arc::new(Registry::default()),
13689            RestartPolicy::new(0, Duration::ZERO),
13690        )
13691        .with_terminal_journal(path, "budget-journal-failure".into());
13692        let runtime = supervisor.runtime_config();
13693        let spec = windowed_crash_spec("budget-journal-failure");
13694        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13695        assert!(matches!(
13696            on_child_exit(
13697                &spec,
13698                runtime.restart_policy,
13699                &supervisor.registry,
13700                &snapshot,
13701                &runtime.terminal_ring,
13702                &runtime.spawn_events,
13703                &runtime.child_roster,
13704                crash_exit_report(1),
13705                true,
13706            )
13707            .await,
13708            NextAction::Stop { .. }
13709        ));
13710        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13711        let history = runtime
13712            .terminal_ring
13713            .lock()
13714            .unwrap()
13715            .durable_history(&spec.module_id);
13716        assert!(history.journal_write_failures > 0);
13717        assert_eq!(history.entries.len(), 1);
13718        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13719    }
13720
13721    #[test]
13722    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13723        let supervisor = Supervisor::new_for_test(
13724            Arc::new(Registry::default()),
13725            RestartPolicy::new(0, Duration::ZERO),
13726        );
13727        let runtime = supervisor.runtime_config();
13728        let spec = windowed_crash_spec("budget-recording-panic");
13729        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13730        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13731        // Exhausting the event sequence makes emit_exited panic before the
13732        // terminal write, exercising publication on the recording unwind.
13733        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13734        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13735            tokio::runtime::Builder::new_current_thread()
13736                .enable_all()
13737                .build()
13738                .unwrap()
13739                .block_on(on_child_exit(
13740                    &spec,
13741                    runtime.restart_policy,
13742                    &supervisor.registry,
13743                    &snapshot,
13744                    &runtime.terminal_ring,
13745                    &runtime.spawn_events,
13746                    &runtime.child_roster,
13747                    crash_exit_report(1),
13748                    true,
13749                ))
13750        }));
13751        let panic = result.err().expect("recording must still unwind");
13752        assert_eq!(
13753            panic.downcast_ref::<String>().map(String::as_str),
13754            Some("spawn event sequence exhausted")
13755        );
13756        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13757    }
13758
13759    /// A stop the supervisor itself requests still stops a protocol-none
13760    /// module, even though the child answers the SIGTERM with exit 0.
13761    #[cfg(unix)]
13762    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13763    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13764        for disable in [false, true] {
13765            let label = if disable {
13766                "none-requested-disable"
13767            } else {
13768                "none-requested-stop"
13769            };
13770            let dir = cortexkit_test_support::ScratchDir::new(label);
13771            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13772            let supervisor = Supervisor::new_for_test(
13773                Arc::new(Registry::default()),
13774                RestartPolicy::new(3, Duration::ZERO),
13775            );
13776            let module = supervisor.spawn(spec).unwrap();
13777            wait_for_file(&ready).await;
13778
13779            if disable {
13780                module.set_enabled(false).await.unwrap();
13781            } else {
13782                module.stop().await.unwrap();
13783            }
13784            assert!(
13785                marker.exists(),
13786                "{label}: the child must have left through its SIGTERM handler with exit 0"
13787            );
13788
13789            // Long enough for a zero-backoff respawn to have happened if the
13790            // exit had been treated as a crash.
13791            sleep(Duration::from_millis(500)).await;
13792            let status = module.status().unwrap();
13793            let expected = if disable {
13794                ModuleState::Disabled
13795            } else {
13796                ModuleState::Stopped
13797            };
13798            assert_eq!(status.state, expected, "{label}");
13799            assert_eq!(
13800                status.spawn_generation, 1,
13801                "{label}: respawned after a requested stop"
13802            );
13803            let history = module.terminal_history();
13804            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13805            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13806            assert_ne!(
13807                history.entries[0].disposition,
13808                TerminalDisposition::Restarting,
13809                "{label}"
13810            );
13811        }
13812    }
13813
13814    /// A subc-wire module that exits 0 on its own is still a stop: the
13815    /// protocol-none rule must not reach it.
13816    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13817    async fn subc_wire_clean_exit_is_still_a_stop() {
13818        let supervisor = Supervisor::new_for_test(
13819            Arc::new(Registry::default()),
13820            RestartPolicy::new(3, Duration::ZERO),
13821        );
13822        let module = supervisor
13823            .spawn(ModuleSpec {
13824                module_id: "wire-clean-exit".to_string(),
13825                program: fake_aft_stub_path(),
13826                args: Vec::new(),
13827                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13828                reserved: false,
13829                reserved_prefixes: Vec::new(),
13830                protocol: ModuleProtocol::Subc,
13831                overlap: Default::default(),
13832                carry_containment: false,
13833            })
13834            .unwrap();
13835
13836        let deadline = Instant::now() + Duration::from_secs(10);
13837        while module.terminal_history().entries.is_empty() {
13838            assert!(Instant::now() < deadline, "module never exited");
13839            sleep(Duration::from_millis(10)).await;
13840        }
13841        // Long enough for a zero-backoff respawn to have happened.
13842        sleep(Duration::from_millis(500)).await;
13843        let status = module.status().unwrap();
13844        assert_eq!(status.state, ModuleState::Stopped);
13845        assert_eq!(status.spawn_generation, 1);
13846        let history = module.terminal_history();
13847        assert_eq!(history.entries.len(), 1, "{history:?}");
13848        assert_eq!(history.entries[0].exit_code, Some(0));
13849        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13850    }
13851
13852    #[cfg(unix)]
13853    #[tokio::test]
13854    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13855        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13856        let record = dir.join("live-children.json");
13857        let supervisor = Supervisor::new_for_test(
13858            Arc::new(Registry::default()),
13859            RestartPolicy::new(0, Duration::ZERO),
13860        );
13861        let mut runtime = supervisor.runtime_config();
13862        runtime.child_roster.record_to(record.clone());
13863        let gate = Arc::new(super::ReloadExitRecordGate::default());
13864        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13865        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13866        let spec = ModuleSpec {
13867            module_id: "reload-exit-roster".into(),
13868            program: fake_aft_stub_path(),
13869            args: Vec::new(),
13870            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13871            reserved: false,
13872            reserved_prefixes: Vec::new(),
13873            protocol: ModuleProtocol::Subc,
13874            overlap: Default::default(),
13875            carry_containment: false,
13876        };
13877        let mut child = None;
13878        let reload = super::finish_reload_child(
13879            &spec,
13880            &runtime,
13881            &supervisor.registry,
13882            &supervisor.process_liveness,
13883            &snapshot,
13884            &mut child,
13885        );
13886        tokio::pin!(reload);
13887        tokio::select! {
13888            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13889            _ = gate.reached.notified() => {}
13890        }
13891        assert!(runtime
13892            .terminal_ring
13893            .lock()
13894            .unwrap()
13895            .snapshot()
13896            .entries
13897            .is_empty());
13898        assert_eq!(
13899            crate::live_children::read_record(&record).unwrap().len(),
13900            1,
13901            "shutdown must still wait for the reaped child until its terminal record exists"
13902        );
13903        runtime.child_roster.close();
13904        gate.resume.notify_one();
13905        assert!(reload.await.is_err());
13906        assert!(crate::live_children::read_record(&record)
13907            .unwrap()
13908            .is_empty());
13909        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13910        assert_eq!(history.entries.len(), 1);
13911        assert_eq!(
13912            history.entries[0].disposition,
13913            TerminalDisposition::DaemonShutdown
13914        );
13915    }
13916
13917    /// Each restart-producing arm has its own state transition. Keeping their
13918    /// lifetime count assertions adjacent prevents a later new arm from silently
13919    /// spending budget without recording the historical restart.
13920    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13921    async fn every_restart_increment_path_advances_lifetime_count() {
13922        let supervisor = Supervisor::new_for_test(
13923            Arc::new(Registry::default()),
13924            RestartPolicy::new(1, Duration::ZERO),
13925        );
13926        let runtime = supervisor.runtime_config();
13927        let spec = ModuleSpec {
13928            module_id: "lifetime-increment-path".to_string(),
13929            program: PathBuf::from("/unused/lifetime-increment-path"),
13930            args: Vec::new(),
13931            env: Vec::new(),
13932            reserved: false,
13933            reserved_prefixes: Vec::new(),
13934            protocol: ModuleProtocol::Subc,
13935            overlap: Default::default(),
13936            carry_containment: false,
13937        };
13938
13939        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13940        assert!(matches!(
13941            on_child_exit(
13942                &spec,
13943                runtime.restart_policy,
13944                &supervisor.registry,
13945                &crash_snapshot,
13946                &runtime.terminal_ring,
13947                &runtime.spawn_events,
13948                &runtime.child_roster,
13949                ExitReport {
13950                    kind: ExitKind::Crash,
13951                    code: Some(1),
13952                    signal: None,
13953                    at_ms: 1,
13954                },
13955                true,
13956            )
13957            .await,
13958            NextAction::Restart { schedule: _ }
13959        ));
13960        let (crash_restarts, crash_lifetime) = {
13961            let state = lock_snapshot(&crash_snapshot).unwrap();
13962            (state.crash_restarts.len(), state.lifetime_restarts)
13963        };
13964        assert_eq!(crash_restarts, 1);
13965        assert_eq!(crash_lifetime, 1);
13966
13967        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13968        let mut health_child = None;
13969        assert!(matches!(
13970            health_restart_child(
13971                &spec,
13972                &runtime,
13973                &supervisor.registry,
13974                &supervisor.process_liveness,
13975                &health_snapshot,
13976                &mut health_child,
13977                SupervisorHealthStatus::Failing,
13978                None,
13979                2,
13980            )
13981            .await,
13982            Ok(())
13983        ));
13984        assert!(health_child.is_none());
13985        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13986        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13987        let (health_restarts, health_lifetime) = {
13988            let state = lock_snapshot(&health_snapshot).unwrap();
13989            (state.crash_restarts.len(), state.lifetime_restarts)
13990        };
13991        assert_eq!(health_restarts, 1);
13992        assert_eq!(health_lifetime, 1);
13993
13994        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13995        let mut reload_child = None;
13996        assert!(matches!(
13997            handle_reload_spawn_failure(
13998                &spec,
13999                &runtime,
14000                &supervisor.process_liveness,
14001                &reload_snapshot,
14002                &mut reload_child,
14003                "forced reload spawn failure".to_string(),
14004            )
14005            .await,
14006            Err(SuperviseError::ReloadFailed { .. })
14007        ));
14008        let (reload_restarts, reload_lifetime) = {
14009            let state = lock_snapshot(&reload_snapshot).unwrap();
14010            (state.crash_restarts.len(), state.lifetime_restarts)
14011        };
14012        assert_eq!(reload_restarts, 1);
14013        assert_eq!(reload_lifetime, 1);
14014    }
14015
14016    #[tokio::test]
14017    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14018        let supervisor = Supervisor::new_for_test(
14019            Arc::new(Registry::default()),
14020            RestartPolicy::new(3, Duration::ZERO),
14021        );
14022        let runtime = supervisor.runtime_config();
14023        let spec = ModuleSpec {
14024            module_id: "deliberately-severed".to_string(),
14025            program: PathBuf::from("/unused/deliberately-severed"),
14026            args: Vec::new(),
14027            env: Vec::new(),
14028            reserved: false,
14029            reserved_prefixes: Vec::new(),
14030            protocol: ModuleProtocol::Subc,
14031            overlap: Default::default(),
14032            carry_containment: false,
14033        };
14034        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14035        let process = ProcessIdentity {
14036            pid: 41,
14037            start_time: 101,
14038        };
14039        record_deliberate_severance(&snapshot, process).unwrap();
14040        let exit_report = apply_deliberate_severance_marker(
14041            &snapshot,
14042            Some(process),
14043            ExitReport {
14044                kind: ExitKind::Crash,
14045                code: Some(1),
14046                signal: None,
14047                at_ms: 1,
14048            },
14049        );
14050        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14051
14052        assert!(matches!(
14053            on_child_exit(
14054                &spec,
14055                runtime.restart_policy,
14056                &supervisor.registry,
14057                &snapshot,
14058                &runtime.terminal_ring,
14059                &runtime.spawn_events,
14060                &runtime.child_roster,
14061                exit_report,
14062                true,
14063            )
14064            .await,
14065            NextAction::Restart { schedule: _ }
14066        ));
14067        let state = lock_snapshot(&snapshot).unwrap();
14068        assert_eq!(state.lifetime_restarts, 1);
14069        assert_eq!(state.crash_restarts.len(), 0);
14070    }
14071
14072    #[tokio::test]
14073    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14074        let supervisor = Supervisor::new_for_test(
14075            Arc::new(Registry::default()),
14076            RestartPolicy::new(3, Duration::ZERO),
14077        );
14078        let runtime = supervisor.runtime_config();
14079        let spec = ModuleSpec {
14080            module_id: "genuine-crash".to_string(),
14081            program: PathBuf::from("/unused/genuine-crash"),
14082            args: Vec::new(),
14083            env: Vec::new(),
14084            reserved: false,
14085            reserved_prefixes: Vec::new(),
14086            protocol: ModuleProtocol::Subc,
14087            overlap: Default::default(),
14088            carry_containment: false,
14089        };
14090        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14091
14092        assert!(matches!(
14093            on_child_exit(
14094                &spec,
14095                runtime.restart_policy,
14096                &supervisor.registry,
14097                &snapshot,
14098                &runtime.terminal_ring,
14099                &runtime.spawn_events,
14100                &runtime.child_roster,
14101                ExitReport {
14102                    kind: ExitKind::Crash,
14103                    code: Some(1),
14104                    signal: None,
14105                    at_ms: 1,
14106                },
14107                true,
14108            )
14109            .await,
14110            NextAction::Restart { schedule: _ }
14111        ));
14112        let state = lock_snapshot(&snapshot).unwrap();
14113        assert_eq!(state.lifetime_restarts, 1);
14114        assert_eq!(state.crash_restarts.len(), 1);
14115    }
14116
14117    fn crash_exit_report(at_ms: u64) -> ExitReport {
14118        ExitReport {
14119            kind: ExitKind::Crash,
14120            code: Some(1),
14121            signal: None,
14122            at_ms,
14123        }
14124    }
14125
14126    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14127        ModuleSpec {
14128            module_id: module_id.to_string(),
14129            program: PathBuf::from("/unused").join(module_id),
14130            args: Vec::new(),
14131            env: Vec::new(),
14132            reserved: false,
14133            reserved_prefixes: Vec::new(),
14134            protocol: ModuleProtocol::Subc,
14135            overlap: Default::default(),
14136            carry_containment: false,
14137        }
14138    }
14139
14140    /// A real crash loop still stops. Three crashes with nothing aging out spend
14141    /// a budget of two and the third respawn is refused, and both surfaces an
14142    /// operator has -- the log line and the retained terminal record -- name the
14143    /// window rather than only the cap, because `max_restarts=2` alone is what
14144    /// this budget used to mean.
14145    #[tokio::test]
14146    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14147        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14148        let supervisor = Supervisor::new_for_test(
14149            Arc::new(Registry::default()),
14150            RestartPolicy::new(2, Duration::ZERO),
14151        );
14152        let runtime = supervisor.runtime_config();
14153        let spec = windowed_crash_spec("crash-loop-in-window");
14154        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14155
14156        for attempt in 1..=2 {
14157            assert!(
14158                matches!(
14159                    on_child_exit(
14160                        &spec,
14161                        runtime.restart_policy,
14162                        &supervisor.registry,
14163                        &snapshot,
14164                        &runtime.terminal_ring,
14165                        &runtime.spawn_events,
14166                        &runtime.child_roster,
14167                        crash_exit_report(attempt),
14168                        true,
14169                    )
14170                    .await,
14171                    NextAction::Restart { schedule: _ }
14172                ),
14173                "crash {attempt} is inside the budget and must respawn"
14174            );
14175        }
14176
14177        assert!(matches!(
14178            on_child_exit(
14179                &spec,
14180                runtime.restart_policy,
14181                &supervisor.registry,
14182                &snapshot,
14183                &runtime.terminal_ring,
14184                &runtime.spawn_events,
14185                &runtime.child_roster,
14186                crash_exit_report(3),
14187                true,
14188            )
14189            .await,
14190            NextAction::Stop { .. }
14191        ));
14192
14193        {
14194            let state = lock_snapshot(&snapshot).unwrap();
14195            assert_eq!(state.state, ModuleState::Failed);
14196            assert_eq!(state.crash_restarts.len(), 2);
14197            assert_eq!(state.lifetime_restarts, 2);
14198        }
14199
14200        let history = runtime
14201            .terminal_ring
14202            .lock()
14203            .expect("terminal ring is not poisoned")
14204            .snapshot();
14205        let last = history
14206            .entries
14207            .last()
14208            .expect("the refused crash is retained");
14209        assert_eq!(last.disposition, TerminalDisposition::Failed);
14210        assert_eq!(
14211            last.disposition_detail.as_deref(),
14212            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14213        );
14214
14215        let captured = crate::router::test_log::captured_logs(&logs);
14216        assert!(
14217            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14218            "the stop must be logged with its window: {captured}"
14219        );
14220    }
14221
14222    /// The rate, stated as a test: three crashes where the first has aged past
14223    /// the window are two crashes as far as the budget is concerned, so the
14224    /// third respawn is allowed and the ring holds only the two recent ones.
14225    ///
14226    /// This is the case a lifetime counter got wrong -- and the case the daemon
14227    /// now hits routinely, since a module exits non-zero every time its
14228    /// connection to the daemon drops.
14229    #[tokio::test]
14230    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14231        let supervisor = Supervisor::new_for_test(
14232            Arc::new(Registry::default()),
14233            RestartPolicy::new(2, Duration::ZERO),
14234        );
14235        let runtime = supervisor.runtime_config();
14236        let spec = windowed_crash_spec("crash-across-windows");
14237        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14238
14239        for attempt in 1..=2 {
14240            assert!(matches!(
14241                on_child_exit(
14242                    &spec,
14243                    runtime.restart_policy,
14244                    &supervisor.registry,
14245                    &snapshot,
14246                    &runtime.terminal_ring,
14247                    &runtime.spawn_events,
14248                    &runtime.child_roster,
14249                    crash_exit_report(attempt),
14250                    true,
14251                )
14252                .await,
14253                NextAction::Restart { schedule: _ }
14254            ));
14255        }
14256
14257        // The oldest crash moves out of the window; nothing else about the
14258        // module changes.
14259        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14260            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14261        })
14262        .unwrap();
14263
14264        assert!(
14265            matches!(
14266                on_child_exit(
14267                    &spec,
14268                    runtime.restart_policy,
14269                    &supervisor.registry,
14270                    &snapshot,
14271                    &runtime.terminal_ring,
14272                    &runtime.spawn_events,
14273                    &runtime.child_roster,
14274                    crash_exit_report(3),
14275                    true,
14276                )
14277                .await,
14278                NextAction::Restart { schedule: _ }
14279            ),
14280            "a crash older than the window must not hold a budget slot"
14281        );
14282
14283        let state = lock_snapshot(&snapshot).unwrap();
14284        assert_eq!(state.state, ModuleState::Restarting);
14285        assert_eq!(
14286            state.crash_restarts.len(),
14287            2,
14288            "the aged instant is dropped and the new one takes its place"
14289        );
14290        assert_eq!(
14291            state.lifetime_restarts, 3,
14292            "the ledger counts every restart, including the ones the window forgot"
14293        );
14294    }
14295
14296    /// An operator restart hands the budget back whole, and the ledger keeps
14297    /// counting. Those are different questions -- "how close is this module to
14298    /// being stopped" and "how many times has it been replaced" -- and the
14299    /// operator action answers only the first.
14300    #[tokio::test]
14301    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14302        let supervisor = Supervisor::new_for_test(
14303            Arc::new(Registry::default()),
14304            RestartPolicy::new(2, Duration::ZERO),
14305        );
14306        let runtime = supervisor.runtime_config();
14307        let spec = windowed_crash_spec("operator-cleared-budget");
14308        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14309
14310        for attempt in 1..=2 {
14311            assert!(matches!(
14312                on_child_exit(
14313                    &spec,
14314                    runtime.restart_policy,
14315                    &supervisor.registry,
14316                    &snapshot,
14317                    &runtime.terminal_ring,
14318                    &runtime.spawn_events,
14319                    &runtime.child_roster,
14320                    crash_exit_report(attempt),
14321                    true,
14322                )
14323                .await,
14324                NextAction::Restart { schedule: _ }
14325            ));
14326        }
14327
14328        reset_restart_count(&snapshot, &spec.module_id).unwrap();
14329        {
14330            let state = lock_snapshot(&snapshot).unwrap();
14331            assert!(
14332                state.crash_restarts.is_empty(),
14333                "an operator restart returns the full budget"
14334            );
14335            assert_eq!(
14336                state.lifetime_restarts, 2,
14337                "clearing the budget must not unmake the crashes"
14338            );
14339        }
14340
14341        assert!(
14342            matches!(
14343                on_child_exit(
14344                    &spec,
14345                    runtime.restart_policy,
14346                    &supervisor.registry,
14347                    &snapshot,
14348                    &runtime.terminal_ring,
14349                    &runtime.spawn_events,
14350                    &runtime.child_roster,
14351                    crash_exit_report(3),
14352                    true,
14353                )
14354                .await,
14355                NextAction::Restart { schedule: _ }
14356            ),
14357            "the cleared budget must be spendable again"
14358        );
14359        let state = lock_snapshot(&snapshot).unwrap();
14360        assert_eq!(state.crash_restarts.len(), 1);
14361        assert_eq!(state.lifetime_restarts, 3);
14362    }
14363
14364    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14365    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14366        let severed = ProcessIdentity {
14367            pid: 41,
14368            start_time: 101,
14369        };
14370        let successor = ProcessIdentity {
14371            pid: 41,
14372            start_time: 202,
14373        };
14374        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14375        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14376            state.pid = Some(successor.pid);
14377            state.process_start_time = Some(successor.start_time);
14378        })
14379        .unwrap();
14380        assert!(!module.record_deliberate_severance(severed).unwrap());
14381
14382        let exit_report = apply_deliberate_severance_marker(
14383            &module.inner.snapshot,
14384            Some(successor),
14385            ExitReport {
14386                kind: ExitKind::Crash,
14387                code: Some(1),
14388                signal: None,
14389                at_ms: 1,
14390            },
14391        );
14392
14393        assert_eq!(exit_report.kind, ExitKind::Crash);
14394    }
14395
14396    #[tokio::test]
14397    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14398        let registry = Registry::default();
14399        let supervisor = Supervisor::new_for_test(
14400            Arc::new(Registry::default()),
14401            RestartPolicy::new(3, Duration::ZERO),
14402        );
14403        let runtime = supervisor.runtime_config();
14404        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14405        let spec = ModuleSpec {
14406            module_id: "drain-deliberate-severance".to_string(),
14407            program: fake_aft_stub_path(),
14408            args: Vec::new(),
14409            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14410            reserved: false,
14411            reserved_prefixes: Vec::new(),
14412            protocol: ModuleProtocol::Subc,
14413            overlap: Default::default(),
14414            carry_containment: false,
14415        };
14416        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14417        let process = ProcessIdentity {
14418            pid: 41,
14419            start_time: 101,
14420        };
14421        child.process_identity = Some(process);
14422        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14423            state.pid = Some(process.pid);
14424            state.process_start_time = Some(process.start_time);
14425        })
14426        .unwrap();
14427        record_deliberate_severance(&snapshot, process).unwrap();
14428
14429        drain_child_to_state(
14430            &spec.module_id,
14431            spec.protocol,
14432            // The child exits on its own; no signal may change the exit this
14433            // test classifies.
14434            StopNotice::SentOverConnection,
14435            &registry,
14436            None,
14437            &snapshot,
14438            &runtime.terminal_ring,
14439            &runtime.spawn_events,
14440            child,
14441            Duration::from_secs(1),
14442            ModuleState::Stopped,
14443            Some(false),
14444            &super::FailureContext::operator(),
14445        )
14446        .await
14447        .unwrap();
14448
14449        let state = lock_snapshot(&snapshot).unwrap();
14450        assert_eq!(
14451            state.last_exit.as_ref().map(|exit| exit.kind),
14452            Some(ExitKind::DeliberateSeverance)
14453        );
14454        assert_eq!(state.lifetime_restarts, 1);
14455        assert_eq!(state.crash_restarts.len(), 0);
14456        drop(state);
14457        let history = runtime.terminal_ring.lock().unwrap().snapshot();
14458        assert_eq!(
14459            history.entries[0].exit_kind,
14460            subc_control::TerminalExitKind::DeliberateSeverance
14461        );
14462    }
14463
14464    #[tokio::test]
14465    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14466        let registry = Registry::default();
14467        let supervisor = Supervisor::new_for_test(
14468            Arc::new(Registry::default()),
14469            RestartPolicy::new(3, Duration::ZERO),
14470        );
14471        let runtime = supervisor.runtime_config();
14472        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14473        let spec = ModuleSpec {
14474            module_id: "ordinary-drain".to_string(),
14475            program: fake_aft_stub_path(),
14476            args: Vec::new(),
14477            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14478            reserved: false,
14479            reserved_prefixes: Vec::new(),
14480            protocol: ModuleProtocol::Subc,
14481            overlap: Default::default(),
14482            carry_containment: false,
14483        };
14484        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14485
14486        drain_child_to_state(
14487            &spec.module_id,
14488            spec.protocol,
14489            // The child exits on its own; no signal may change the exit this
14490            // test classifies.
14491            StopNotice::SentOverConnection,
14492            &registry,
14493            None,
14494            &snapshot,
14495            &runtime.terminal_ring,
14496            &runtime.spawn_events,
14497            child,
14498            Duration::from_secs(1),
14499            ModuleState::Stopped,
14500            Some(false),
14501            &super::FailureContext::operator(),
14502        )
14503        .await
14504        .unwrap();
14505
14506        let state = lock_snapshot(&snapshot).unwrap();
14507        assert_eq!(
14508            state.last_exit.as_ref().map(|exit| exit.kind),
14509            Some(ExitKind::Crash)
14510        );
14511        assert_eq!(state.lifetime_restarts, 0);
14512        assert_eq!(state.crash_restarts.len(), 0);
14513    }
14514
14515    #[test]
14516    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14517        // The server's generic fatal-routing branch only knows that the
14518        // connection failed; it does not know that the daemon deliberately
14519        // initiated a process-killing severance. Keep this seam explicit so a
14520        // future connection error path cannot silently reintroduce the stale
14521        // exemption that mislabels a later genuine crash.
14522        assert!(!include_str!("server.rs")
14523            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14524    }
14525
14526    /// The `route.closed` `drained` value must be the quiescence wait's own
14527    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
14528    /// measurement at all and `false` is the one honest constant. This is the exact
14529    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
14530    /// on every return path, including the one that used to return early via `?`
14531    /// with `route.closing` already sent and no `route.closed` ever following.
14532    #[test]
14533    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14534        assert!(drained_after_quiescence_wait(&Ok(true)));
14535        assert!(!drained_after_quiescence_wait(&Ok(false)));
14536        assert!(!drained_after_quiescence_wait(&Err(
14537            SuperviseError::StatePoisoned { module_id: None }
14538        )));
14539    }
14540
14541    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
14542    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
14543    /// already reaped out-of-band) still leaves a terminal record rather than none
14544    /// at all. Triggering the real `wait()` I/O error from an integration test would
14545    /// need a genuine already-reaped-child race, which is OS-specific and not
14546    /// something this suite attempts elsewhere; this test instead verifies the
14547    /// record produced for that arm end-to-end through the real `TerminalRing`, and
14548    /// the call site itself is verified by inspection to sit in that exact arm.
14549    #[test]
14550    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14551        let ring = Arc::new(Mutex::new(TerminalRing::new(
14552            TerminalRingConfig::default(),
14553            0,
14554        )));
14555        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14556
14557        let snapshot = ring.lock().unwrap().snapshot();
14558        assert_eq!(snapshot.entries.len(), 1);
14559        let entry = &snapshot.entries[0];
14560        assert_eq!(entry.exit_code, None);
14561        assert_eq!(entry.exit_signal, None);
14562        assert_eq!(entry.disposition, TerminalDisposition::Failed);
14563    }
14564
14565    #[test]
14566    fn wait_error_exit_path_preserves_spawn_event_density() {
14567        let feed = super::SpawnEventFeed::default();
14568        feed.configure_incarnation("wait-error-density".to_string());
14569        feed.emit_spawned("wait-error", 41, 1);
14570        let ring = Arc::new(Mutex::new(TerminalRing::new(
14571            TerminalRingConfig::default(),
14572            0,
14573        )));
14574
14575        record_wait_error_terminal("wait-error", &ring, &feed);
14576        feed.emit_spawned("after-wait-error", 42, 2);
14577
14578        let state = feed.0.lock().unwrap();
14579        let sequences = state
14580            .events
14581            .iter()
14582            .map(|event| event.cursor.seq)
14583            .collect::<Vec<_>>();
14584        assert_eq!(sequences, vec![1, 2, 3]);
14585        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14586        assert_eq!(state.events[1].exit_code, None);
14587        assert_eq!(state.events[1].exit_signal, None);
14588    }
14589
14590    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
14591    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
14592    /// not a clean exit it never actually observed.
14593    #[test]
14594    fn wait_error_exit_report_is_classified_as_a_crash() {
14595        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14596    }
14597}
14598
14599#[cfg(all(test, unix))]
14600mod restart_ordering_tests {
14601    use super::*;
14602    use std::io::Write;
14603    use tokio::io::AsyncReadExt;
14604
14605    #[test]
14606    #[ignore = "child process fixture"]
14607    fn serving_child() {
14608        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14609            return;
14610        };
14611        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14612        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14613        for stream in listener.incoming() {
14614            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14615        }
14616    }
14617
14618    async fn restart_before_publication(requests: usize) {
14619        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14620        let address_file = home.path().join("address");
14621        let spec = ModuleSpec {
14622            module_id: "restart-serving-child".into(),
14623            program: std::env::current_exe().unwrap(),
14624            args: vec![
14625                "--exact".into(),
14626                "supervise::restart_ordering_tests::serving_child".into(),
14627                "--ignored".into(),
14628                "--nocapture".into(),
14629            ],
14630            env: vec![(
14631                "RESTART_TEST_ADDRESS_FILE".into(),
14632                address_file.display().to_string(),
14633            )],
14634            reserved: false,
14635            reserved_prefixes: vec![],
14636            protocol: ModuleProtocol::None,
14637            overlap: Default::default(),
14638            carry_containment: false,
14639        };
14640        let mut supervisor =
14641            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14642        let (spawned_tx, spawned_rx) = oneshot::channel();
14643        let spawned_tx = Mutex::new(Some(spawned_tx));
14644        let (release_tx, release_rx) = std::sync::mpsc::channel();
14645        let release_rx = Mutex::new(release_rx);
14646        // Hold publication, not the child: it must answer a real request before
14647        // the restart enters the queue, even though status still has generation 0.
14648        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14649            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14650            release_rx
14651                .lock()
14652                .unwrap()
14653                .recv_timeout(Duration::from_secs(20))
14654                .unwrap();
14655        })));
14656        let registered = supervisor
14657            .register_configured_with_health(
14658                spec.clone(),
14659                true,
14660                HealthConfig::default(),
14661                None,
14662                RestartPolicy::new(3, Duration::ZERO),
14663                DEFAULT_REGISTER_TIMEOUT,
14664            )
14665            .unwrap();
14666        let module = registered.module.clone();
14667        let startup = tokio::task::spawn_blocking(move || {
14668            supervisor.start_registered_configured(registered, None, true)
14669        });
14670        spawned_rx.await.unwrap();
14671        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14672            let address = loop {
14673                if let Ok(address) = std::fs::read_to_string(&address_file) {
14674                    if !address.is_empty() {
14675                        break address;
14676                    }
14677                }
14678                tokio::time::sleep(Duration::from_millis(1)).await;
14679            };
14680            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14681            let mut reply = String::new();
14682            stream.read_to_string(&mut reply).await.unwrap();
14683            reply.trim().parse::<u32>().unwrap()
14684        })
14685        .await
14686        .expect("child must serve before generation publication");
14687        assert_eq!(
14688            lock_snapshot(&module.inner.snapshot)
14689                .unwrap()
14690                .spawn_generation,
14691            0
14692        );
14693        let mut replies = Vec::new();
14694        for _ in 0..requests {
14695            let mut restart = Box::pin(module.restart(Some(0)));
14696            // Poll the public API through queue admission, leaving only its ACK
14697            // pending. The monitor cannot consume the command until released.
14698            std::future::poll_fn(|cx| {
14699                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14700                std::task::Poll::Ready(())
14701            })
14702            .await;
14703            replies.push(restart);
14704        }
14705        release_tx.send(()).unwrap();
14706        startup.await.unwrap().unwrap();
14707        for reply in replies {
14708            reply.await.unwrap();
14709        }
14710        // Restart schedules its replacement after completing the handler. Wait
14711        // for that separate actor turn before placing the command barrier.
14712        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14713            loop {
14714                if lock_snapshot(&module.inner.snapshot)
14715                    .unwrap()
14716                    .spawn_generation
14717                    > 1
14718                {
14719                    break;
14720                }
14721                tokio::time::sleep(Duration::from_millis(1)).await;
14722            }
14723        })
14724        .await;
14725        // Commands are handled serially. The update-spec reply proves every
14726        // restart handler has finished before we inspect the snapshot; restart's
14727        // own ACK only acknowledges initiation, not replacement completion.
14728        module.update_spec_for_test(spec).await.unwrap();
14729        let (generation, pid) = {
14730            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14731            (state.spawn_generation, state.pid)
14732        };
14733        module.stop().await.unwrap();
14734        replacement.expect("restart must publish a replacement for the already-serving child");
14735        assert_ne!(
14736            pid,
14737            Some(old_pid),
14738            "a serving child predating the restart must be replaced"
14739        );
14740        assert_eq!(generation, 2, "queued restarts must share one replacement");
14741    }
14742
14743    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14744    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14745        restart_before_publication(1).await;
14746    }
14747
14748    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14749    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14750        restart_before_publication(2).await;
14751    }
14752
14753    #[tokio::test]
14754    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14755        let supervisor =
14756            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14757        let registered = supervisor
14758            .register_configured_with_health(
14759                ModuleSpec {
14760                    module_id: "queued-before-replacement".into(),
14761                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14762                    // macOS a module whose running image is not the configured
14763                    // program is refused once its exec is confirmed (see
14764                    // `record_write_stall_tests::sleeping_spec`).
14765                    program: "/bin/sleep".into(),
14766                    args: vec!["600".into()],
14767                    env: vec![],
14768                    reserved: false,
14769                    reserved_prefixes: vec![],
14770                    protocol: ModuleProtocol::None,
14771                    overlap: Default::default(),
14772                    carry_containment: false,
14773                },
14774                true,
14775                HealthConfig::default(),
14776                None,
14777                RestartPolicy::default(),
14778                DEFAULT_REGISTER_TIMEOUT,
14779            )
14780            .unwrap();
14781        let RegisteredConfiguredModule {
14782            mut spec,
14783            mut runtime,
14784            module,
14785            mut commands,
14786        } = registered;
14787        let snapshot = &module.inner.snapshot;
14788        let mut restart = Box::pin(module.restart(Some(0)));
14789        std::future::poll_fn(|cx| {
14790            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14791            std::task::Poll::Ready(())
14792        })
14793        .await;
14794        // Drive the actor explicitly so a replacement is definitely spawned
14795        // after receipt, but before this queued command is dequeued.
14796        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14797        let replacement_pid = child.as_ref().unwrap().id();
14798        let command = commands.recv().await.unwrap();
14799        assert!(
14800            handle_supervisor_command(
14801                command,
14802                &mut spec,
14803                &mut runtime,
14804                &supervisor.registry,
14805                &supervisor.process_liveness,
14806                snapshot,
14807                &mut child,
14808                &mut commands,
14809                &mut VecDeque::new(),
14810            )
14811            .await
14812        );
14813        restart.await.unwrap();
14814        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14815        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14816        if let Some(mut child) = child {
14817            child.start_kill().unwrap();
14818            child.wait().await.unwrap();
14819        }
14820        assert_eq!(
14821            kept_pid, replacement_pid,
14822            "a newer replacement must survive"
14823        );
14824        assert!(
14825            !pending,
14826            "a satisfied restart must not schedule another spawn"
14827        );
14828    }
14829}
14830
14831#[cfg(all(test, unix))]
14832mod health_event_tests {
14833    use super::*;
14834    use cortexkit_test_support::ScratchDir;
14835    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14836
14837    struct Actor {
14838        module: SupervisedModule,
14839        registry: Arc<Registry>,
14840        forwarding: Arc<ForwardingTable>,
14841        spec: ModuleSpec,
14842        health: HealthConfig,
14843        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14844        roster: ChildRoster,
14845        _home: ScratchDir,
14846    }
14847
14848    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14849        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14850        // for an observed condition, rather than letting paused time auto-advance
14851        // deadlines or assuming a fixed number of yields finishes the work.
14852        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14853        let tick = Instant::now();
14854        loop {
14855            if let Some(value) = observe() {
14856                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14857                return value;
14858            }
14859            assert!(
14860                std::time::Instant::now() < deadline,
14861                "timed out waiting for {reason}"
14862            );
14863            tokio::task::yield_now().await;
14864        }
14865    }
14866
14867    impl Actor {
14868        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14869            Self::start_with_policy(
14870                protocol,
14871                cadence,
14872                DEFAULT_REGISTER_TIMEOUT,
14873                RestartPolicy::new(3, Duration::ZERO),
14874            )
14875            .await
14876        }
14877
14878        async fn start_with_policy(
14879            protocol: ModuleProtocol,
14880            cadence: Duration,
14881            register_timeout: Duration,
14882            policy: RestartPolicy,
14883        ) -> Self {
14884            let home = ScratchDir::new("health-events");
14885            let registry = Arc::new(Registry::default());
14886            let forwarding = Arc::new(ForwardingTable::default());
14887            let health = HealthConfig {
14888                cadence,
14889                deadline: Duration::from_secs(3600),
14890                ..HealthConfig::default()
14891            };
14892            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14893                .with_forwarding(forwarding.clone())
14894                .with_health_config(health.clone());
14895            let spec = ModuleSpec {
14896                module_id: "event-health-child".into(),
14897                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14898                // the privacy trampoline refuses a module whose running image
14899                // is not the configured program, and `start` below waits for
14900                // the process to be confirmed (`reported_pid`). The shell form
14901                // passed only while the image check ran before the shell had
14902                // replaced itself with `sleep` (see
14903                // `record_write_stall_tests::sleeping_spec`).
14904                program: PathBuf::from("/bin/sleep"),
14905                // The process stays alive but has no real bus peer. Registrations
14906                // below exercise the same registry writes as accepted HELLOs.
14907                args: vec!["600".into()],
14908                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14909                    .into_iter()
14910                    .map(|key| (key.into(), home.path().display().to_string()))
14911                    .collect(),
14912                reserved: false,
14913                reserved_prefixes: vec![],
14914                protocol,
14915                overlap: Default::default(),
14916                carry_containment: false,
14917            };
14918            let module = supervisor
14919                .supervise_configured_requested(
14920                    spec.clone(),
14921                    true,
14922                    health.clone(),
14923                    Some(0),
14924                    policy,
14925                    register_timeout,
14926                    None,
14927                )
14928                .unwrap();
14929            let (_, rx) = mpsc::channel(8);
14930            let actor = Self {
14931                module,
14932                registry,
14933                forwarding,
14934                spec,
14935                health,
14936                rx,
14937                roster: supervisor.child_roster.clone(),
14938                _home: home,
14939            };
14940            actor
14941                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14942                .await;
14943            actor
14944        }
14945
14946        async fn expire_registration_wait(&self, remaining: Duration) {
14947            tokio::time::advance(remaining).await;
14948            wait_for("registration timeout TERM grace", || {
14949                lock_snapshot(&self.module.inner.snapshot)
14950                    .unwrap()
14951                    .actor_select
14952                    .is_none()
14953                    .then_some(())
14954            })
14955            .await;
14956            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14957        }
14958
14959        fn turns(&self) -> u64 {
14960            lock_snapshot(&self.module.inner.snapshot)
14961                .unwrap()
14962                .actor_turns
14963        }
14964
14965        async fn wait_for_select(
14966            &self,
14967            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14968        ) -> ActorSelectCheckpoint {
14969            wait_for("actor select checkpoint", || {
14970                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14971                state.actor_select.filter(|checkpoint| {
14972                    checkpoint.generation == state.spawn_generation
14973                        && state.state == ModuleState::Running
14974                        && state.process_alive
14975                        && state.reported_pid().is_some()
14976                        && expected(checkpoint)
14977                })
14978            })
14979            .await
14980        }
14981
14982        fn hello(&mut self, connection: u64, health: bool) {
14983            let manifest =
14984                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14985                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14986                    .build();
14987            let connection = ConnectionId::new(connection);
14988            let (tx, rx) = mpsc::channel(8);
14989            self.rx = rx;
14990            self.forwarding
14991                .register_module_connection(
14992                    connection,
14993                    self.spec.module_id.clone(),
14994                    subc_protocol::PROTOCOL_VERSION,
14995                    Concurrency::ModuleManaged,
14996                    FrameSink::new(tx),
14997                )
14998                .unwrap();
14999            self.registry
15000                .register_with_control_ops(
15001                    manifest,
15002                    subc_protocol::PROTOCOL_VERSION,
15003                    connection,
15004                    if health {
15005                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15006                    } else {
15007                        vec![]
15008                    },
15009                )
15010                .unwrap();
15011        }
15012
15013        async fn probe(&mut self) -> crate::router::OutboundFrame {
15014            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15015                Ok(frame) => Some(frame),
15016                Err(mpsc::error::TryRecvError::Empty) => None,
15017                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15018            })
15019            .await;
15020            let body: Value = serde_json::from_slice(&frame.body).unwrap();
15021            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15022            frame
15023        }
15024
15025        fn no_probe(&mut self) {
15026            assert!(matches!(
15027                self.rx.try_recv(),
15028                Err(mpsc::error::TryRecvError::Empty)
15029            ));
15030        }
15031
15032        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15033            self.forwarding
15034                .complete_module_control_rpc(
15035                    ConnectionId::new(connection),
15036                    frame.header.corr,
15037                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15038                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15039                        status: HealthStatus::Ok,
15040                        detail: None,
15041                        metrics: None,
15042                    }),
15043                )
15044                .unwrap();
15045        }
15046    }
15047
15048    #[tokio::test(start_paused = true)]
15049    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15050        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15051        let initial = actor.module.status().unwrap();
15052        tokio::time::advance(Duration::from_secs(119)).await;
15053        tokio::task::yield_now().await;
15054        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15055        assert!(actor.module.terminal_history().entries.is_empty());
15056        actor.expire_registration_wait(Duration::from_secs(1)).await;
15057        actor
15058            .wait_for_select(|at| at.generation > initial.spawn_generation)
15059            .await;
15060        let status = actor.module.status().unwrap();
15061        assert_ne!(status.pid, initial.pid);
15062        assert!(!cortexkit_test_support::process_alive(
15063            initial.pid.unwrap() as i32
15064        ));
15065        assert_eq!(status.restart_count, 1);
15066        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15067        let entries = actor.module.terminal_history().entries;
15068        assert_eq!(entries.len(), 1);
15069        assert_eq!(
15070            entries[0].disposition_detail.as_deref(),
15071            Some("registration_timeout")
15072        );
15073        actor.module.stop().await.unwrap();
15074    }
15075
15076    #[tokio::test(start_paused = true)]
15077    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15078        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15079        let pid = actor.module.status().unwrap().pid;
15080        tokio::time::advance(Duration::from_secs(119)).await;
15081        actor.hello(1, false);
15082        let registered = actor
15083            .wait_for_select(|at| at.registered_connection.is_some())
15084            .await;
15085        assert_eq!(
15086            registered.wake_after, None,
15087            "HELLO must cancel the deadline"
15088        );
15089        tokio::time::advance(Duration::from_secs(300)).await;
15090        tokio::task::yield_now().await;
15091        assert_eq!(actor.module.status().unwrap().pid, pid);
15092        assert!(actor.module.terminal_history().entries.is_empty());
15093        // A disconnect after the first HELLO must not re-arm a startup timer.
15094        actor
15095            .registry
15096            .deregister_connection(ConnectionId::new(1))
15097            .unwrap();
15098        actor
15099            .wait_for_select(|at| at.registered_connection.is_none())
15100            .await;
15101        tokio::time::advance(Duration::from_secs(300)).await;
15102        tokio::task::yield_now().await;
15103        assert_eq!(actor.module.status().unwrap().pid, pid);
15104        assert!(actor.module.terminal_history().entries.is_empty());
15105        actor.module.stop().await.unwrap();
15106
15107        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15108        let pid = transient.module.status().unwrap().pid;
15109        tokio::time::advance(Duration::from_secs(119)).await;
15110        let turns = transient.turns();
15111        // Both writes happen before the supervisor can refresh the registry.
15112        // An accepted HELLO must remain evidence after its connection is gone.
15113        transient.hello(1, false);
15114        transient
15115            .registry
15116            .deregister_connection(ConnectionId::new(1))
15117            .unwrap();
15118        let parked = transient.wait_for_select(|at| at.turn > turns).await;
15119        assert_eq!(
15120            parked.wake_after, None,
15121            "transient HELLO must cancel the deadline"
15122        );
15123        tokio::time::advance(Duration::from_secs(300)).await;
15124        tokio::task::yield_now().await;
15125        assert_eq!(transient.module.status().unwrap().pid, pid);
15126        assert!(transient.module.terminal_history().entries.is_empty());
15127        transient.module.stop().await.unwrap();
15128    }
15129
15130    #[tokio::test(start_paused = true)]
15131    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15132        let actor = Actor::start_with_policy(
15133            ModuleProtocol::Subc,
15134            Duration::from_secs(30),
15135            Duration::from_secs(10),
15136            RestartPolicy::new(1, Duration::ZERO),
15137        )
15138        .await;
15139        let generation = actor.module.status().unwrap().spawn_generation;
15140        actor
15141            .expire_registration_wait(Duration::from_secs(10))
15142            .await;
15143        actor.wait_for_select(|at| at.generation > generation).await;
15144        actor
15145            .expire_registration_wait(Duration::from_secs(10))
15146            .await;
15147        let retry = wait_for("failed retry schedule", || {
15148            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15149            (state.state == ModuleState::Failed)
15150                .then_some(state.failed_retry.pending)
15151                .flatten()
15152        })
15153        .await;
15154        assert_eq!(retry.delay, Duration::from_secs(600));
15155        assert_eq!(retry.attempt, 1);
15156        let entries = actor.module.terminal_history().entries;
15157        assert_eq!(entries.len(), 2);
15158        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15159        assert!(entries[1]
15160            .disposition_detail
15161            .as_deref()
15162            .unwrap()
15163            .contains("registration_timeout"));
15164        tokio::time::advance(Duration::from_secs(599)).await;
15165        tokio::task::yield_now().await;
15166        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15167        tokio::time::advance(Duration::from_secs(1)).await;
15168        actor
15169            .wait_for_select(|at| at.generation > generation + 1)
15170            .await;
15171        // The slow retry is another ordinary spawn, with its own HELLO budget.
15172        actor
15173            .expire_registration_wait(Duration::from_secs(10))
15174            .await;
15175        wait_for("slow retry registration timeout", || {
15176            (actor.module.terminal_history().entries.len() == 3).then_some(())
15177        })
15178        .await;
15179        actor.module.stop().await.unwrap();
15180    }
15181
15182    #[tokio::test(start_paused = true)]
15183    async fn registration_timeout_honours_ten_second_override() {
15184        let actor = Actor::start_with_policy(
15185            ModuleProtocol::Subc,
15186            Duration::from_secs(30),
15187            Duration::from_secs(10),
15188            RestartPolicy::new(3, Duration::ZERO),
15189        )
15190        .await;
15191        let generation = actor.module.status().unwrap().spawn_generation;
15192        tokio::time::advance(Duration::from_secs(9)).await;
15193        tokio::task::yield_now().await;
15194        assert!(actor.module.terminal_history().entries.is_empty());
15195        actor.expire_registration_wait(Duration::from_secs(1)).await;
15196        actor.wait_for_select(|at| at.generation > generation).await;
15197        assert_eq!(actor.module.terminal_history().entries.len(), 1);
15198        actor.module.stop().await.unwrap();
15199    }
15200
15201    #[tokio::test]
15202    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15203        use std::os::unix::process::{CommandExt, ExitStatusExt};
15204
15205        struct Helper(std::process::Child);
15206        impl Drop for Helper {
15207            fn drop(&mut self) {
15208                // This handle still owns the helper until wait returns. Never
15209                // clean up its old process group after the leader is reaped.
15210                if matches!(self.0.try_wait(), Ok(None)) {
15211                    let _ = self.0.kill();
15212                    let _ = self.0.wait();
15213                }
15214            }
15215        }
15216
15217        let home = ScratchDir::new("registration-timeout-helper");
15218        let supervisor = Supervisor::new_for_test(
15219            Arc::new(Registry::default()),
15220            RestartPolicy::new(0, Duration::ZERO),
15221        );
15222        let spec = ModuleSpec {
15223            module_id: "registration-timeout-group".into(),
15224            program: "/bin/sleep".into(),
15225            args: vec!["120".into()],
15226            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15227                .into_iter()
15228                .map(|key| (key.into(), home.path().display().to_string()))
15229                .collect(),
15230            reserved: false,
15231            reserved_prefixes: vec![],
15232            protocol: ModuleProtocol::Subc,
15233            overlap: Default::default(),
15234            carry_containment: false,
15235        };
15236        let module = supervisor
15237            .supervise_configured_requested(
15238                spec,
15239                true,
15240                HealthConfig::default(),
15241                None,
15242                RestartPolicy::new(0, Duration::ZERO),
15243                Duration::from_secs(10),
15244                None,
15245            )
15246            .unwrap();
15247        let leader = timeout(Duration::from_secs(5), async {
15248            loop {
15249                if let Some(pid) = module.status().unwrap().pid {
15250                    break pid;
15251                }
15252                sleep(Duration::from_millis(5)).await;
15253            }
15254        })
15255        .await
15256        .expect("supervised group leader must be reported before the deadline");
15257        assert!(leader > 1);
15258        let ready = home.join("helper.ready");
15259        // Own the helper separately so even a broken group kill can be cleaned
15260        // up by its own unreaped child handle, without signalling a stale pgid.
15261        let mut helper = Helper(
15262            std::process::Command::new("/bin/sh")
15263                .args([
15264                    "-c",
15265                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15266                    "helper",
15267                ])
15268                .arg(&ready)
15269                .process_group(i32::try_from(leader).unwrap())
15270                .stdout(std::process::Stdio::null())
15271                .stderr(std::process::Stdio::null())
15272                .spawn()
15273                .unwrap(),
15274        );
15275        timeout(Duration::from_secs(5), async {
15276            while !ready.exists() {
15277                assert!(helper.0.try_wait().unwrap().is_none());
15278                sleep(Duration::from_millis(5)).await;
15279            }
15280        })
15281        .await
15282        .expect("helper must install TERM ignore before the registration deadline");
15283        let helper_pid =
15284            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15285        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15286        sleep(Duration::from_millis(20)).await;
15287        assert!(
15288            helper.0.try_wait().unwrap().is_none(),
15289            "helper must ignore TERM"
15290        );
15291
15292        timeout(Duration::from_secs(20), async {
15293            while module.state().unwrap() != ModuleState::Failed {
15294                sleep(Duration::from_millis(5)).await;
15295            }
15296        })
15297        .await
15298        .expect("registration timeout must stop and reap the group leader");
15299        let helper_exit = timeout(Duration::from_secs(3), async {
15300            loop {
15301                if let Some(status) = helper.0.try_wait().unwrap() {
15302                    break status;
15303                }
15304                sleep(Duration::from_millis(5)).await;
15305            }
15306        })
15307        .await
15308        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15309        assert_eq!(helper_exit.signal(), Some(9));
15310        assert!(!cortexkit_test_support::process_alive(
15311            i32::try_from(leader).unwrap()
15312        ));
15313        assert_eq!(
15314            module.status().unwrap().last_exit.unwrap().kind,
15315            ExitKind::Crash
15316        );
15317        assert!(module.terminal_history().entries[0]
15318            .disposition_detail
15319            .as_deref()
15320            .unwrap()
15321            .contains("registration_timeout"));
15322        module.stop().await.unwrap();
15323    }
15324
15325    #[tokio::test(start_paused = true)]
15326    async fn registration_timeout_does_not_count_an_earlier_exit() {
15327        let actor = Actor::start_with_policy(
15328            ModuleProtocol::Subc,
15329            Duration::from_secs(30),
15330            DEFAULT_REGISTER_TIMEOUT,
15331            RestartPolicy::new(0, Duration::ZERO),
15332        )
15333        .await;
15334        let pid = actor.module.status().unwrap().pid.unwrap();
15335        rustix::process::kill_process(
15336            rustix::process::Pid::from_raw(pid as i32).unwrap(),
15337            rustix::process::Signal::TERM,
15338        )
15339        .unwrap();
15340        wait_for("early exit", || {
15341            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15342        })
15343        .await;
15344        tokio::time::advance(Duration::from_secs(121)).await;
15345        tokio::task::yield_now().await;
15346        let entries = actor.module.terminal_history().entries;
15347        assert_eq!(entries.len(), 1);
15348        assert!(!entries[0]
15349            .disposition_detail
15350            .as_deref()
15351            .unwrap()
15352            .contains("registration_timeout"));
15353        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15354        actor.module.stop().await.unwrap();
15355    }
15356
15357    #[tokio::test(start_paused = true)]
15358    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15359        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15360        let pid = actor.module.status().unwrap().pid;
15361        tokio::time::advance(Duration::from_secs(119)).await;
15362        actor.roster.close();
15363        let turns = actor.turns();
15364        tokio::time::advance(Duration::from_secs(300)).await;
15365        let parked = actor.wait_for_select(|at| at.turn > turns).await;
15366        assert_eq!(parked.wake_after, None);
15367        assert_eq!(actor.module.status().unwrap().pid, pid);
15368        assert!(actor.module.terminal_history().entries.is_empty());
15369        actor.module.stop().await.unwrap();
15370        let status = actor.module.status().unwrap();
15371        assert_eq!(status.restart_count, 0);
15372        assert_eq!(
15373            actor.module.terminal_history().entries[0].disposition,
15374            TerminalDisposition::DaemonShutdown
15375        );
15376    }
15377
15378    #[tokio::test(start_paused = true)]
15379    async fn no_health_children_park_for_sixty_seconds() {
15380        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15381        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15382        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15383        unadvertised.hello(1, false);
15384        // Include any one-off registration wake in the budget; receiving a
15385        // no-health HELLO must not turn a parked child into a polling child.
15386        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
15387        // a single 60 s jump would only observe one expired timer.
15388        for _ in 0..6000 {
15389            tokio::time::advance(Duration::from_millis(10)).await;
15390            tokio::task::yield_now().await;
15391        }
15392        for actor in [&none, &unregistered, &unadvertised] {
15393            assert!(
15394                actor.turns() <= 5,
15395                "no-health actor woke {} times",
15396                actor.turns()
15397            );
15398            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15399        }
15400    }
15401
15402    #[tokio::test(start_paused = true)]
15403    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15404        // With a zero cadence the first probe is due at once, so this measures
15405        // only how fast the registration notification starts health probing,
15406        // not the normal 30-33 s wait before a first probe (checked below).
15407        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15408        let tick = Instant::now();
15409        actor.hello(1, true);
15410        actor.probe().await;
15411        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15412    }
15413
15414    #[tokio::test(start_paused = true)]
15415    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15416        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15417        let hello_tick = Instant::now();
15418        actor.hello(1, true);
15419        let armed = actor
15420            .wait_for_select(|checkpoint| {
15421                checkpoint.registered_connection == Some(ConnectionId::new(1))
15422                    && checkpoint.next_probe_at.is_some()
15423            })
15424            .await;
15425        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15426        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15427        let first_deadline = hello_tick + delay;
15428        assert_eq!(armed.next_probe_at, Some(first_deadline));
15429        tokio::time::advance(Duration::from_secs(29)).await;
15430        let turns = actor.turns();
15431        actor
15432            .registry
15433            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15434            .unwrap();
15435        let updated = actor
15436            .wait_for_select(|checkpoint| checkpoint.turn > turns)
15437            .await;
15438        assert_eq!(
15439            updated.next_probe_at,
15440            Some(first_deadline),
15441            "catalog update must not reset cadence"
15442        );
15443        actor.no_probe();
15444        tokio::time::advance(first_deadline - Instant::now()).await;
15445        let frame = actor.probe().await;
15446        actor.answer(1, frame);
15447        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15448        let next_deadline = Instant::now() + next;
15449        let rearmed = actor
15450            .wait_for_select(|checkpoint| {
15451                checkpoint
15452                    .next_probe_at
15453                    .is_some_and(|deadline| deadline > first_deadline)
15454            })
15455            .await;
15456        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15457        tokio::time::advance(next - Duration::from_nanos(1)).await;
15458        actor.no_probe();
15459        tokio::time::advance(Duration::from_nanos(1)).await;
15460        actor.probe().await;
15461    }
15462
15463    #[tokio::test(start_paused = true)]
15464    async fn disconnect_disarms_and_reconnect_rearms_health() {
15465        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15466        actor.hello(1, true);
15467        actor
15468            .wait_for_select(|checkpoint| {
15469                checkpoint.registered_connection == Some(ConnectionId::new(1))
15470                    && checkpoint.next_probe_at.is_some()
15471            })
15472            .await;
15473        let turns = actor.turns();
15474        let tick = Instant::now();
15475        actor
15476            .registry
15477            .deregister_connection(ConnectionId::new(1))
15478            .unwrap();
15479        let disarmed = actor
15480            .wait_for_select(|checkpoint| {
15481                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15482            })
15483            .await;
15484        assert_eq!(disarmed.next_probe_at, None);
15485        assert_eq!(disarmed.wake_after, None);
15486        assert_eq!(
15487            Instant::now(),
15488            tick,
15489            "disconnect must disarm in the notification tick"
15490        );
15491        tokio::time::advance(Duration::from_secs(60)).await;
15492        actor.no_probe();
15493        let reconnect_tick = Instant::now();
15494        actor.hello(2, true);
15495        let rearmed = actor
15496            .wait_for_select(|checkpoint| {
15497                checkpoint.registered_connection == Some(ConnectionId::new(2))
15498                    && checkpoint.next_probe_at.is_some()
15499            })
15500            .await;
15501        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15502        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15503        tokio::time::advance(delay).await;
15504        actor.probe().await;
15505    }
15506
15507    #[tokio::test(start_paused = true)]
15508    async fn reregistration_without_health_disarms_the_old_schedule() {
15509        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15510        actor.hello(1, true);
15511        actor
15512            .wait_for_select(|checkpoint| {
15513                checkpoint.registered_connection == Some(ConnectionId::new(1))
15514                    && checkpoint.next_probe_at.is_some()
15515            })
15516            .await;
15517        actor
15518            .registry
15519            .deregister_connection(ConnectionId::new(1))
15520            .unwrap();
15521        actor.hello(2, false);
15522        let disarmed = actor
15523            .wait_for_select(|checkpoint| {
15524                checkpoint.registered_connection == Some(ConnectionId::new(2))
15525            })
15526            .await;
15527        assert_eq!(disarmed.next_probe_at, None);
15528        assert_eq!(disarmed.wake_after, None);
15529        tokio::time::advance(Duration::from_secs(60)).await;
15530        actor.no_probe();
15531    }
15532
15533    #[tokio::test(start_paused = true)]
15534    async fn restart_waits_for_its_own_hello_before_rearming_health() {
15535        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15536        actor.hello(1, true);
15537        let armed = actor
15538            .wait_for_select(|checkpoint| {
15539                checkpoint.registered_connection == Some(ConnectionId::new(1))
15540                    && checkpoint.next_probe_at.is_some()
15541            })
15542            .await;
15543        // Simulate the old process's connection closing before the restart
15544        // tears the process down. The replacement process has started but has
15545        // not sent its HELLO, so it is not registered.
15546        actor
15547            .registry
15548            .deregister_connection(ConnectionId::new(1))
15549            .unwrap();
15550        actor.module.restart(Some(0)).await.unwrap();
15551        let parked = actor
15552            .wait_for_select(|checkpoint| {
15553                checkpoint.generation > armed.generation
15554                    && checkpoint.registered_connection.is_none()
15555                    && checkpoint.next_probe_at.is_none()
15556            })
15557            .await;
15558        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15559        actor.rx.close();
15560        // Drain the stop notice the restart queued for the old process's connection,
15561        // so it isn't mistaken for traffic to the replacement.
15562        while actor.rx.try_recv().is_ok() {}
15563        let turns = actor.turns();
15564        tokio::time::advance(Duration::from_secs(60)).await;
15565        let after = actor.turns();
15566        eprintln!(
15567            "restart park turns: before={turns} after={after} delta={}",
15568            after - turns
15569        );
15570        assert_eq!(after, turns, "replacement must park before HELLO");
15571        let hello_tick = Instant::now();
15572        actor.hello(2, true);
15573        let rearmed = actor
15574            .wait_for_select(|checkpoint| {
15575                checkpoint.registered_connection == Some(ConnectionId::new(2))
15576                    && checkpoint.next_probe_at.is_some()
15577            })
15578            .await;
15579        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15580        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15581        tokio::time::advance(delay).await;
15582        actor.probe().await;
15583    }
15584
15585    #[tokio::test(start_paused = true)]
15586    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15587        use tokio::{
15588            io::{AsyncReadExt, AsyncWriteExt},
15589            net::TcpListener,
15590        };
15591        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15592        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15593        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15594        let (started_tx, mut started_rx) = mpsc::channel(8);
15595        let peer = tokio::spawn(async move {
15596            loop {
15597                let (mut socket, _) = listener.accept().await.unwrap();
15598                let mut request = [0; 1024];
15599                assert!(socket.read(&mut request).await.unwrap() > 0);
15600                started_tx.send(Instant::now()).await.unwrap();
15601                socket
15602                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15603                    .await
15604                    .unwrap();
15605            }
15606        });
15607        actor
15608            .module
15609            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15610            .await
15611            .unwrap();
15612        let first_tick = Instant::now();
15613        let first = actor
15614            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15615            .await;
15616        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15617        assert!(started_rx.try_recv().is_err());
15618
15619        // Restart just before the old deadline: reusing it would give the new
15620        // process only one second to start, rather than a full cadence.
15621        tokio::time::advance(
15622            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15623        )
15624        .await;
15625        actor.module.restart(Some(0)).await.unwrap();
15626        let spawn_tick = Instant::now();
15627        let replacement = actor
15628            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15629            .await;
15630        let deadline = replacement.next_probe_at.unwrap();
15631        assert!(
15632            deadline >= spawn_tick + actor.health.cadence,
15633            "a replacement must not inherit its predecessor's probe deadline"
15634        );
15635        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15636        assert!(
15637            started_rx.try_recv().is_err(),
15638            "no HTTP probe during startup grace"
15639        );
15640        tokio::time::advance(deadline - Instant::now()).await;
15641        let landed = wait_for("first replacement HTTP probe", || {
15642            started_rx.try_recv().ok()
15643        })
15644        .await;
15645        assert!(landed >= spawn_tick + actor.health.cadence);
15646        wait_for("healthy replacement", || {
15647            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15648                .then_some(())
15649        })
15650        .await;
15651        peer.abort();
15652    }
15653
15654    #[tokio::test(start_paused = true)]
15655    async fn rescan_adds_and_removes_http_health_without_polling() {
15656        use tokio::{
15657            io::{AsyncReadExt, AsyncWriteExt},
15658            net::TcpListener,
15659        };
15660        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15661        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15662        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15663        // One millisecond separates successive probes without a polling delay.
15664        actor.health.cadence = Duration::from_millis(1);
15665        let (started_tx, mut started_rx) = mpsc::channel(8);
15666        let peer = tokio::spawn(async move {
15667            loop {
15668                let (mut socket, _) = listener.accept().await.unwrap();
15669                let mut request = [0; 1024];
15670                assert!(socket.read(&mut request).await.unwrap() > 0);
15671                started_tx.send(()).await.unwrap();
15672                socket
15673                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15674                    .await
15675                    .unwrap();
15676            }
15677        });
15678        let tick = Instant::now();
15679        let turns = actor.turns();
15680        actor
15681            .module
15682            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15683            .await
15684            .unwrap();
15685        let armed = actor
15686            .wait_for_select(|checkpoint| {
15687                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15688            })
15689            .await;
15690        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15691        assert_eq!(Instant::now(), tick);
15692        tokio::time::advance(Duration::from_millis(1)).await;
15693        let mut started = false;
15694        wait_for("successful HTTP probe", || {
15695            started |= started_rx.try_recv().is_ok();
15696            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15697                .then_some(())
15698        })
15699        .await;
15700        assert!(started, "added HTTP check must start at its first deadline");
15701        assert_eq!(
15702            actor.module.status().unwrap().health.status,
15703            SupervisorHealthStatus::Ok
15704        );
15705        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15706        actor.health.http = None;
15707        let turns = actor.turns();
15708        actor
15709            .module
15710            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15711            .await
15712            .unwrap();
15713        let parked = actor
15714            .wait_for_select(|checkpoint| {
15715                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15716            })
15717            .await;
15718        assert_eq!(parked.wake_after, None);
15719        let turns = actor.turns();
15720        tokio::time::advance(Duration::from_secs(60)).await;
15721        assert_eq!(
15722            actor.turns(),
15723            turns,
15724            "removed HTTP check must park the actor"
15725        );
15726        assert!(
15727            started_rx.try_recv().is_err(),
15728            "removed HTTP check must stay disarmed"
15729        );
15730        assert_eq!(
15731            actor.module.status().unwrap().health,
15732            ModuleHealthStatus::default()
15733        );
15734        peer.abort();
15735    }
15736}
15737
15738#[cfg(test)]
15739mod drain_busy_reprobe_tests {
15740    //! The drain's busy-gauge wait, driven against an in-process module
15741    //! connection whose health answers the test scripts. Time is paused, so
15742    //! every elapsed duration below is exact virtual time.
15743    use super::*;
15744    use std::collections::BTreeMap;
15745    use std::sync::Mutex as StdMutex;
15746    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15747
15748    const MODULE_ID: &str = "drain-reprobe-module";
15749    const CONNECTION: u64 = 7;
15750    const GAUGE: &str = "runs_in_flight";
15751    /// The production default for both the health cadence and the drain
15752    /// ceiling, which is the pairing that used to cost a full ceiling.
15753    const CADENCE: Duration = Duration::from_secs(30);
15754    const CEILING: Duration = Duration::from_secs(30);
15755
15756    /// How the scripted module answers each `health.check`.
15757    #[derive(Clone, Copy)]
15758    enum Answer {
15759        /// Report the gauge busy until this instant, quiet from then on, after
15760        /// taking `latency` to answer.
15761        BusyUntil { until: Instant, latency: Duration },
15762        /// Answer at once, with metrics that leave the declared gauge out.
15763        OmitsGauge,
15764        /// Never answer.
15765        Silent,
15766    }
15767
15768    struct Drain {
15769        forwarding: Arc<ForwardingTable>,
15770        runtime: SupervisorRuntimeConfig,
15771        endpoint: crate::ModuleEndpointId,
15772        /// When each probe reached the module.
15773        probes: Arc<StdMutex<Vec<Instant>>>,
15774        responder: tokio::task::JoinHandle<()>,
15775    }
15776
15777    impl Drain {
15778        fn start(answer: Answer) -> Self {
15779            let registry = Arc::new(Registry::default());
15780            let forwarding = Arc::new(ForwardingTable::default());
15781            let health = HealthConfig {
15782                cadence: CADENCE,
15783                ..HealthConfig::default()
15784            };
15785            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15786                .with_forwarding(forwarding.clone())
15787                .with_health_config(health)
15788                .runtime_config();
15789            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15790            let endpoint = forwarding
15791                .register_module_connection(
15792                    ConnectionId::new(CONNECTION),
15793                    MODULE_ID.to_string(),
15794                    subc_protocol::PROTOCOL_VERSION,
15795                    Concurrency::ModuleManaged,
15796                    FrameSink::new(tx),
15797                )
15798                .unwrap();
15799            let probes = Arc::new(StdMutex::new(Vec::new()));
15800            let responder = tokio::spawn({
15801                let forwarding = forwarding.clone();
15802                let probes = probes.clone();
15803                async move {
15804                    while let Some(frame) = rx.recv().await {
15805                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15806                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15807                        probes.lock().unwrap().push(Instant::now());
15808                        let metrics = match answer {
15809                            Answer::BusyUntil { until, latency } => {
15810                                sleep(latency).await;
15811                                let busy = u64::from(Instant::now() < until);
15812                                serde_json::json!({ GAUGE: busy })
15813                            }
15814                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15815                            Answer::Silent => continue,
15816                        };
15817                        // A probe abandoned at its deadline has no RPC left to
15818                        // complete, so a late answer may be refused.
15819                        let _ = forwarding.complete_module_control_rpc(
15820                            ConnectionId::new(CONNECTION),
15821                            frame.header.corr,
15822                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15823                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15824                                status: HealthStatus::Ok,
15825                                detail: None,
15826                                metrics: Some(metrics),
15827                            }),
15828                        );
15829                    }
15830                }
15831            });
15832            Self {
15833                forwarding,
15834                runtime,
15835                endpoint,
15836                probes,
15837                responder,
15838            }
15839        }
15840
15841        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15842        /// how long it took.
15843        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15844            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15845            (drained, elapsed)
15846        }
15847
15848        /// [`Self::run`], also returning what the drain's probes read.
15849        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15850            let started = Instant::now();
15851            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15852            let drained = wait_for_forwarding_quiescence(
15853                &self.forwarding,
15854                MODULE_ID,
15855                &self.runtime,
15856                self.endpoint,
15857                started + ceiling,
15858                &[GAUGE.to_string()],
15859                DrainScope::Active,
15860                &mut trace,
15861            )
15862            .await
15863            .unwrap();
15864            (drained, started.elapsed(), trace)
15865        }
15866
15867        fn probes(&self) -> Vec<Instant> {
15868            self.probes.lock().unwrap().clone()
15869        }
15870    }
15871
15872    impl Drop for Drain {
15873        fn drop(&mut self) {
15874            self.responder.abort();
15875        }
15876    }
15877
15878    #[tokio::test(start_paused = true)]
15879    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15880        let drain = Drain::start(Answer::BusyUntil {
15881            until: Instant::now() + Duration::from_secs(1),
15882            latency: Duration::ZERO,
15883        });
15884        let (drained, elapsed) = drain.run(CEILING).await;
15885        assert!(
15886            drained,
15887            "gauges that read zero during the drain must let it finish drained, \
15888             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15889        );
15890        assert!(
15891            elapsed < Duration::from_secs(3),
15892            "drain took {elapsed:?} after the gauges went quiet at 1s"
15893        );
15894        let probes = drain.probes();
15895        assert!(
15896            probes.len() >= 2,
15897            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15898        );
15899    }
15900
15901    /// The same shape in real time on a multi-threaded runtime: the first
15902    /// probe is answered busy at once, within a millisecond of the drain
15903    /// starting, and later probes read idle. The drain must keep probing
15904    /// after that first instant answer and end soon after the gauge falls.
15905    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15906    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15907        let drain = Drain::start(Answer::BusyUntil {
15908            until: Instant::now() + Duration::from_millis(600),
15909            latency: Duration::ZERO,
15910        });
15911        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15912        assert!(drained, "the gauge fell at 600ms: {trace}");
15913        assert!(
15914            elapsed < Duration::from_secs(3),
15915            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15916        );
15917        let probes = drain.probes();
15918        assert!(
15919            probes.len() >= 2 && trace.probes as usize == probes.len(),
15920            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15921        );
15922    }
15923
15924    #[tokio::test(start_paused = true)]
15925    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15926        let started = Instant::now();
15927        let drain = Drain::start(Answer::BusyUntil {
15928            until: started + Duration::from_secs(3600),
15929            latency: Duration::ZERO,
15930        });
15931        let (drained, elapsed) = drain.run(CEILING).await;
15932        assert!(!drained, "a module busy throughout must not drain");
15933        assert!(
15934            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15935            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15936        );
15937        // Re-probing is paced by the interval, not a busy loop: at most one
15938        // probe per interval across the ceiling, plus the first at time zero.
15939        let probes = drain.probes();
15940        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15941        assert!(
15942            probes.len() as u128 <= max_probes,
15943            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15944            probes.len()
15945        );
15946        for pair in probes.windows(2) {
15947            assert!(
15948                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15949                "consecutive probes {:?} apart",
15950                pair[1] - pair[0]
15951            );
15952        }
15953    }
15954
15955    #[tokio::test(start_paused = true)]
15956    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15957        // Each answer takes longer than the re-probe interval; the next probe
15958        // must wait for the previous answer and then the interval.
15959        let latency = Duration::from_secs(1);
15960        let drain = Drain::start(Answer::BusyUntil {
15961            until: Instant::now() + Duration::from_secs(3600),
15962            latency,
15963        });
15964        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15965        assert!(!drained);
15966        let probes = drain.probes();
15967        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15968        for pair in probes.windows(2) {
15969            assert!(
15970                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15971                "a probe started {:?} after the previous one, before it could finish",
15972                pair[1] - pair[0]
15973            );
15974        }
15975    }
15976
15977    #[tokio::test(start_paused = true)]
15978    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15979        let drain = Drain::start(Answer::Silent);
15980        let ceiling = Duration::from_secs(2);
15981        let (drained, elapsed) = drain.run(ceiling).await;
15982        assert!(!drained, "an unanswered probe counts as busy");
15983        assert!(
15984            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15985            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15986        );
15987        assert_eq!(
15988            drain.probes().len(),
15989            1,
15990            "the unanswered probe holds the only slot until the ceiling"
15991        );
15992    }
15993
15994    /// Every event logged while it is installed, as `message` plus fields.
15995    #[derive(Clone, Default)]
15996    struct LogCapture {
15997        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15998    }
15999
16000    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16001        fn on_event(
16002            &self,
16003            event: &tracing::Event<'_>,
16004            _context: tracing_subscriber::layer::Context<'_, S>,
16005        ) {
16006            struct Fields(BTreeMap<String, String>);
16007            impl tracing::field::Visit for Fields {
16008                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16009                    self.0
16010                        .insert(field.name().to_string(), format!("{value:?}"));
16011                }
16012            }
16013            let mut fields = Fields(BTreeMap::new());
16014            event.record(&mut fields);
16015            self.events.lock().unwrap().push(fields.0);
16016        }
16017    }
16018
16019    impl LogCapture {
16020        fn install(&self) -> tracing::subscriber::DefaultGuard {
16021            use tracing_subscriber::layer::SubscriberExt;
16022            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16023        }
16024
16025        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16026            self.events
16027                .lock()
16028                .unwrap()
16029                .iter()
16030                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16031                .cloned()
16032                .collect()
16033        }
16034    }
16035
16036    /// A module that declares a busy gauge but leaves it out of its health
16037    /// answers holds a drain to its ceiling. The timeout line must say so by
16038    /// name, and the drain must warn about the missing gauge once, not on
16039    /// every probe.
16040    #[tokio::test(start_paused = true)]
16041    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16042        let capture = LogCapture::default();
16043        let _guard = capture.install();
16044        let drain = Drain::start(Answer::OmitsGauge);
16045        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16046        assert!(!drained, "a gauge never reported cannot read quiet");
16047        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16048
16049        let missing = capture.with_message("lacks a declared busy gauge");
16050        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16051        assert_eq!(
16052            missing[0].get("missing_gauges").map(String::as_str),
16053            Some(GAUGE)
16054        );
16055
16056        warn_drain_timed_out(
16057            MODULE_ID,
16058            Duration::from_secs(2),
16059            RouteCloseReason::Restart,
16060            &Default::default(),
16061            &trace,
16062        );
16063        let timeouts = capture.with_message("route drain timed out");
16064        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16065        let gauges = &timeouts[0]["busy_gauges"];
16066        assert!(
16067            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16068            "the timeout line must name the gauge and why it read busy: {gauges}"
16069        );
16070        assert!(
16071            gauges.ends_with("; no value ever read"),
16072            "no value was ever read for the gauge: {gauges}"
16073        );
16074        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16075    }
16076
16077    /// The trace names a busy value and when it was read. For a probe that
16078    /// never got an answer, it names the probe's error kind and says that no
16079    /// value was ever read.
16080    #[tokio::test(start_paused = true)]
16081    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16082        let busy = Drain::start(Answer::BusyUntil {
16083            until: Instant::now() + Duration::from_secs(3600),
16084            latency: Duration::ZERO,
16085        });
16086        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16087        let rendered = trace.to_string();
16088        assert!(
16089            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16090                && rendered.contains("; last value non_zero(1) @"),
16091            "{rendered}"
16092        );
16093
16094        let silent = Drain::start(Answer::Silent);
16095        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16096        // The probe is capped at the 1 s ceiling, and its reading is stamped
16097        // when it gave up.
16098        assert_eq!(
16099            trace.to_string(),
16100            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16101        );
16102        assert_eq!(trace.probes, 1);
16103    }
16104
16105    /// A gauge value that is not a count reads busy and is shown as sent.
16106    #[test]
16107    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16108        let gauges = [GAUGE.to_string()];
16109        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16110        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16111        assert_eq!(
16112            busy_gauge_observation(&verdicts),
16113            BusyGaugeObservation::Busy
16114        );
16115        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16116        assert_eq!(
16117            busy_gauge_observation(&zero),
16118            BusyGaugeObservation::Quiescent
16119        );
16120        let none = gauge_verdicts(None, &gauges);
16121        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16122    }
16123}
16124
16125#[cfg(test)]
16126mod health_evidence_tests {
16127    use super::{HealthProbeError, HealthProbeEvidence};
16128    use std::collections::HashSet;
16129
16130    /// The evidential asymmetry, asserted rather than described.
16131    ///
16132    /// Exactly ONE observation is proof a module cannot serve, and the one that
16133    /// fires under CPU starvation is not it. Before the split, all fifteen
16134    /// construction sites collapsed into a single String, so a timeout carried the
16135    /// same weight as a dead lane -- which is how a healthy module was restarted
16136    /// three times in one day.
16137    #[test]
16138    fn only_a_dead_lane_is_proof_of_death() {
16139        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16140        // Three non-proof classes, each for a different reason: silence is
16141        // consistent with health, a bad answer proves the module ALIVE, and a
16142        // daemon-side fault never reached the module at all.
16143        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16144        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16145        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16146    }
16147
16148    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
16149    ///
16150    /// A shared label renders two different observations identically in the line an
16151    /// operator reads after an unexplained restart -- the exact confusion this
16152    /// change removes.
16153    #[test]
16154    fn every_evidence_class_has_a_distinct_label() {
16155        let labels = [
16156            HealthProbeError::lane_dead("").label(),
16157            HealthProbeError::no_answer("").label(),
16158            HealthProbeError::bad_answer("").label(),
16159            HealthProbeError::misconfigured("").label(),
16160        ];
16161        let unique: HashSet<_> = labels.iter().collect();
16162        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16163    }
16164
16165    /// The class is additional information, not a replacement.
16166    ///
16167    /// An operator needs both "this was silence" and the specific text saying how
16168    /// long we waited; a classification that swallowed the message would trade one
16169    /// missing distinction for another.
16170    #[test]
16171    fn classification_preserves_the_original_message() {
16172        let err = HealthProbeError::no_answer("module did not answer within 5s");
16173        assert_eq!(err.to_string(), "module did not answer within 5s");
16174        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16175    }
16176}
16177
16178#[cfg(test)]
16179mod health_tombstone_tests {
16180    use std::{path::PathBuf, sync::Arc, time::Duration};
16181
16182    use subc_protocol::{
16183        manifest::Concurrency,
16184        session::{HealthStatus, ModuleControlResponse},
16185    };
16186    use tokio::sync::mpsc;
16187
16188    use super::{
16189        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16190        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16191    };
16192    use crate::{
16193        control::ControlHandler,
16194        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16195        registry::{ConnectionId, Registry},
16196        router::FrameSink,
16197    };
16198
16199    struct ProbeHarness {
16200        spec: ModuleSpec,
16201        runtime: SupervisorRuntimeConfig,
16202        forwarding: Arc<ForwardingTable>,
16203        module_connection: ConnectionId,
16204        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16205        handler: ControlHandler,
16206        module: super::SupervisedModule,
16207    }
16208
16209    fn probe_harness() -> ProbeHarness {
16210        let registry = Arc::new(Registry::default());
16211        let forwarding = Arc::new(ForwardingTable::default());
16212        let supervisor_handle = super::SupervisorHandle::new();
16213        let health = HealthConfig {
16214            http: None,
16215            cadence: Duration::from_secs(30),
16216            deadline: Duration::from_secs(5),
16217            failure_threshold: 3,
16218            on_degraded: HealthAction::Report,
16219            on_failing: HealthAction::Report,
16220            critical: false,
16221        };
16222        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16223            .with_forwarding(Arc::clone(&forwarding))
16224            .with_handle(supervisor_handle.clone())
16225            .with_health_config(health);
16226        let spec = ModuleSpec {
16227            module_id: "late-health-module".to_string(),
16228            program: PathBuf::from("disabled-module"),
16229            args: Vec::new(),
16230            env: Vec::new(),
16231            reserved: false,
16232            reserved_prefixes: Vec::new(),
16233            protocol: ModuleProtocol::Subc,
16234            overlap: Default::default(),
16235            carry_containment: false,
16236        };
16237        let module = supervisor
16238            .supervise_configured(spec.clone(), false)
16239            .unwrap();
16240        let runtime = supervisor.runtime_config();
16241        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16242            .with_supervisor(supervisor_handle);
16243        let module_connection = ConnectionId::new(700);
16244        let (module_tx, module_rx) = mpsc::channel(8);
16245        forwarding
16246            .register_module_connection(
16247                module_connection,
16248                spec.module_id.clone(),
16249                subc_protocol::PROTOCOL_VERSION,
16250                Concurrency::ModuleManaged,
16251                FrameSink::new(module_tx),
16252            )
16253            .unwrap();
16254
16255        ProbeHarness {
16256            spec,
16257            runtime,
16258            forwarding,
16259            module_connection,
16260            module_rx,
16261            handler,
16262            module,
16263        }
16264    }
16265
16266    async fn finish_after(
16267        harness: &mut ProbeHarness,
16268        stall: Duration,
16269    ) -> ModuleControlRpcCompletion {
16270        assert!(stall > harness.runtime.health.deadline);
16271        let deadline = harness.runtime.health.deadline;
16272        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16273        let answer = async {
16274            let frame = harness.module_rx.recv().await.expect("health.check frame");
16275            tokio::time::advance(deadline).await;
16276            tokio::task::yield_now().await;
16277            tokio::time::advance(stall - deadline).await;
16278            harness
16279                .forwarding
16280                .complete_module_control_rpc(
16281                    harness.module_connection,
16282                    frame.header.corr,
16283                    Some("health.check"),
16284                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16285                        status: HealthStatus::Ok,
16286                        detail: None,
16287                        metrics: None,
16288                    }),
16289                )
16290                .unwrap()
16291        };
16292        let (probe_result, completion) = tokio::join!(probe, answer);
16293        let err = probe_result.expect_err("probe must miss its deadline");
16294        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16295        completion
16296    }
16297
16298    async fn time_out_without_answer(harness: &mut ProbeHarness) {
16299        let deadline = harness.runtime.health.deadline;
16300        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16301        let exhaust_deadline = async {
16302            let _frame = harness.module_rx.recv().await.expect("health.check frame");
16303            tokio::time::advance(deadline).await;
16304            tokio::task::yield_now().await;
16305        };
16306        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16307        let err = probe_result.expect_err("probe must miss its deadline");
16308        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16309    }
16310
16311    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16312        let registry = Arc::clone(&harness.module.inner.registry);
16313        let snapshot = Arc::clone(&harness.module.inner.snapshot);
16314        let process_liveness = super::SupervisorProcessLiveness::default();
16315        let mut child = None;
16316        let cycle = super::run_health_probe_cycle(
16317            &harness.spec,
16318            &harness.runtime,
16319            &registry,
16320            &process_liveness,
16321            &snapshot,
16322            &mut child,
16323        );
16324        let peer = async {
16325            let frame = harness.module_rx.recv().await.expect("health.check frame");
16326            if answer {
16327                harness
16328                    .forwarding
16329                    .complete_module_control_rpc(
16330                        harness.module_connection,
16331                        frame.header.corr,
16332                        Some("health.check"),
16333                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16334                            status: HealthStatus::Ok,
16335                            detail: None,
16336                            metrics: Some(serde_json::json!({"ready": true})),
16337                        }),
16338                    )
16339                    .unwrap();
16340            } else {
16341                tokio::time::advance(harness.runtime.health.deadline).await;
16342                tokio::task::yield_now().await;
16343            }
16344        };
16345        tokio::join!(cycle, peer);
16346    }
16347
16348    #[tokio::test(start_paused = true)]
16349    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16350        let mut harness = probe_harness();
16351        // Drive the probe cycle directly with an in-memory wire peer. Stop the
16352        // disabled module's monitor so only this test owns lifecycle transitions;
16353        // no OS process is launched, and a restart is observed at scheduling.
16354        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16355        monitor.abort();
16356        let _ = monitor.await;
16357        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16358            state.enabled = true;
16359            state.state = super::ModuleState::Running;
16360            state.process_alive = true;
16361        })
16362        .unwrap();
16363
16364        run_probe_cycle(&mut harness, true).await;
16365        assert_eq!(
16366            harness.module.status().unwrap().health.status,
16367            super::SupervisorHealthStatus::Ok
16368        );
16369
16370        for failures in 1..harness.runtime.health.failure_threshold {
16371            run_probe_cycle(&mut harness, false).await;
16372            let status = harness.module.status().unwrap();
16373            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16374            assert_eq!(status.health.consecutive_failures, failures);
16375            assert!(status.health.last_probe_ms.is_some());
16376            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16377            assert!(status.health.metrics.is_none());
16378            assert_eq!(status.state, super::ModuleState::Running);
16379            assert!(status.process_alive);
16380            assert_eq!(status.restart_count, 0);
16381            assert_eq!(status.lifetime_restarts, 0);
16382            assert!(status.health.last_action.is_none());
16383            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16384        }
16385
16386        run_probe_cycle(&mut harness, true).await;
16387        let recovered = harness.module.status().unwrap();
16388        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16389        assert_eq!(recovered.health.consecutive_failures, 0);
16390        assert!(recovered.health.detail.is_none());
16391        assert_eq!(
16392            recovered.health.metrics,
16393            Some(serde_json::json!({"ready": true}))
16394        );
16395        assert_eq!(recovered.lifetime_restarts, 0);
16396
16397        for failures in 1..=harness.runtime.health.failure_threshold {
16398            run_probe_cycle(&mut harness, false).await;
16399            let status = harness.module.status().unwrap();
16400            assert_eq!(status.health.consecutive_failures, failures);
16401            if failures < harness.runtime.health.failure_threshold {
16402                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16403                assert_eq!(status.state, super::ModuleState::Running);
16404                assert_eq!(status.lifetime_restarts, 0);
16405                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16406            } else {
16407                assert_eq!(
16408                    status.health.status,
16409                    super::SupervisorHealthStatus::Unresponsive
16410                );
16411                assert_eq!(status.state, super::ModuleState::Restarting);
16412                assert_eq!(status.restart_count, 1);
16413                assert_eq!(status.lifetime_restarts, 1);
16414                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16415                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16416            }
16417        }
16418    }
16419
16420    #[tokio::test(start_paused = true)]
16421    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16422        let mut harness = probe_harness();
16423
16424        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16425        let first_latency = match &first {
16426            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16427            other => panic!("late answer was not retained: {other:?}"),
16428        };
16429        assert!(harness.handler.observe_module_control_completion(first));
16430
16431        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16432        let second_latency = match &second {
16433            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16434            other => panic!("late answer was not retained: {other:?}"),
16435        };
16436        assert!(harness.handler.observe_module_control_completion(second));
16437
16438        assert_eq!(first_latency, Duration::from_secs(8));
16439        assert_eq!(
16440            second_latency - first_latency,
16441            Duration::from_secs(3),
16442            "latency must grow linearly with the additional stall"
16443        );
16444        let health = harness.module.status().unwrap().health;
16445        assert_eq!(health.late_answer_count, 2);
16446        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16447    }
16448
16449    /// A module that answers every probe late must never march to the kill
16450    /// threshold: the late answer proves it is alive, so it must clear the miss
16451    /// streak the timeout recorded. Without the reset, a CPU-starved module
16452    /// that serves every probe seconds past the deadline accumulates
16453    /// `consecutive_failures` to the threshold and is killed — the exact
16454    /// sequence from the 2026-08-14 aft disable, where the daemon logged
16455    /// "proves the module is alive" five times while counting five misses.
16456    #[tokio::test(start_paused = true)]
16457    async fn late_answer_clears_the_consecutive_failure_streak() {
16458        let mut harness = probe_harness();
16459
16460        // Timeout recorded first: the probe path saw no answer in time.
16461        time_out_without_answer(&mut harness).await;
16462        harness
16463            .module
16464            .record_health_probe_failure_for_test("[no-answer] test miss")
16465            .unwrap();
16466        assert_eq!(
16467            harness.module.status().unwrap().health.consecutive_failures,
16468            1,
16469            "precondition: the miss must be on the streak before the late answer"
16470        );
16471
16472        // The stalled reply then lands: proof of life.
16473        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16474        assert!(matches!(
16475            late,
16476            ModuleControlRpcCompletion::LateHealthAnswer { .. }
16477        ));
16478        assert!(harness.handler.observe_module_control_completion(late));
16479
16480        let health = harness.module.status().unwrap().health;
16481        assert_eq!(
16482            health.consecutive_failures, 0,
16483            "a late answer is an answer: the streak must reset"
16484        );
16485        assert_eq!(health.late_answer_count, 1);
16486    }
16487
16488    #[tokio::test(start_paused = true)]
16489    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16490        let mut harness = probe_harness();
16491
16492        for _ in 0..20 {
16493            time_out_without_answer(&mut harness).await;
16494            assert_eq!(
16495                harness.forwarding.health_probe_tombstone_count().unwrap(),
16496                1
16497            );
16498        }
16499    }
16500}
16501
16502#[cfg(test)]
16503mod child_env_tests {
16504    #[cfg(windows)]
16505    use super::{
16506        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16507    };
16508    #[cfg(windows)]
16509    use std::sync::{Arc, OnceLock};
16510    #[cfg(windows)]
16511    #[test]
16512    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16513        let spec = spec(Vec::new());
16514        let mut command = Command::new("unused");
16515        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16516            .unwrap()
16517            .unwrap();
16518        let name = command
16519            .as_std()
16520            .get_envs()
16521            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16522            .and_then(|(_, value)| value)
16523            .unwrap();
16524        assert_eq!(name, OsStr::new(handoff.name()));
16525        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16526        assert!(command.as_std().get_envs().any(|(key, value)| key
16527            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16528            && value == Some(OsStr::new("env"))));
16529        assert!(command
16530            .as_std()
16531            .get_envs()
16532            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16533    }
16534
16535    #[cfg(windows)]
16536    #[test]
16537    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16538        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16539        snapshot.process_alive = true;
16540        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16541        assert_eq!(
16542            observed_launch_nonce_source(&snapshot).as_deref(),
16543            Some("env")
16544        );
16545        let consumed = Arc::new(OnceLock::new());
16546        snapshot.pipe_consumed = Some(consumed.clone());
16547        assert_eq!(
16548            observed_launch_nonce_source(&snapshot).as_deref(),
16549            Some("env")
16550        );
16551        consumed.set(()).unwrap();
16552        assert_eq!(
16553            observed_launch_nonce_source(&snapshot).as_deref(),
16554            Some("pipe")
16555        );
16556        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16557        assert_eq!(
16558            observed_launch_nonce_source(&snapshot).as_deref(),
16559            Some("env")
16560        );
16561        clear_current_process_facts(&mut snapshot);
16562        assert_eq!(observed_launch_nonce_source(&snapshot), None);
16563    }
16564    use super::{
16565        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16566        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16567        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16568    };
16569    use std::{ffi::OsStr, path::PathBuf};
16570    use tokio::process::Command;
16571
16572    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16573        ModuleSpec {
16574            module_id: "env-plan".to_string(),
16575            program: PathBuf::from("/nonexistent"),
16576            args: Vec::new(),
16577            env,
16578            reserved: false,
16579            reserved_prefixes: Vec::new(),
16580            protocol: ModuleProtocol::Subc,
16581            overlap: Default::default(),
16582            carry_containment: false,
16583        }
16584    }
16585
16586    #[test]
16587    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16588        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16589        let digest = LaunchSpecDigest::from_spec(&base);
16590        let mut reordered = base.clone();
16591        reordered.env.reverse();
16592        reordered.module_id = "different-id".into();
16593        reordered.reserved = true;
16594        reordered.overlap = Default::default();
16595        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16596        let mut delimiter_collision = base.clone();
16597        delimiter_collision.env[0] = ("A=B".into(), "C".into());
16598        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16599        let mut changed = base.clone();
16600        changed.protocol = ModuleProtocol::None;
16601        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16602        changed = base.clone();
16603        changed.args.push("--flag".into());
16604        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16605        changed = base.clone();
16606        changed.program = PathBuf::from("/other");
16607        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16608        changed = base.clone();
16609        changed.env[1].1 = "other".into();
16610        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16611        assert!(!format!("{digest:?}").contains("B=C"));
16612        assert!(!format!("{digest:?}").contains("[u8"));
16613    }
16614
16615    #[cfg(unix)]
16616    #[test]
16617    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16618        use std::os::unix::ffi::OsStringExt;
16619        let mut left = spec(Vec::new());
16620        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16621        let mut right = left.clone();
16622        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16623        assert_eq!(
16624            left.program.to_string_lossy(),
16625            right.program.to_string_lossy()
16626        );
16627        assert_ne!(
16628            LaunchSpecDigest::from_spec(&left),
16629            LaunchSpecDigest::from_spec(&right)
16630        );
16631    }
16632
16633    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
16634    /// one still gets its own.
16635    ///
16636    /// This is the narrow goal `env_clear()` was reached for, and the reason the
16637    /// fix is `env_remove` rather than deleting the line: an operator's ambient
16638    /// filter silently becoming an unconfigured module's log level is a real
16639    /// defect, just a much smaller one than clearing the environment.
16640    ///
16641    /// Asserted on the command plan rather than a spawned child because proving
16642    /// the ABSENCE of an inherited variable needs the parent's environment
16643    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
16644    /// removal as `(key, None)`, which is exactly the distinction wanted: not
16645    /// "absent because nobody set it" but "explicitly unset for the child".
16646    #[test]
16647    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16648        let mut command = Command::new("/nonexistent");
16649        apply_child_env(&mut command, &spec(Vec::new()));
16650        let removed = command
16651            .as_std()
16652            .get_envs()
16653            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16654        assert!(
16655            removed,
16656            "ambient CK_LOG must be explicitly removed for an unconfigured module"
16657        );
16658
16659        let mut configured = Command::new("/nonexistent");
16660        apply_child_env(
16661            &mut configured,
16662            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16663        );
16664        let effective = configured
16665            .as_std()
16666            .get_envs()
16667            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16668            .last()
16669            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16670        assert_eq!(
16671            effective,
16672            Some(Some("debug".to_string())),
16673            "a module's configured CK_LOG must survive the ambient removal"
16674        );
16675    }
16676
16677    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
16678    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
16679    /// the same reason as the CK_LOG test above.
16680    ///
16681    /// The argument is the load-bearing half: a stock binary exits on an
16682    /// unknown flag before it listens, so with `--subc` appended the mode
16683    /// could not supervise the one process it exists for. Found by the first
16684    /// conformance run (nats-server: `flag provided but not defined: -subc`).
16685    #[test]
16686    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16687        let connection_file = std::path::Path::new("/run/subc-connection.json");
16688        let handle = SupervisorHandle::new();
16689
16690        let mut none_spec = spec(Vec::new());
16691        none_spec.protocol = ModuleProtocol::None;
16692        let mut none = Command::new("/nonexistent");
16693        let none_handoff =
16694            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16695                .expect("protocol-none spawn args apply");
16696        assert!(
16697            none_handoff.is_none(),
16698            "protocol:none spawn must not receive a nonce descriptor"
16699        );
16700        #[cfg(windows)]
16701        assert!(!none.as_std().get_envs().any(|(key, value)| key
16702            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16703            && value.is_some()));
16704        assert!(
16705            !none.as_std().get_envs().any(|(key, value)| key
16706                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16707                && value.is_some()),
16708            "protocol:none spawn must not name a nonce descriptor"
16709        );
16710        let none_args: Vec<String> = none
16711            .as_std()
16712            .get_args()
16713            .map(|a| a.to_string_lossy().into_owned())
16714            .collect();
16715        assert!(
16716            !none_args.iter().any(|a| a == SUBC_ARG),
16717            "protocol:none argv must not carry --subc; got {none_args:?}"
16718        );
16719        let none_has_nonce = none
16720            .as_std()
16721            .get_envs()
16722            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16723        assert!(
16724            !none_has_nonce,
16725            "protocol:none spawn must not receive a launch nonce"
16726        );
16727        let none_has_module_id = none
16728            .as_std()
16729            .get_envs()
16730            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16731        assert!(
16732            none_has_module_id,
16733            "SUBC_MODULE_ID is inert and stays on every path"
16734        );
16735        assert!(
16736            handle.spawn_nonce(&none_spec.module_id).is_none(),
16737            "no nonce record for a process that will never present one"
16738        );
16739
16740        // Control: the subc-wire path is unchanged by the branch above.
16741        let wire_spec = spec(Vec::new());
16742        let mut wire = Command::new("/nonexistent");
16743        let wire_handoff =
16744            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16745                .expect("subc-wire spawn args apply");
16746        let wire_fd_env = wire
16747            .as_std()
16748            .get_envs()
16749            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16750            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16751        #[cfg(unix)]
16752        assert_eq!(
16753            wire_fd_env,
16754            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16755            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16756        );
16757        #[cfg(windows)]
16758        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16759        #[cfg(not(any(unix, windows)))]
16760        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16761        let wire_args: Vec<String> = wire
16762            .as_std()
16763            .get_args()
16764            .map(|a| a.to_string_lossy().into_owned())
16765            .collect();
16766        assert_eq!(
16767            wire_args,
16768            vec![
16769                SUBC_ARG.to_string(),
16770                connection_file.to_string_lossy().into_owned()
16771            ],
16772            "a subc-wire spawn still carries --subc <path>"
16773        );
16774        assert_eq!(
16775            wire.as_std()
16776                .get_envs()
16777                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16778            !cfg!(unix),
16779            "only Windows supplies the environment nonce"
16780        );
16781        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16782    }
16783
16784    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16785    /// spec tries to set it; only a swap candidate carries it.
16786    ///
16787    /// "Set it only on candidates" is not enough, because spawn applies the
16788    /// spec's env verbatim and the daemon's own environment is inherited: either
16789    /// could hand a plain restart the swap role, and a module reading it would
16790    /// warm on its long swap budget while callers wait. Asserted as an explicit
16791    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16792    /// test above gives.
16793    #[test]
16794    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16795        let role = |command: &Command| {
16796            command
16797                .as_std()
16798                .get_envs()
16799                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16800                .last()
16801                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16802        };
16803        let forged = spec(vec![(
16804            SUBC_SPAWN_ROLE_ENV.to_string(),
16805            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16806        )]);
16807
16808        let mut plain = Command::new("/nonexistent");
16809        apply_child_env(&mut plain, &forged);
16810        apply_spawn_role(&mut plain, SpawnRole::Plain);
16811        assert_eq!(
16812            role(&plain),
16813            Some(None),
16814            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16815        );
16816
16817        let mut candidate = Command::new("/nonexistent");
16818        apply_child_env(&mut candidate, &spec(Vec::new()));
16819        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16820        assert_eq!(
16821            role(&candidate),
16822            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16823        );
16824    }
16825
16826    /// Daemon-private capture retention keys never reach the child.
16827    ///
16828    /// cortexkit-log exposes retention as a Rust struct with no environment
16829    /// names, so these entries are supervisor metadata. Passing them through
16830    /// would invent a public child-process contract by accident.
16831    #[test]
16832    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16833        let mut command = Command::new("/nonexistent");
16834        apply_child_env(
16835            &mut command,
16836            &spec(vec![
16837                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16838                ("KEPT".to_string(), "yes".to_string()),
16839            ]),
16840        );
16841        let keys: Vec<String> = command
16842            .as_std()
16843            .get_envs()
16844            .filter(|(_, value)| value.is_some())
16845            .map(|(key, _)| key.to_string_lossy().into_owned())
16846            .collect();
16847        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16848        assert!(
16849            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16850            "daemon-private capture key leaked to the child: {keys:?}"
16851        );
16852    }
16853}
16854
16855#[cfg(test)]
16856mod jitter_tests {
16857    use super::jittered_health_delay;
16858    use std::{collections::HashSet, time::Duration};
16859
16860    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16861    /// that actually occur rather than invented ones.
16862    ///
16863    /// This is a SAMPLE, not a registry: the property under test is that distinct
16864    /// ids disperse, which holds for any set of distinct strings. Several entries
16865    /// are already historical (modules get renamed), and that costs nothing here --
16866    /// but it means a reader must not mistake this for the live module set, and a
16867    /// rename sweep will match it without there being anything to change.
16868    const FLEET: [&str; 14] = [
16869        "aft",
16870        "alfonso-core",
16871        "magic-context",
16872        "broca",
16873        "thalamus",
16874        "quota",
16875        "engram",
16876        "plexus",
16877        "cerebellum",
16878        "astrocyte",
16879        "synapse",
16880        "subc-mcp",
16881        "cortexkit-credentials",
16882        "subc-federation",
16883    ];
16884
16885    /// Probes must not converge after a fleet-wide restart.
16886    ///
16887    /// This is the property the jitter exists for: every module reconnects at
16888    /// once, and without dispersal all fourteen would then probe on the same
16889    /// tick forever. Nothing failed visibly when this went untested -- a
16890    /// convergent fleet still probes correctly, just in a burst, so the symptom
16891    /// is a periodic load spike that looks like whatever else is running.
16892    #[test]
16893    fn probe_delays_disperse_across_the_fleet() {
16894        let cadence = Duration::from_secs(30);
16895        let delays: HashSet<Duration> = FLEET
16896            .iter()
16897            .map(|id| jittered_health_delay(id, 0, cadence))
16898            .collect();
16899        assert_eq!(
16900            delays.len(),
16901            FLEET.len(),
16902            "every supervised module must land on its own probe offset"
16903        );
16904    }
16905
16906    /// The offset may only ever DELAY a probe, never bring it forward.
16907    ///
16908    /// A delay below the cadence would probe a module more often than
16909    /// configured, which is the opposite of what an operator asked for and
16910    /// would tighten the failure budget without anyone changing it.
16911    #[test]
16912    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16913        let cadence = Duration::from_secs(30);
16914        let span = cadence / 10;
16915        for id in FLEET {
16916            for probe_index in 0..8 {
16917                let delay = jittered_health_delay(id, probe_index, cadence);
16918                assert!(
16919                    delay >= cadence,
16920                    "{id}#{probe_index}: jitter must not shorten the cadence"
16921                );
16922                assert!(
16923                    delay < cadence + span,
16924                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16925                );
16926            }
16927        }
16928    }
16929
16930    /// A module keeps its offset across daemon restarts.
16931    ///
16932    /// The delay is derived rather than randomised precisely so a restart does
16933    /// not re-roll every module into a fresh chance of collision. A random
16934    /// source would satisfy the dispersal test above and quietly lose this.
16935    #[test]
16936    fn a_module_offset_is_stable_across_restarts() {
16937        let cadence = Duration::from_secs(30);
16938        for id in FLEET {
16939            assert_eq!(
16940                jittered_health_delay(id, 0, cadence),
16941                jittered_health_delay(id, 0, cadence),
16942                "{id}: the same module and probe index must produce the same offset"
16943            );
16944        }
16945    }
16946
16947    /// A zero cadence disables probing rather than producing a busy loop.
16948    #[test]
16949    fn zero_cadence_yields_zero_delay() {
16950        assert_eq!(
16951            jittered_health_delay("aft", 0, Duration::ZERO),
16952            Duration::ZERO
16953        );
16954    }
16955}
16956
16957#[cfg(all(test, target_os = "linux"))]
16958mod cgroup_placement_tests {
16959    use super::{
16960        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16961        SuperviseError, SupervisedChild,
16962    };
16963    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16964    use cortexkit_test_support::ScratchDir;
16965    use std::{
16966        fs, io,
16967        path::{Path, PathBuf},
16968        sync::{Arc, Mutex},
16969    };
16970    use tokio::process::Command;
16971
16972    #[tokio::test]
16973    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16974        use super::*;
16975        let dir = ScratchDir::new("unique-spawn-cgroups");
16976        let root = PathBuf::from(format!(
16977            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16978            std::process::id(),
16979            unix_ms_now()
16980        ));
16981        if let Err(error) = fs::create_dir(&root) {
16982            assert!(
16983                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16984                "required cgroup test cannot execute: {error}"
16985            );
16986            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16987            return;
16988        }
16989        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16990        let group_count = || {
16991            fs::read_dir(root.join("subc-modules"))
16992                .unwrap()
16993                .map(|entry| entry.unwrap().file_type().unwrap())
16994                .filter(|kind| kind.is_dir())
16995                .count()
16996        };
16997        let supervisor =
16998            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16999                .with_cgroup_placement(Some(placement.clone()));
17000        let runtime = supervisor.runtime_config();
17001        let mut spec = ModuleSpec {
17002            module_id: "unique-spawn".into(),
17003            program: PathBuf::from("/bin/sleep"),
17004            args: vec!["60".into()],
17005            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17006                .into_iter()
17007                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17008                .collect(),
17009            reserved: false,
17010            reserved_prefixes: vec![],
17011            protocol: ModuleProtocol::None,
17012            overlap: Default::default(),
17013            carry_containment: false,
17014        };
17015        let spawn = |spec: &ModuleSpec| {
17016            spawn_child(
17017                spec,
17018                None,
17019                None,
17020                &runtime.stderr_ring,
17021                None,
17022                &runtime.child_roster,
17023                Some(&placement),
17024            )
17025            .unwrap()
17026        };
17027        let mut live = spawn(&spec);
17028        for _ in 0..3 {
17029            // A new process can enter the old slot while retirement is pending.
17030            let next = spawn(&spec);
17031            assert_ne!(live.module_id, next.module_id);
17032            live.start_kill().unwrap();
17033            live.wait().await.unwrap();
17034            live = next;
17035            assert!(
17036                live.child.try_wait().unwrap().is_none(),
17037                "retiring the old slot must not kill the replacement"
17038            );
17039            assert_eq!(
17040                group_count(),
17041                1,
17042                "only the live spawn's cgroup should remain"
17043            );
17044        }
17045        supervisor.begin_daemon_shutdown();
17046        let reap = tokio::spawn(async move {
17047            live.wait().await.unwrap();
17048        });
17049        supervisor
17050            .end_children_for_daemon_shutdown(false, std::future::pending())
17051            .await;
17052        reap.await.unwrap();
17053        assert_eq!(group_count(), 0);
17054        // A normal exit uses the same tree-cleanup path as a killed spawn.
17055        spec.program = PathBuf::from("/bin/true");
17056        spec.args.clear();
17057        let fresh_roster = ChildRoster::default();
17058        let mut short = spawn_child(
17059            &spec,
17060            None,
17061            None,
17062            &runtime.stderr_ring,
17063            None,
17064            &fresh_roster,
17065            Some(&placement),
17066        )
17067        .unwrap();
17068        short.wait().await.unwrap();
17069        assert_eq!(group_count(), 0);
17070        spec.module_id = "_".repeat(255);
17071        let mut long_id = spawn_child(
17072            &spec,
17073            None,
17074            None,
17075            &runtime.stderr_ring,
17076            None,
17077            &fresh_roster,
17078            Some(&placement),
17079        )
17080        .unwrap();
17081        long_id.wait().await.unwrap();
17082        assert_eq!(
17083            group_count(),
17084            0,
17085            "valid long module IDs must not exceed cgroup NAME_MAX"
17086        );
17087        fs::remove_dir(root.join("subc-modules")).unwrap();
17088        fs::remove_dir(root).unwrap();
17089        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17090    }
17091
17092    #[test]
17093    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17094        let path = Path::new("/definitely-missing-subc-cgroup");
17095        let mut command = Command::new("true");
17096        let error = apply_cgroup_placement(
17097            &mut command,
17098            &ModuleSpec {
17099                module_id: "broken-cgroup".to_string(),
17100                program: PathBuf::from("true"),
17101                args: Vec::new(),
17102                env: Vec::new(),
17103                reserved: false,
17104                reserved_prefixes: Vec::new(),
17105                protocol: ModuleProtocol::Subc,
17106                overlap: Default::default(),
17107                carry_containment: false,
17108            },
17109            path,
17110        )
17111        .expect_err("a parent cgroup open failure must reject the supervised spawn");
17112        let reason = error.to_string();
17113
17114        assert!(
17115            matches!(error, SuperviseError::Cgroup { .. }),
17116            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17117        );
17118        assert!(
17119            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17120            "parent cgroup open failure must name cgroup.procs: {reason}"
17121        );
17122    }
17123
17124    #[tokio::test]
17125    async fn reaping_a_child_removes_its_empty_module_cgroup() {
17126        let root = ScratchDir::new("supervisor-reap-cgroup");
17127        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17128        let placement = subc_cgroup::prepare_at(&root)
17129            .expect("prepare scratch cgroup root")
17130            .expect("scratch root has a cgroup.procs marker");
17131        let module_id = "reaped-module";
17132        let module = placement
17133            .module_path(module_id)
17134            .expect("create scratch module cgroup");
17135        let child = Command::new("true")
17136            .env("XDG_DATA_HOME", root.path())
17137            .env("XDG_RUNTIME_DIR", root.path())
17138            .env("XDG_CONFIG_HOME", root.path())
17139            .spawn()
17140            .expect("spawn short-lived child");
17141        let pid = child.id().expect("spawned child has pid");
17142        let mut child = SupervisedChild {
17143            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17144            child: crate::supervise::containment::ProcessChild::plain(child, pid),
17145            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17146            protocol: ModuleProtocol::Subc,
17147            module_id: module_id.to_string(),
17148            cgroup_placement: Some(placement),
17149            stdout_pump: None,
17150            stderr_pump: None,
17151            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17152            spawned_at: tokio::time::Instant::now(),
17153            registration_deadline: None,
17154            registration_watch: None,
17155            spawned_at_ms: 0,
17156            spawned_from: PathBuf::from("true"),
17157            launch_spec_digest: LaunchSpecDigest([0; 32]),
17158            spawned_file_identity: None,
17159            process_start_time: None,
17160            process_identity: None,
17161            pid,
17162            roster_guard: None,
17163            #[cfg(target_os = "macos")]
17164            privacy_exec: None,
17165            spawn_failure: None,
17166        };
17167
17168        child.wait().await.expect("reap short-lived child");
17169
17170        assert!(
17171            !module.exists(),
17172            "waiting for a non-carried child must remove its empty cgroup before returning"
17173        );
17174        child.finish_exit(false).await.unwrap();
17175    }
17176
17177    #[test]
17178    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17179        let root = ScratchDir::new("supervisor-non-empty-cgroup");
17180        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17181        let placement = subc_cgroup::prepare_at(&root)
17182            .expect("prepare scratch cgroup root")
17183            .expect("scratch root has a cgroup.procs marker");
17184        let module = placement
17185            .module_path("surviving-module")
17186            .expect("create scratch module cgroup");
17187        fs::write(module.join("surviving-process"), b"still present")
17188            .expect("make scratch cgroup non-empty");
17189        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17190
17191        remove_module_cgroup(&placement, "surviving-module");
17192
17193        let logs = crate::router::test_log::captured_logs(&logs);
17194        assert!(
17195            module.exists(),
17196            "failed removal must leave the cgroup intact"
17197        );
17198        assert!(
17199            logs.contains("could not remove module cgroup after process exit; continuing teardown")
17200                && logs.contains("surviving-module"),
17201            "best-effort removal must report the failure without returning it: {logs}"
17202        );
17203    }
17204
17205    #[test]
17206    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17207        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17208        let reason = SuperviseError::Spawn {
17209            program: PathBuf::from("/bin/true"),
17210            source: io::Error::from_raw_os_error(13),
17211            cgroup_path: Some(cgroup_path.clone()),
17212        }
17213        .to_string();
17214
17215        assert!(
17216            reason.contains(&cgroup_path.display().to_string()),
17217            "a pre_exec spawn failure must name the cgroup path: {reason}"
17218        );
17219    }
17220}
17221
17222#[cfg(test)]
17223mod spawn_subscriber_lag_tests {
17224    use super::*;
17225
17226    /// A subscriber whose connection stops draining is dropped once its frame
17227    /// channel fills. The client must learn that from a terminal Error frame
17228    /// after the frames already queued for it, not from a stream that simply
17229    /// goes quiet.
17230    #[tokio::test]
17231    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17232        let feed = SpawnEventFeed::default();
17233        feed.configure_incarnation("lag-incarnation".to_string());
17234        // A one-slot connection queue that nobody reads until the emits are
17235        // done: the forwarder parks on it and the subscriber channel fills.
17236        let (tx, mut rx) = mpsc::channel(1);
17237        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17238            .expect("subscribe");
17239        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17240        for index in 0..emitted {
17241            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17242            // Let the forwarder take what it can so the fill point is the
17243            // subscriber channel, not a scheduling accident.
17244            tokio::task::yield_now().await;
17245        }
17246        assert_eq!(
17247            feed.subscriber_count(),
17248            0,
17249            "the lagged subscriber must be removed"
17250        );
17251
17252        let mut data = Vec::new();
17253        let mut last = None;
17254        loop {
17255            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17256                .await
17257                .expect("the forwarder must finish once the subscriber is dropped");
17258            let Some(outbound) = next else { break };
17259            let frame = outbound.frame;
17260            if frame.header.ty == FrameType::StreamData {
17261                assert!(last.is_none(), "no data may follow the terminal frame");
17262                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17263                data.push(event.cursor.seq);
17264            } else {
17265                assert!(last.is_none(), "exactly one terminal frame");
17266                last = Some(frame);
17267            }
17268        }
17269        assert!(!data.is_empty(), "queued frames drain before the terminal");
17270        for pair in data.windows(2) {
17271            assert_eq!(
17272                pair[1],
17273                pair[0] + 1,
17274                "queued frames arrive dense and in order"
17275            );
17276        }
17277        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17278        assert_eq!(terminal.header.ty, FrameType::Error);
17279        assert_eq!(terminal.header.corr, 7);
17280        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17281        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17282        let detail = body.detail.expect("lagged error carries detail");
17283        assert_eq!(
17284            detail["first_undelivered_cursor"]["seq"],
17285            data.last().unwrap() + 1,
17286            "the named cursor is the first event the subscriber did not receive"
17287        );
17288        assert_eq!(
17289            detail["first_undelivered_cursor"]["daemon_incarnation"],
17290            "lag-incarnation"
17291        );
17292    }
17293}
17294
17295#[cfg(test)]
17296mod terminal_history_read_concurrency_tests {
17297    use super::*;
17298    use crate::terminal_journal::read_pause;
17299    use cortexkit_test_support::ScratchDir;
17300    use std::sync::mpsc as std_mpsc;
17301
17302    fn journaled_ring(
17303        journal: &Arc<crate::terminal_journal::TerminalJournal>,
17304    ) -> Arc<Mutex<TerminalRing>> {
17305        Arc::new(Mutex::new(
17306            TerminalRing::new(TerminalRingConfig::default(), 1)
17307                .with_journal(Some(Arc::clone(journal))),
17308        ))
17309    }
17310
17311    fn crash(at_ms: u64) -> ExitReport {
17312        ExitReport {
17313            kind: ExitKind::Crash,
17314            code: Some(1),
17315            signal: None,
17316            at_ms,
17317        }
17318    }
17319
17320    /// Record an exit on another thread and report whether it finished within
17321    /// `bound`. The recorder thread is left running if it did not.
17322    fn record_within(
17323        module_id: &'static str,
17324        ring: &Arc<Mutex<TerminalRing>>,
17325        at_ms: u64,
17326        bound: Duration,
17327    ) -> bool {
17328        let ring = Arc::clone(ring);
17329        let (done, done_rx) = std_mpsc::channel();
17330        std::thread::spawn(move || {
17331            record_terminal(
17332                module_id,
17333                &ring,
17334                &SpawnEventFeed::default(),
17335                &crash(at_ms),
17336                TerminalDisposition::Restarting,
17337            );
17338            let _ = done.send(());
17339        });
17340        done_rx.recv_timeout(bound).is_ok()
17341    }
17342
17343    /// A history read in progress must not hold the journal writer (which every
17344    /// module's exit recording needs) or the module's own ring. Exits recorded
17345    /// while the read is paused complete promptly; the paused read answers as of
17346    /// the moment it started, and the next read has each exit exactly once.
17347    #[test]
17348    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17349        let dir = ScratchDir::new("terminal-history-concurrent-read");
17350        let path = dir.join("terminals.jsonl");
17351        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17352            path.clone(),
17353            "daemon".into(),
17354        ));
17355        let reader_ring = journaled_ring(&journal);
17356        let other_ring = journaled_ring(&journal);
17357        assert!(record_within(
17358            "reader-module",
17359            &reader_ring,
17360            10,
17361            Duration::from_secs(5)
17362        ));
17363
17364        let (started, release) = read_pause::install(&path);
17365        let reading = {
17366            let ring = Arc::clone(&reader_ring);
17367            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17368        };
17369        started
17370            .recv_timeout(Duration::from_secs(5))
17371            .expect("the history read reached its pause");
17372
17373        let bound = Duration::from_secs(1);
17374        assert!(
17375            record_within("other-module", &other_ring, 20, bound),
17376            "another module's exit waited on a history read (journal writer held)"
17377        );
17378        assert!(
17379            record_within("reader-module", &reader_ring, 30, bound),
17380            "the read module's own exit waited on its history read (ring held)"
17381        );
17382
17383        drop(release);
17384        let paused = reading.join().unwrap();
17385        assert_eq!(
17386            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17387            vec![10],
17388            "an exit recorded after the read began lands in neither half of it"
17389        );
17390        assert_eq!(paused.journal_skipped_lines, 0);
17391        assert_eq!(paused.journal_read_errors, 0);
17392
17393        let after = durable_terminal_history_of(&reader_ring, "reader-module");
17394        assert_eq!(
17395            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17396            vec![10, 30],
17397            "the next read merges ring and journal with no duplicate"
17398        );
17399        assert_eq!(after.journal_skipped_lines, 0);
17400    }
17401}
17402
17403/// What a restart does with the exited process's output readers. These drive
17404/// the same settle functions the supervisor calls, with a reader the test
17405/// holds, so a reader that has not been scheduled by the bound is a controlled
17406/// input rather than something only a loaded machine produces.
17407#[cfg(test)]
17408mod stderr_settle_tests {
17409    use std::{
17410        future::Future,
17411        io,
17412        pin::Pin,
17413        sync::{Arc, Mutex},
17414        task::{Context, Poll},
17415        time::Duration,
17416    };
17417
17418    use tokio::{
17419        io::{AsyncRead, ReadBuf},
17420        sync::oneshot,
17421        time::Instant,
17422    };
17423
17424    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17425    use crate::stderr_tail::{
17426        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17427        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17428    };
17429
17430    const BOUND: Duration = Duration::from_millis(250);
17431
17432    /// Yields `before`, then stays pending until the gate is released, then
17433    /// yields `after` and reaches EOF. The bytes after the gate were written
17434    /// by a process that has already exited; only the reader is behind.
17435    struct HeldReader {
17436        before: Option<Vec<u8>>,
17437        gate: Option<oneshot::Receiver<()>>,
17438        after: io::Cursor<Vec<u8>>,
17439    }
17440
17441    impl AsyncRead for HeldReader {
17442        fn poll_read(
17443            mut self: Pin<&mut Self>,
17444            cx: &mut Context<'_>,
17445            buf: &mut ReadBuf<'_>,
17446        ) -> Poll<io::Result<()>> {
17447            if let Some(bytes) = self.before.take() {
17448                buf.put_slice(&bytes);
17449                return Poll::Ready(Ok(()));
17450            }
17451            if let Some(gate) = self.gate.as_mut() {
17452                match Pin::new(gate).poll(cx) {
17453                    Poll::Pending => return Poll::Pending,
17454                    Poll::Ready(_) => self.gate = None,
17455                }
17456            }
17457            Pin::new(&mut self.after).poll_read(cx, buf)
17458        }
17459    }
17460
17461    struct DiscardSink;
17462
17463    impl OutputSink for DiscardSink {
17464        fn write_line(&mut self, _line: &[u8]) {}
17465    }
17466
17467    fn line(text: &str) -> TailEntry {
17468        TailEntry::Line {
17469            text: text.to_string(),
17470            truncated: false,
17471            at_ms: None,
17472        }
17473    }
17474
17475    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17476        ring.lock().unwrap()
17477    }
17478
17479    /// Start a reader for a new process generation that delivers `before`
17480    /// immediately and `after` only once the returned sender fires (or is
17481    /// dropped).
17482    fn held_pump(
17483        ring: &Arc<Mutex<StderrRing>>,
17484        before: &str,
17485        after: &str,
17486    ) -> (StderrPump, oneshot::Sender<()>) {
17487        let generation = lock(ring).begin_process();
17488        let (release, gate) = oneshot::channel();
17489        let reader = HeldReader {
17490            before: Some(before.as_bytes().to_vec()),
17491            gate: Some(gate),
17492            after: io::Cursor::new(after.as_bytes().to_vec()),
17493        };
17494        let task = tokio::spawn(pump_stderr_to(
17495            reader,
17496            Arc::clone(ring),
17497            generation,
17498            DiscardSink,
17499        ));
17500        (StderrPump { task, generation }, release)
17501    }
17502
17503    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17504        for _ in 0..1000 {
17505            if done(&lock(ring)) {
17506                return;
17507            }
17508            tokio::time::sleep(Duration::from_millis(1)).await;
17509        }
17510        panic!(
17511            "ring never reached the expected state: {:?}",
17512            lock(ring).snapshot(None, None)
17513        );
17514    }
17515
17516    #[tokio::test(start_paused = true)]
17517    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17518        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17519        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17520
17521        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17522        let before_release = lock(&ring).snapshot(None, None);
17523        assert!(
17524            matches!(before_release.capture, CaptureState::Incomplete { .. }),
17525            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17526        );
17527
17528        // The restart: the next process starts and writes before the old
17529        // reader catches up.
17530        let next = lock(&ring).begin_process();
17531        lock(&ring).push_line_from(next, "next process booting");
17532        release.send(()).unwrap();
17533        wait_until(&ring, |ring| {
17534            ring.snapshot(None, None).capture == CaptureState::Captured
17535        })
17536        .await;
17537
17538        assert_eq!(
17539            untimed(lock(&ring).snapshot(None, None).entries),
17540            vec![
17541                line("booting"),
17542                line("config error: missing storage"),
17543                TailEntry::ProcessStart,
17544                line("next process booting"),
17545            ],
17546            "the crash's last line must survive a slow reader and stay in the crashed process's section"
17547        );
17548    }
17549
17550    #[tokio::test(start_paused = true)]
17551    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17552    ) {
17553        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17554        // `_held` is never fired: a descendant keeps the pipe open for the
17555        // whole test.
17556        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17557
17558        let started = Instant::now();
17559        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17560        assert_eq!(
17561            started.elapsed(),
17562            BOUND,
17563            "the restart must wait exactly the bound for a pipe that stays open, no longer"
17564        );
17565
17566        let next = lock(&ring).begin_process();
17567        lock(&ring).push_line_from(next, "next process booting");
17568        tokio::time::sleep(Duration::from_secs(60)).await;
17569
17570        let snapshot = lock(&ring).snapshot(None, None);
17571        match &snapshot.capture {
17572            CaptureState::Incomplete { reason } => assert!(
17573                reason.contains("had not reached EOF") && reason.contains("250ms"),
17574                "the reason must say what is missing and after how long: {reason}"
17575            ),
17576            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17577        }
17578        assert_eq!(
17579            untimed(snapshot.entries),
17580            vec![
17581                line("parent exiting"),
17582                TailEntry::ProcessStart,
17583                line("next process booting"),
17584            ]
17585        );
17586    }
17587
17588    #[tokio::test(start_paused = true)]
17589    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17590        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17591        let (pump, release) = held_pump(&ring, "one\n", "two\n");
17592        release.send(()).unwrap();
17593
17594        settle_stderr_pump("clean", &ring, pump, BOUND).await;
17595
17596        let snapshot = lock(&ring).snapshot(None, None);
17597        assert_eq!(snapshot.capture, CaptureState::Captured);
17598        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17599    }
17600
17601    #[tokio::test(start_paused = true)]
17602    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17603        const BURST: usize = 150;
17604        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17605        let path = capture.join("two-pipe.stderr.log");
17606        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17607        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17608        let generation = lock(&ring).begin_process();
17609        let stdout = (0..BURST)
17610            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17611            .collect::<String>();
17612        let stderr = (0..BURST)
17613            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17614            .collect::<String>();
17615        // Model a child that has written both bursts and exited. Hold the stdout
17616        // reader after one line and a partial prefix, as if pipe readiness or
17617        // capture-file writes had delayed it. The final line also needs EOF to
17618        // flush: neither complete nor partial buffered output may be discarded.
17619        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17620        let (release, gate) = oneshot::channel();
17621        let reader = HeldReader {
17622            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17623            gate: Some(gate),
17624            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17625        };
17626        let (finished, completion) = oneshot::channel();
17627        let stdout_sink = sink.clone();
17628        let pump = tokio::spawn(async move {
17629            pump_stdout_to(reader, stdout_sink).await;
17630            finished.send(()).unwrap();
17631        });
17632        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17633
17634        let started = Instant::now();
17635        settle_stdout_pump("two-pipe", pump, BOUND).await;
17636        assert_eq!(
17637            started.elapsed(),
17638            BOUND,
17639            "a late reader must not hold up restart"
17640        );
17641        assert_eq!(
17642            std::fs::read_to_string(&path).unwrap().lines().count(),
17643            BURST + 1
17644        );
17645
17646        release
17647            .send(())
17648            .expect("stdout reader must survive the exit drain bound");
17649        tokio::time::timeout(Duration::from_secs(1), completion)
17650            .await
17651            .expect("released stdout reader must reach EOF")
17652            .expect("stdout capture must finish at EOF, not be cancelled");
17653        let contents = std::fs::read_to_string(&path).unwrap();
17654        let mut lines = contents
17655            .lines()
17656            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17657            .collect::<Vec<_>>();
17658        let mut expected = (0..BURST)
17659            .flat_map(|i| {
17660                [
17661                    format!("out-{i:04}-{}", "o".repeat(64)),
17662                    format!("err-{i:04}-{}", "e".repeat(64)),
17663                ]
17664            })
17665            .collect::<Vec<_>>();
17666        lines.sort_unstable();
17667        expected.sort_unstable();
17668        assert_eq!(
17669            lines, expected,
17670            "both pipes must retain every complete line exactly once"
17671        );
17672    }
17673}
17674
17675/// Containment of a module's process tree (issue #109).
17676///
17677/// The behaviour these defend against is a module helper surviving its module:
17678/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
17679/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
17680/// compounds it.
17681///
17682/// They run against the SUPERVISOR rather than the job-object crate because the
17683/// claim is about teardown: a crate-level test proves a job can reap a tree, not
17684/// that the daemon's drain path reaches it.
17685///
17686/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
17687/// lane there is a separate containment path with its own tests.
17688#[cfg(all(test, windows))]
17689mod job_containment_tests {
17690    use super::*;
17691    use cortexkit_test_support::ScratchDir;
17692    use std::{
17693        path::{Path, PathBuf},
17694        sync::{Arc, Mutex},
17695        time::{Duration, Instant},
17696    };
17697
17698    /// The stub, expected beside this test executable.
17699    ///
17700    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17701    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17702    /// failure then reads as a broken test rather than an unbuilt dependency.
17703    fn stub_path() -> PathBuf {
17704        let mut path = std::env::current_exe().expect("current_exe available in tests");
17705        path.pop();
17706        path.pop();
17707        path.push("fake-aft-stub.exe");
17708        assert!(
17709            path.exists(),
17710            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17711             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17712            path.display()
17713        );
17714        path
17715    }
17716
17717    /// Poll for the grandchild pid the stub records, and parse it.
17718    fn read_grandchild_pid(path: &Path) -> u32 {
17719        let deadline = Instant::now() + Duration::from_secs(10);
17720        loop {
17721            if let Ok(contents) = std::fs::read_to_string(path) {
17722                if let Ok(pid) = contents.trim().parse() {
17723                    return pid;
17724                }
17725            }
17726            assert!(
17727                Instant::now() < deadline,
17728                "the stub never recorded a grandchild pid at {}",
17729                path.display()
17730            );
17731            std::thread::sleep(Duration::from_millis(10));
17732        }
17733    }
17734
17735    /// Everything one fixture run needs, so the two tests below differ in exactly
17736    /// one place: whether the child is contained.
17737    struct Fixture {
17738        _dir: ScratchDir,
17739        module_id: String,
17740        grandchild: u32,
17741        child: Option<SupervisedChild>,
17742        registry: Arc<Registry>,
17743        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17744        terminal_ring: Arc<Mutex<TerminalRing>>,
17745        spawn_events: SpawnEventFeed,
17746    }
17747
17748    fn fixture(label: &str, module_id: &str) -> Fixture {
17749        let dir = ScratchDir::new(label);
17750        let pid_file = dir.join("grandchild.pid");
17751        let supervisor = Supervisor::new_for_test(
17752            Arc::new(Registry::default()),
17753            RestartPolicy::new(3, Duration::ZERO),
17754        );
17755        let runtime = supervisor.runtime_config();
17756        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17757        let spec = ModuleSpec {
17758            module_id: module_id.to_string(),
17759            program: stub_path(),
17760            // Zero args deliberately: a `--subc` argument would make the stub dial
17761            // a daemon that is not there, and the failure would land in the same
17762            // stderr ring this fixture exists to keep quiet.
17763            args: Vec::new(),
17764            env: vec![
17765                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17766                (
17767                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17768                    pid_file.display().to_string(),
17769                ),
17770            ],
17771            reserved: false,
17772            reserved_prefixes: Vec::new(),
17773            protocol: ModuleProtocol::Subc,
17774            overlap: Default::default(),
17775            carry_containment: false,
17776        };
17777        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17778            .expect("spawn the supervised fixture");
17779        let grandchild = read_grandchild_pid(&pid_file);
17780        Fixture {
17781            _dir: dir,
17782            module_id: module_id.to_string(),
17783            grandchild,
17784            child: Some(child),
17785            registry: Arc::new(Registry::default()),
17786            snapshot,
17787            terminal_ring: Arc::clone(&runtime.terminal_ring),
17788            spawn_events: SpawnEventFeed::default(),
17789        }
17790    }
17791
17792    impl Fixture {
17793        /// Drain through the supervisor's own teardown path.
17794        async fn drain(&mut self) {
17795            let child = self
17796                .child
17797                .take()
17798                .expect("the fixture child is still present");
17799            drain_child_to_state(
17800                &self.module_id,
17801                ModuleProtocol::Subc,
17802                // No forwarding table in this fixture, so nothing reaches the
17803                // child over a connection.
17804                StopNotice::NotSent,
17805                &self.registry,
17806                None,
17807                &self.snapshot,
17808                &self.terminal_ring,
17809                &self.spawn_events,
17810                child,
17811                Duration::from_millis(500),
17812                ModuleState::Stopped,
17813                Some(false),
17814                &FailureContext::operator(),
17815            )
17816            .await
17817            .expect("drain the supervised fixture");
17818        }
17819    }
17820
17821    /// Teardown reaps the grandchild, not merely the direct child.
17822    ///
17823    /// This is the assertion the change exists for. Before containment the
17824    /// grandchild survived: it is a separate process, and `start_kill` is
17825    /// `TerminateProcess` scoped to one pid.
17826    #[tokio::test]
17827    async fn teardown_reaps_the_grandchild() {
17828        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17829        let grandchild = fixture.grandchild;
17830
17831        assert!(
17832            subc_jobobject::process_exists(grandchild),
17833            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17834        );
17835
17836        fixture.drain().await;
17837
17838        assert!(
17839            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17840            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17841        );
17842    }
17843
17844    /// The mutation control: with containment withheld, the grandchild survives
17845    /// the same kill.
17846    ///
17847    /// This is the defect reproduction from #109 — a direct-child kill reaches
17848    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17849    /// supervisor because `spawn_and_mark_running` now always contains on
17850    /// Windows, which is the point: there is no longer a path that spawns
17851    /// uncontained, so the control has to construct one.
17852    ///
17853    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17854    /// grandchild ever dies here, that test is passing for a reason unrelated to
17855    /// the job object and the containment claim is unproven.
17856    #[test]
17857    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17858        let dir = ScratchDir::new("teardown-uncontained");
17859        let pid_file = dir.join("grandchild.pid");
17860        let mut child = std::process::Command::new(stub_path())
17861            .env("FAKE_AFT_NEVER_CONNECT", "1")
17862            .env(
17863                "FAKE_AFT_GRANDCHILD_PID_FILE",
17864                pid_file.display().to_string(),
17865            )
17866            .stdin(std::process::Stdio::null())
17867            .stdout(std::process::Stdio::null())
17868            .stderr(std::process::Stdio::null())
17869            .spawn()
17870            .expect("spawn the uncontained fixture");
17871        let grandchild = read_grandchild_pid(&pid_file);
17872
17873        // Exactly what the pre-fix teardown did: kill the direct child.
17874        child.kill().expect("kill the direct child");
17875        let _ = child.wait();
17876
17877        assert!(
17878            subc_jobobject::process_exists(grandchild),
17879            "grandchild {grandchild} died with the direct child, so this control no longer \
17880             distinguishes contained from uncontained teardown and the regression test is \
17881             passing vacuously"
17882        );
17883
17884        // The orphan this control demonstrates is the leak the fix prevents, so
17885        // the control must not leave one behind.
17886        kill_tree(grandchild);
17887    }
17888
17889    /// Crash durability: closing the containment handle reaps the tree with no
17890    /// teardown code running at all.
17891    ///
17892    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17893    /// call anything — and it is why containment is a kernel property of the
17894    /// handle rather than a step in the drain. Discovered by getting the
17895    /// mutation control wrong: clearing `job` to "disable" containment instead
17896    /// killed the tree, which is the guarantee, not a mistake.
17897    #[tokio::test]
17898    async fn dropping_containment_reaps_the_grandchild() {
17899        let mut fixture = fixture("drop-containment", "tree-drop");
17900        let grandchild = fixture.grandchild;
17901
17902        assert!(subc_jobobject::process_exists(grandchild));
17903
17904        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17905        fixture.child.as_mut().expect("child present").job = None;
17906
17907        assert!(
17908            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17909            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17910             crash would leave the tree behind"
17911        );
17912    }
17913
17914    /// Kill a pid and its tree, then confirm it is gone.
17915    fn kill_tree(pid: u32) {
17916        let _ = std::process::Command::new("taskkill.exe")
17917            .args(["/PID", &pid.to_string(), "/T", "/F"])
17918            .stdin(std::process::Stdio::null())
17919            .stdout(std::process::Stdio::null())
17920            .stderr(std::process::Stdio::null())
17921            .status();
17922        assert!(
17923            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17924            "could not clean up grandchild {pid}"
17925        );
17926    }
17927}
17928
17929#[cfg(test)]
17930mod privacy_trampoline_configuration_tests {
17931    #[tokio::test]
17932    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17933    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17934        #[cfg(target_os = "macos")]
17935        {
17936            let supervisor = super::Supervisor::new(
17937                std::sync::Arc::new(crate::Registry::default()),
17938                super::RestartPolicy::default(),
17939            );
17940            let error = supervisor.spawn(spec()).unwrap_err();
17941            assert!(
17942                error
17943                    .to_string()
17944                    .contains("no privacy trampoline configured"),
17945                "{error}"
17946            );
17947        }
17948    }
17949
17950    #[tokio::test]
17951    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17952    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17953        #[cfg(target_os = "macos")]
17954        {
17955            let supervisor = super::Supervisor::new(
17956                std::sync::Arc::new(crate::Registry::default()),
17957                super::RestartPolicy::default(),
17958            )
17959            .with_privacy_trampoline(std::env::current_exe().unwrap());
17960            let error = supervisor.spawn(spec()).unwrap_err();
17961            assert!(
17962                error
17963                    .to_string()
17964                    .contains("binary does not implement the privacy trampoline protocol"),
17965                "{error}"
17966            );
17967        }
17968    }
17969
17970    #[cfg(target_os = "macos")]
17971    fn spec() -> super::ModuleSpec {
17972        super::ModuleSpec {
17973            module_id: "privacy-configuration".into(),
17974            program: "/bin/sleep".into(),
17975            args: vec!["30".into()],
17976            env: vec![],
17977            reserved: false,
17978            reserved_prefixes: vec![],
17979            protocol: subc_control::ModuleProtocol::None,
17980            overlap: super::ModuleOverlap::Exclusive,
17981            carry_containment: false,
17982        }
17983    }
17984}
17985
17986#[cfg(test)]
17987mod privacy_exec_boundary_tests {
17988    #[cfg(target_os = "macos")]
17989    use super::*;
17990    #[cfg(target_os = "macos")]
17991    use std::{
17992        io::{Read, Write},
17993        net::{TcpListener, TcpStream},
17994    };
17995
17996    /// Unit-test-only pause at the actual early image read, not at a later
17997    /// status read. Production supervisors never inspect this environment key.
17998    #[cfg(target_os = "macos")]
17999    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18000        if let Some((_, path)) = spec
18001            .env
18002            .iter()
18003            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18004        {
18005            let mut barrier = TcpStream::connect(path).unwrap();
18006            barrier
18007                .set_read_timeout(Some(Duration::from_secs(30)))
18008                .unwrap();
18009            barrier.write_all(&pid.to_ne_bytes()).unwrap();
18010            let mut release = [0];
18011            barrier.read_exact(&mut release).unwrap();
18012            assert_eq!(&release, b"X");
18013        }
18014    }
18015
18016    #[cfg(target_os = "macos")]
18017    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18018        ModuleSpec {
18019            module_id: "privacy-boundary".into(),
18020            program: program.into(),
18021            args: args.iter().map(|arg| (*arg).into()).collect(),
18022            env: vec![],
18023            reserved: false,
18024            reserved_prefixes: vec![],
18025            protocol: ModuleProtocol::None,
18026            overlap: ModuleOverlap::Exclusive,
18027            carry_containment: false,
18028        }
18029    }
18030
18031    #[cfg(target_os = "macos")]
18032    async fn accept(listener: TcpListener) -> TcpStream {
18033        // Socket readiness, not elapsed time, establishes both pause points.
18034        let listener = tokio::net::TcpListener::from_std({
18035            listener.set_nonblocking(true).unwrap();
18036            listener
18037        })
18038        .unwrap();
18039        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18040            .await
18041            .unwrap()
18042            .unwrap();
18043        let stream = stream.into_std().unwrap();
18044        stream.set_nonblocking(false).unwrap();
18045        stream
18046            .set_read_timeout(Some(Duration::from_secs(30)))
18047            .unwrap();
18048        stream
18049    }
18050
18051    #[tokio::test]
18052    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18053    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18054        #[cfg(target_os = "macos")]
18055        {
18056            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18057            // Loopback sockets also work when the replay adapter's TMPDIR is
18058            // longer than Darwin's Unix-domain socket path limit.
18059            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18060            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18061            let record = root.join("live-children.json");
18062            let supervisor =
18063                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18064                    .with_live_children_record(&record);
18065            let runtime = supervisor.runtime_config();
18066            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18067            let mut spec = spec("/bin/sleep", &["30"]);
18068            spec.env = vec![
18069                (
18070                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18071                    exec_listener.local_addr().unwrap().to_string(),
18072                ),
18073                (
18074                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18075                    sample_listener.local_addr().unwrap().to_string(),
18076                ),
18077            ];
18078            let spawn = tokio::task::spawn_blocking(move || {
18079                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18080            });
18081            let mut sample = accept(sample_listener).await;
18082            let mut pid = [0; 4];
18083            sample.read_exact(&mut pid).unwrap();
18084            let pid = u32::from_ne_bytes(pid);
18085            let mut exec = accept(exec_listener).await;
18086            let mut ready = [0];
18087            exec.read_exact(&mut ready).unwrap();
18088            assert_eq!(&ready, b"R");
18089            // The early read is guaranteed to see a real, non-null trampoline
18090            // image: the fixture has reached its barrier and cannot exec yet.
18091            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18092            assert_eq!(
18093                observe_spawned_image(pid).unwrap().executable,
18094                Some(trampoline)
18095            );
18096            sample.write_all(b"X").unwrap();
18097            let mut child = spawn.await.unwrap();
18098            let early = crate::live_children::read_record(&record).unwrap();
18099            assert_eq!(early.len(), 1);
18100            assert_eq!(early[0].pid, pid);
18101            assert_eq!(
18102                early[0].executable, None,
18103                "unconfirmed trampoline image entered the roster"
18104            );
18105            assert!(child.report_ready.get().is_none());
18106            // The barrier's duration is unrelated to the production five-second
18107            // exec budget. Start the test's confirmation budget upon release.
18108            child.privacy_exec.as_mut().unwrap().deadline =
18109                tokio::time::Instant::now() + Duration::from_secs(30);
18110            exec.write_all(b"X").unwrap();
18111            child.confirm_privacy_exec().await;
18112            assert_eq!(child.spawn_failure, None);
18113            assert!(child.report_ready.get().is_some());
18114            let confirmed = crate::live_children::read_record(&record).unwrap();
18115            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18116            assert_ne!(module, trampoline);
18117            assert_eq!(confirmed[0].executable, Some(module.into()));
18118            child.start_kill().unwrap();
18119            child.wait().await.unwrap();
18120            child.release_roster();
18121        }
18122    }
18123
18124    #[tokio::test]
18125    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18126    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18127        #[cfg(target_os = "macos")]
18128        {
18129            let registry = Arc::new(Registry::default());
18130            let policy = RestartPolicy::new(0, Duration::ZERO);
18131            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
18132            let runtime = supervisor.runtime_config();
18133            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18134            let spec = spec("/bin/sh", &["-c", "exit 121"]);
18135            // Drive spawn and confirmation separately instead of starting the
18136            // monitor. WNOWAIT observes a real exit without consuming its status,
18137            // so confirmation's first try_wait must take the already-exited arm.
18138            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18139            let pid = child.pid;
18140            tokio::task::spawn_blocking(move || {
18141                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18142            })
18143            .await
18144            .unwrap()
18145            .unwrap();
18146            child.privacy_exec.as_mut().unwrap().deadline =
18147                tokio::time::Instant::now() + Duration::from_secs(30);
18148            let status = child.wait().await.unwrap();
18149            assert_eq!(status.code(), Some(121));
18150            assert!(child.privacy_exec.is_none());
18151            assert!(
18152                child.report_ready.get().is_none(),
18153                "an exited module must not publish a live pid"
18154            );
18155            let report = classify_reaped_child_exit(&snapshot, &child, &status);
18156            on_child_exit(
18157                &spec,
18158                policy,
18159                &registry,
18160                &snapshot,
18161                &runtime.terminal_ring,
18162                &runtime.spawn_events,
18163                &runtime.child_roster,
18164                report,
18165                true,
18166            )
18167            .await;
18168            let state = lock_snapshot(&snapshot).unwrap();
18169            assert_eq!(state.state, ModuleState::Failed);
18170            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18171            assert_eq!(state.reported_pid(), None);
18172            drop(state);
18173            let history = runtime.terminal_ring.lock().unwrap().snapshot();
18174            assert_eq!(history.entries.len(), 1);
18175            let terminal = &history.entries[0];
18176            assert_eq!(terminal.exit_code, Some(121));
18177            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18178            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18179            assert_eq!(
18180                terminal.disposition_detail.as_deref(),
18181                Some(policy.budget_exhausted_detail().as_str()),
18182                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18183            );
18184            assert_eq!(child.spawn_failure, None);
18185            child.release_roster();
18186        }
18187    }
18188}
18189
18190/// The daemon's real spawn path hands a subc-wire child its launch nonce on
18191/// descriptor 3, without an environment copy. The shell records the nonce
18192/// and its environment after exec so these tests observe the real handover.
18193#[cfg(all(test, unix))]
18194mod launch_nonce_descriptor_tests {
18195    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18196    use crate::stderr_tail::{StderrRing, StderrTailConfig};
18197    use cortexkit_test_support::ScratchDir;
18198    use std::{
18199        path::PathBuf,
18200        sync::{Arc, Mutex},
18201        time::{Duration, Instant},
18202    };
18203
18204    async fn probe(role: super::SpawnRole) {
18205        let scratch = ScratchDir::new("launch-nonce-descriptor");
18206        let fd_copy = scratch.join("from-descriptor");
18207        let env_copy = scratch.join("environment");
18208        let script = format!(
18209            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18210            fd = fd_copy.display(), env = env_copy.display(),
18211        );
18212        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18213        let spec = ModuleSpec {
18214            module_id: "nonce-descriptor-probe".to_string(),
18215            program: PathBuf::from("/bin/sh"),
18216            args: vec!["-c".to_string(), script],
18217            env: vec![
18218                xdg("XDG_DATA_HOME"),
18219                xdg("XDG_RUNTIME_DIR"),
18220                xdg("XDG_CONFIG_HOME"),
18221                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18222            ],
18223            reserved: true,
18224            reserved_prefixes: Vec::new(),
18225            protocol: ModuleProtocol::Subc,
18226            overlap: Default::default(),
18227            carry_containment: false,
18228        };
18229        let handle = SupervisorHandle::new();
18230        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18231        let roster = ChildRoster::default();
18232        #[cfg(target_os = "macos")]
18233        {
18234            let path = super::test_privacy_trampoline();
18235            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18236        }
18237        let child = super::spawn_child_in_slot(
18238            &spec,
18239            None,
18240            Some(&handle),
18241            &ring,
18242            None,
18243            &roster,
18244            #[cfg(target_os = "linux")]
18245            None,
18246            role,
18247            matches!(role, super::SpawnRole::SwapCandidate),
18248        )
18249        .expect("spawn probe");
18250        let deadline = Instant::now() + Duration::from_secs(10);
18251        while !(fd_copy.exists() && env_copy.exists()) {
18252            assert!(Instant::now() < deadline, "probe never wrote its copies");
18253            tokio::time::sleep(Duration::from_millis(20)).await;
18254        }
18255        let nonce = std::fs::read_to_string(fd_copy).unwrap();
18256        assert!(!nonce.is_empty());
18257        let environment = std::fs::read_to_string(env_copy).unwrap();
18258        assert!(environment
18259            .lines()
18260            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18261        let copy = environment
18262            .lines()
18263            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18264        assert_eq!(
18265            copy, None,
18266            "Unix children must never receive the environment nonce"
18267        );
18268        if matches!(role, super::SpawnRole::Plain) {
18269            assert_eq!(
18270                handle.spawn_nonce(&spec.module_id).as_deref(),
18271                Some(nonce.as_str())
18272            );
18273        }
18274        drop(child);
18275    }
18276
18277    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18278    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18279        probe(super::SpawnRole::Plain).await;
18280    }
18281
18282    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18283    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18284        probe(super::SpawnRole::SwapCandidate).await;
18285    }
18286}
18287
18288#[cfg(all(test, target_os = "linux"))]
18289#[path = "supervise/cgroup_reap_tests.rs"]
18290mod cgroup_reap_tests;
18291
18292#[cfg(all(test, target_os = "linux"))]
18293mod cgroup_containment_tests {
18294    use super::*;
18295    use cortexkit_test_support::ScratchDir;
18296
18297    #[tokio::test]
18298    async fn linux_subtree_memory_covers_module_children() {
18299        let dir = ScratchDir::new("subtree-memory");
18300        let root = PathBuf::from(format!(
18301            "/sys/fs/cgroup/subc-memory-test-{}-{}",
18302            std::process::id(),
18303            unix_ms_now()
18304        ));
18305        if let Err(error) = std::fs::create_dir(&root) {
18306            assert!(
18307                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18308                "required cgroup test cannot execute: {error}"
18309            );
18310            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18311            return;
18312        }
18313        let placement = subc_cgroup::prepare_at(&root)
18314            .unwrap()
18315            .expect("delegated cgroup");
18316        let supervisor = Supervisor::new_for_test(
18317            Arc::new(Registry::default()),
18318            RestartPolicy::new(3, Duration::ZERO),
18319        )
18320        .with_cgroup_placement(Some(placement));
18321        let ready = dir.join("ready");
18322        let module = supervisor
18323            .spawn(ModuleSpec {
18324                module_id: "memory-tree".into(),
18325                program: PathBuf::from("/bin/sh"),
18326                args: vec![
18327                    "-c".into(),
18328                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18329                    "fixture".into(),
18330                    ready.display().to_string(),
18331                ],
18332                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18333                    .into_iter()
18334                    .map(|key| (key.into(), dir.display().to_string()))
18335                    .collect(),
18336                reserved: false,
18337                reserved_prefixes: Vec::new(),
18338                protocol: ModuleProtocol::None,
18339                overlap: Default::default(),
18340                carry_containment: false,
18341            })
18342            .unwrap();
18343        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18344        while !ready.exists() {
18345            assert!(
18346                tokio::time::Instant::now() < deadline,
18347                "children never started"
18348            );
18349            tokio::time::sleep(Duration::from_millis(10)).await;
18350        }
18351        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18352        let reading = &readings[0];
18353        assert!(
18354            reading.process_count.is_some_and(|count| count >= 4),
18355            "{reading:?}"
18356        );
18357        assert!(
18358            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18359            "{reading:?}"
18360        );
18361        module.stop().await.unwrap();
18362        drop(module);
18363        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18364        std::fs::remove_dir(&root).unwrap();
18365        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18366    }
18367
18368    fn running(pid: u32) -> bool {
18369        // An orphan can remain a zombie until the container init reaps it.
18370        std::fs::read_to_string(format!("/proc/{pid}/stat"))
18371            .ok()
18372            .and_then(|stat| {
18373                stat.rsplit_once(") ")
18374                    .map(|(_, rest)| rest.starts_with('Z'))
18375            })
18376            .is_some_and(|zombie| !zombie)
18377    }
18378
18379    #[tokio::test]
18380    async fn linux_teardown_reaps_the_grandchild() {
18381        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18382    }
18383
18384    #[tokio::test]
18385    async fn linux_shutdown_straggler_reaps_the_grandchild() {
18386        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18387    }
18388
18389    async fn teardown_tree(test_name: &str, shutdown: bool) {
18390        let dir = ScratchDir::new(test_name);
18391        let root = PathBuf::from(format!(
18392            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18393            std::process::id(),
18394            unix_ms_now()
18395        ));
18396        if let Err(error) = std::fs::create_dir(&root) {
18397            assert!(
18398                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18399                "required cgroup test cannot execute: {error}"
18400            );
18401            eprintln!(
18402                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18403                root.display()
18404            );
18405            return;
18406        }
18407        let placement = subc_cgroup::prepare_at(&root)
18408            .expect("prepare isolated kernel cgroup")
18409            .expect("isolated cgroup is delegated");
18410        let module_id = "tree-teardown";
18411        let module = placement
18412            .module_path(module_id)
18413            .expect("create isolated module cgroup");
18414        if !module.join("cgroup.kill").exists() {
18415            std::fs::remove_dir(&module).unwrap();
18416            std::fs::remove_dir(root.join("subc-modules")).unwrap();
18417            std::fs::remove_dir(&root).unwrap();
18418            assert!(
18419                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18420                "required cgroup.kill interface unavailable"
18421            );
18422            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18423            return;
18424        }
18425        let supervisor = Supervisor::new_for_test(
18426            Arc::new(Registry::default()),
18427            RestartPolicy::new(3, Duration::ZERO),
18428        )
18429        .with_cgroup_placement(Some(placement));
18430        let mut runtime = supervisor.runtime_config();
18431        runtime.child_roster = runtime
18432            .child_roster
18433            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18434        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18435        let pid_file = dir.join("grandchild.pid");
18436        let spec = ModuleSpec {
18437            module_id: module_id.to_string(),
18438            program: PathBuf::from("/bin/sh"),
18439            args: vec![
18440                "-c".into(),
18441                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18442                "fixture".into(),
18443                pid_file.display().to_string(),
18444            ],
18445            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18446                .into_iter()
18447                .map(|key| (key.to_string(), dir.display().to_string()))
18448                .collect(),
18449            reserved: false,
18450            reserved_prefixes: Vec::new(),
18451            protocol: ModuleProtocol::None,
18452            overlap: Default::default(),
18453            carry_containment: false,
18454        };
18455        let child =
18456            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18457        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18458        let grandchild: u32 = loop {
18459            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18460                if let Ok(pid) = contents.trim().parse() {
18461                    break pid;
18462                }
18463            }
18464            assert!(
18465                tokio::time::Instant::now() < deadline,
18466                "grandchild pid was not recorded"
18467            );
18468            tokio::time::sleep(Duration::from_millis(10)).await;
18469        };
18470        assert!(
18471            running(grandchild),
18472            "grandchild must be alive before teardown"
18473        );
18474        if shutdown {
18475            let mut child = child;
18476            crate::child_roster::end_children_for_daemon_shutdown(
18477                &runtime.child_roster,
18478                false,
18479                std::future::pending(),
18480            )
18481            .await;
18482            child.wait().await.expect("reap shutdown straggler");
18483        } else {
18484            drain_child_to_state(
18485                module_id,
18486                ModuleProtocol::None,
18487                StopNotice::NotSent,
18488                &Registry::default(),
18489                None,
18490                &snapshot,
18491                &runtime.terminal_ring,
18492                &SpawnEventFeed::default(),
18493                child,
18494                Duration::from_millis(100),
18495                ModuleState::Stopped,
18496                Some(false),
18497                &FailureContext::operator(),
18498            )
18499            .await
18500            .expect("real supervisor teardown");
18501        }
18502        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18503        while running(grandchild) && tokio::time::Instant::now() < deadline {
18504            tokio::time::sleep(Duration::from_millis(10)).await;
18505        }
18506        let survived = running(grandchild);
18507        // Kill a surviving grandchild so a failed test does not leave it behind.
18508        if survived {
18509            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18510            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18511            tokio::time::sleep(Duration::from_millis(100)).await;
18512        }
18513        if module.exists() {
18514            std::fs::remove_dir(&module).expect("remove empty module cgroup");
18515        }
18516        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18517        std::fs::remove_dir(&root).unwrap();
18518        assert!(
18519            !survived,
18520            "grandchild {grandchild} outlived module teardown"
18521        );
18522        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18523    }
18524}
18525
18526/// A slow live-children record write on the restart path must not stop the
18527/// daemon from answering the replacement module's authentication handshake.
18528#[cfg(all(test, unix))]
18529mod record_write_stall_tests {
18530    use super::*;
18531    use std::sync::atomic::{AtomicBool, Ordering};
18532    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18533
18534    /// `subc-client-rs` gives the whole handshake this long before the module
18535    /// exits with `Timeout { stage: ServerProof }`.
18536    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18537    /// How long the replacement's admission write is held: longer than the
18538    /// module's deadline, standing in for the delay a busy disk can add
18539    /// across the two syncs in that write.
18540    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18541
18542    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
18543    /// On macOS every module is launched through the privacy trampoline, and
18544    /// a module that is not a `#!` script is admitted only if its running
18545    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
18546    /// the shell soon replaces itself with `/bin/sleep`, which does not match
18547    /// the configured `/bin/sh`. The image is read only after the spawn's
18548    /// live-children record write, and here that write lasted long enough for
18549    /// the shell to have exec'd every time, so each process was refused and
18550    /// killed right after spawn, and the supervisor crash-looped it with
18551    /// backoff until its restart budget ran out. The restart this test
18552    /// measures would then never produce a module at all.
18553    fn sleeping_spec() -> ModuleSpec {
18554        ModuleSpec {
18555            module_id: "record-write-stall".into(),
18556            program: "/bin/sleep".into(),
18557            args: vec!["600".into()],
18558            env: vec![],
18559            reserved: false,
18560            reserved_prefixes: vec![],
18561            protocol: ModuleProtocol::None,
18562            overlap: Default::default(),
18563            carry_containment: false,
18564        }
18565    }
18566
18567    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18568    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18569        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18570        let supervisor =
18571            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18572                .with_live_children_record(home.join("live-children.json"));
18573        let module = supervisor.spawn(sleeping_spec()).unwrap();
18574        // `reported_pid` names a process only once it is confirmed as the
18575        // module (on macOS, once the trampoline's image check has passed), so
18576        // a process the daemon is about to refuse and kill never counts as
18577        // the running module, here or in the final wait below.
18578        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18579            loop {
18580                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18581                    .unwrap()
18582                    .reported_pid()
18583                {
18584                    return pid;
18585                }
18586                tokio::time::sleep(Duration::from_millis(10)).await;
18587            }
18588        })
18589        .await
18590        .expect("the first process was admitted as the module");
18591
18592        // The daemon's listener, served by this runtime's workers like the
18593        // supervisor's own tasks.
18594        let key = vec![0x42; 32];
18595        let daemon_id = [0x24; 16];
18596        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18597        let port = listener.local_addr().unwrap().port();
18598        let server = tokio::spawn(crate::server::serve_listener(
18599            listener,
18600            Arc::new(crate::router::Router::with_default_self_handler()),
18601            crate::server::ServerAuth::with_limits(
18602                key.clone(),
18603                daemon_id,
18604                "test",
18605                crate::server::DEFAULT_AUTH_DEADLINE,
18606                4,
18607            ),
18608        ));
18609
18610        // Hold the write that admits the replacement, once, and say when it
18611        // has started.
18612        let (held_tx, held_rx) = std::sync::mpsc::channel();
18613        let armed = AtomicBool::new(true);
18614        supervisor
18615            .child_roster
18616            .set_record_write_hook_for_test(move |live| {
18617                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18618                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18619                    let _ = held_tx.send(());
18620                    std::thread::sleep(HELD_RECORD_WRITE);
18621                }
18622            });
18623
18624        // The replacement module, as a separate process would be: its own
18625        // thread and runtime, so only the daemon's runtime can be stalled.
18626        let module_process = std::thread::spawn(move || {
18627            held_rx
18628                .recv_timeout(Duration::from_secs(30))
18629                .expect("the restart reached the replacement's record write");
18630            let conn = ConnectionInfo {
18631                schema: SCHEMA_VERSION,
18632                wire_version: None,
18633                endpoints: vec![Endpoint {
18634                    host: "127.0.0.1".to_owned(),
18635                    port,
18636                }],
18637                key,
18638                daemon_id,
18639                pid: std::process::id(),
18640                daemon_ver: "test".to_owned(),
18641            };
18642            tokio::runtime::Builder::new_current_thread()
18643                .enable_all()
18644                .build()
18645                .unwrap()
18646                .block_on(async {
18647                    let started = Instant::now();
18648                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18649                        .await
18650                        .unwrap();
18651                    let result =
18652                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18653                    (result, started.elapsed())
18654                })
18655        });
18656
18657        module.restart(Some(0)).await.unwrap();
18658        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18659            .await
18660            .unwrap();
18661        assert!(
18662            auth.is_ok(),
18663            "the daemon must answer a module's handshake while a restart waits on the \
18664             live-children record write; got {auth:?} after {elapsed:?}"
18665        );
18666
18667        tokio::time::timeout(Duration::from_secs(20), async {
18668            while lock_snapshot(&module.inner.snapshot)
18669                .unwrap()
18670                .reported_pid()
18671                .is_none_or(|pid| pid == old_pid)
18672            {
18673                tokio::time::sleep(Duration::from_millis(10)).await;
18674            }
18675        })
18676        .await
18677        .expect("the restart published its replacement");
18678        module.stop().await.unwrap();
18679        server.abort();
18680    }
18681}
18682
18683#[cfg(all(test, unix))]
18684mod retained_status_tests {
18685    use super::*;
18686
18687    #[tokio::test]
18688    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18689        use crate::live_children::{ContainerIdentity, RetainedReason};
18690        for (reason, carried, expected) in [
18691            (
18692                RetainedReason::ContainerSurvivors(vec![4242]),
18693                true,
18694                Some("waiting: container survivors (pids [4242])".to_owned()),
18695            ),
18696            (
18697                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18698                true,
18699                Some(
18700                    "waiting: container survivors could not be verified: member inspection denied"
18701                        .to_owned(),
18702                ),
18703            ),
18704            (RetainedReason::Survived, false, None),
18705        ] {
18706            let supervisor =
18707                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18708            let spec = ModuleSpec {
18709                module_id: "retained-status-fixture".into(),
18710                program: PathBuf::from("/not-spawned"),
18711                args: Vec::new(),
18712                env: Vec::new(),
18713                reserved: false,
18714                reserved_prefixes: Vec::new(),
18715                protocol: ModuleProtocol::None,
18716                overlap: ModuleOverlap::Exclusive,
18717                carry_containment: carried,
18718            };
18719            let module = supervisor
18720                .supervise_configured(spec.clone(), false)
18721                .unwrap();
18722            // No process is launched. Invalid group zero is deliberate: if a
18723            // status read wrongly rechecks, verification changes the reason
18724            // and fails the assertions without signalling any real process.
18725            let guard = supervisor
18726                .child_roster
18727                .admit(
18728                    spec.module_id,
18729                    0,
18730                    ModuleProtocol::None,
18731                    None,
18732                    crate::child_roster::RecordedIdentity {
18733                        container: carried.then_some(ContainerIdentity {
18734                            pgid: Some(0),
18735                            created_start_time: Some(0),
18736                            cgroup_path: None,
18737                            ..ContainerIdentity::test_metadata()
18738                        }),
18739                        ..crate::child_roster::RecordedIdentity::default()
18740                    },
18741                )
18742                .unwrap();
18743            guard.retain(reason);
18744            let status = module.status().unwrap();
18745            assert_eq!(status.waiting_for_previous_detail, expected.clone());
18746            assert_eq!(status.health.detail, expected);
18747            assert!(!status.process_alive);
18748        }
18749    }
18750}