Skip to main content

subc_daemon/
supervise.rs

1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5    collections::{HashMap, HashSet, VecDeque},
6    error::Error,
7    fmt, io,
8    path::PathBuf,
9    process::{ExitStatus, Stdio},
10    sync::{Arc, Mutex, OnceLock},
11    time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22    manifest::{SelfSignalKind, SignalAnchor},
23    session::{
24        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25        MODULE_CONTROL_OP_HEALTH_CHECK,
26    },
27    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30    process::{Child, Command},
31    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32    task::JoinHandle,
33    time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38    child_roster::ChildRoster,
39    daemon_config::{
40        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41    },
42    forwarding::{
43        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44        ModuleDrainTarget, PendingModuleControlRpc,
45    },
46    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47    registry::{ConnectionId, RegistrationEndReason, RegistryError},
48    stderr_tail::{
49        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50        StderrTailSnapshot,
51    },
52    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53    Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75mod host_health;
76use host_health::HostHealth;
77
78#[cfg(all(test, unix))]
79mod slow_retry_tests;
80
81/// Command-line flag used by supervised modules to find subc.
82///
83/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
84/// The path points at the TCP+key connection file; it is not an ambient signal and
85/// is never inherited by standalone children.
86pub const SUBC_ARG: &str = "--subc";
87
88const DEFAULT_MAX_RESTARTS: u32 = 3;
89const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
90const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
91/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
92/// to contain a real crash loop (which respawns in seconds) and short enough
93/// that unrelated crashes hours apart never accumulate into a permanent stop.
94const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
95/// How long a drain waits for already-dispatched requests to finalize before
96/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
97/// not RPC-scale: the original 2s value silently cut nearly every real tool
98/// call at the fence, making the wait-for-finalize design decorative for the
99/// workloads it existed for. Quiescence short-circuits, so an idle module
100/// restarts immediately regardless of this value; the budget is spent only
101/// when a genuine in-flight request is worth finishing. Per-module override:
102/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
103/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
104/// where a stuck request will never settle).
105pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
106pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
107const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
108const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
109/// How long a drain waits after one busy-gauge probe completes before it asks
110/// again.
111///
112/// A drain re-probes on this short interval of its own rather than on the
113/// module's health cadence. The default cadence (30 s) equals the default
114/// drain ceiling, so a cadence-paced drain that found the module busy at its
115/// first probe never asked a second time: it waited out the whole ceiling and
116/// forced GOODBYE even after the gauges had read zero. The interval is counted
117/// from the moment the previous probe finished, and probes run one at a time,
118/// so a slow module is never asked more often than it can answer. Periodic
119/// health checks outside a drain keep using the configured cadence.
120const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
121/// How long a restart waits for the exited process's output readers.
122///
123/// The restart does not depend on either output reader finishing. A stderr
124/// reader still running at this bound is left running, and its later output goes
125/// into the exited process's own section of the stderr ring (see
126/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
127/// bound no longer decides whether a crash's last lines are kept: under load
128/// the reader may simply not have been scheduled yet, and cutting it there
129/// discarded exactly the lines that explained the crash.
130///
131/// What the bound still decides is when the tail starts reporting
132/// `Incomplete`: a pipe open past it usually means a descendant of the exited
133/// process still holds it, and the tail cannot claim to be whole until that
134/// pipe closes. That is also the only case in which the wait costs the restart
135/// anything, because a pipe with no other holder reaches EOF when the process
136/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
137/// `Captured` at EOF with nothing lost.
138///
139/// The stdout reader carries no ring, only the capture file. It also continues
140/// to EOF after this bound: child exit closes the writer but does not mean the
141/// reader has consumed all buffered bytes. Late output from either pipe can
142/// therefore reach the shared capture file after the successor starts.
143const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
144/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
145pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
146const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
147/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
148/// dropped because the subscriber stopped draining its frames. The detail's
149/// `first_undelivered_cursor` names the first event it did not receive; the
150/// client resubscribes from the last cursor it did receive.
151pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
152
153#[cfg(target_os = "macos")]
154struct PrivacyExec {
155    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
156    deadline: tokio::time::Instant,
157    expected: Option<subc_os::FileIdentity>,
158    trampoline: Option<subc_os::FileIdentity>,
159    script: bool,
160    module_id: String,
161    #[cfg(test)]
162    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
163}
164
165#[cfg(all(test, target_os = "macos"))]
166pub(crate) fn test_privacy_trampoline() -> PathBuf {
167    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
168    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
169    let path = std::env::current_exe()
170        .unwrap()
171        .parent()
172        .unwrap()
173        .parent()
174        .unwrap()
175        .join("privacy-trampoline-fixture");
176    // Without the fixture every macOS spawn is refused, and the tests that
177    // spawn fail later as a module in state Failed, which names the wrong
178    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
179    assert!(
180        path.exists(),
181        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
182         --bins --features test-support` or `cargo test -p subc-daemon` first",
183        path.display()
184    );
185    path
186}
187
188#[cfg(target_os = "macos")]
189fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
190    use std::io::Read;
191    let mut probe = std::process::Command::new(path)
192        .args(["__disclaim-exec", "--probe"])
193        .stdin(Stdio::null())
194        .stdout(Stdio::piped())
195        .stderr(Stdio::piped())
196        .spawn()
197        .map_err(|error| {
198            format!(
199                "privacy trampoline probe failed for {}: {error}",
200                path.display()
201            )
202        })?;
203    let deadline = std::time::Instant::now() + Duration::from_secs(5);
204    let status = loop {
205        match probe.try_wait() {
206            Ok(Some(status)) => break status,
207            Ok(None) if std::time::Instant::now() < deadline => {
208                std::thread::sleep(Duration::from_millis(5))
209            }
210            result => {
211                let _ = probe.kill();
212                let _ = probe.wait();
213                return Err(format!(
214                    "privacy trampoline probe failed or timed out for {}: {result:?}",
215                    path.display()
216                ));
217            }
218        }
219    };
220    let mut answer = String::new();
221    if let Some(stdout) = probe.stdout.take() {
222        let _ = stdout.take(256).read_to_string(&mut answer);
223    }
224    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
225        return Ok(());
226    }
227    let mut diagnostic = String::new();
228    if let Some(stderr) = probe.stderr.take() {
229        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
230    }
231    let cause = diagnostic
232        .trim()
233        .strip_prefix("ck-subc: own privacy identity refused: ")
234        .unwrap_or("binary does not implement the privacy trampoline protocol");
235    Err(format!(
236        "{cause}: probe of {} exited {status}",
237        path.display()
238    ))
239}
240
241#[cfg(target_os = "macos")]
242fn privacy_command(
243    spec: &ModuleSpec,
244    roster: &ChildRoster,
245) -> Result<
246    (
247        Command,
248        Option<PrivacyExec>,
249        subc_os::privacy_identity::ExecAcknowledgement,
250    ),
251    SuperviseError,
252> {
253    let failure = |cause: String| {
254        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
255        SuperviseError::Spawn {
256            program: spec.program.clone(),
257            source: io::Error::other(cause),
258            cgroup_path: None,
259        }
260    };
261    let trampoline = roster.privacy_trampoline().map_err(failure)?;
262    // Resolve PATH with the same environment the Command will receive. For
263    // scripts retain the existing orphan-identity rule: the kernel chooses
264    // the interpreter, and its observed image is the one recorded. Do not
265    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
266    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
267        let path = spec
268            .env
269            .iter()
270            .find(|(key, _)| key == "PATH")
271            .map(|(_, value)| std::ffi::OsString::from(value))
272            .or_else(|| std::env::var_os("PATH"))
273            .unwrap_or_else(|| "/usr/bin:/bin".into());
274        std::env::split_paths(&path)
275            .map(|dir| dir.join(&spec.program))
276            .find(|path| path.is_file())
277            .unwrap_or_else(|| spec.program.clone())
278    } else {
279        spec.program.clone()
280    };
281    let expected = subc_os::file_identity(&program);
282    let trampoline_image = subc_os::file_identity(&trampoline);
283    let script = {
284        use std::io::Read;
285        let mut prefix = [0u8; 2];
286        std::fs::File::open(&program)
287            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
288    };
289    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
290        return Err(failure(
291            "privacy identity module executable is missing or is the trampoline itself".to_string(),
292        ));
293    }
294    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
295        .map_err(|error| failure(error.to_string()))?;
296    let reader =
297        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
298    let mut command = Command::new(&trampoline);
299    command
300        .arg("__disclaim-exec")
301        .arg(ack.fd().to_string())
302        .arg(&program);
303    Ok((
304        command,
305        Some(PrivacyExec {
306            reader,
307            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
308            expected,
309            trampoline: trampoline_image,
310            script,
311            module_id: spec.module_id.clone(),
312            #[cfg(test)]
313            confirmation_pause: spec
314                .env
315                .iter()
316                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
317                .map(|(_, address)| (address.clone(), None)),
318        }),
319        ack,
320    ))
321}
322
323struct SupervisedChild {
324    subtree_target: crate::subtree_resources::Target,
325    child: containment::ProcessChild,
326    exec: crate::activity::ExecOwner,
327    #[cfg(windows)]
328    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
329    #[cfg(target_os = "macos")]
330    privacy_exec: Option<PrivacyExec>,
331    /// Set once this launch's exec acknowledgement confirms the module image.
332    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
333    /// `subc_os::privacy_identity`), which then replaces itself with the
334    /// module. The supervisor owns and can kill that pid from spawn, but
335    /// status readers report it only after this latch is set, so nothing
336    /// reports the trampoline's image as the module's.
337    #[cfg(target_os = "macos")]
338    report_ready: Arc<OnceLock<()>>,
339    /// Supervisor failure detail retained for the terminal record, including
340    /// image refusals and first-HELLO timeouts after a successful spawn.
341    spawn_failure: Option<String>,
342    /// The protocol this process was launched with. A reload can store a new
343    /// launch spec with a different protocol, but that takes effect only at the
344    /// next spawn, so this process keeps being handled by the protocol it
345    /// actually speaks.
346    protocol: ModuleProtocol,
347    /// This container's cgroup name: a bounded module/slot label followed by a
348    /// unique suffix (when cgroup placement is on). Carried replacements reuse
349    /// the name; ordinary launches each get a fresh container. A
350    /// retired process in a slot may still be draining when a later one is
351    /// spawned into that slot, so the suffix keeps the later process out of
352    /// the retired one's cgroup, which is the domain a kill applies to.
353    #[cfg(target_os = "linux")]
354    module_id: String,
355    #[cfg(target_os = "linux")]
356    cgroup_placement: Option<subc_cgroup::Placement>,
357    /// The job for an ordinary, non-carried launch. For carried
358    /// launches the actor owns it and the child holds only a weak container link.
359    ///
360    /// Dropping this handle is what reaps a surviving tree when no supervisor
361    /// code runs — a daemon crash — because the job carries
362    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
363    ///
364    /// That limit is not crash-only, and the difference is worth knowing: a
365    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
366    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
367    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
368    /// module at once. Before this change they survived that, saw EOF on the
369    /// control socket, and ran their own teardown; Unix keeps that path
370    /// deliberately, so a module can seal a WAL or close a capture rather than
371    /// be killed mid-write. So this trades graceful teardown on every Windows
372    /// daemon stop for containment on a crash, which is the right way round
373    /// today: orphaned GPU workers are a reported, recurring problem, and the
374    /// modules that write most heavily do not run on Windows.
375    ///
376    /// The fix is a real Windows stop path — the daemon draining before it
377    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
378    /// reaches only what the drain left behind, which is what it should reach.
379    #[cfg(windows)]
380    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
381    stdout_pump: Option<JoinHandle<()>>,
382    stderr_pump: Option<StderrPump>,
383    stderr_ring: Arc<Mutex<StderrRing>>,
384    /// Conservative monotonic bound sampled before `command.spawn()`, not when
385    /// identity recording finishes or the spawn generation is published.
386    spawned_at: Instant,
387    /// Ordinary spawns must send their first HELLO within this budget. Once
388    /// accepted, a later disconnect must not restart the registration clock.
389    registration_deadline: Option<Instant>,
390    registration_watch: Option<RegistrationWatch>,
391    spawned_at_ms: u64,
392    spawned_from: PathBuf,
393    launch_spec_digest: LaunchSpecDigest,
394    spawned_file_identity: Option<SpawnedFileIdentity>,
395    process_start_time: Option<u64>,
396    process_identity: Option<ProcessIdentity>,
397    pid: u32,
398    /// This process's entry in the daemon's child roster, released when the
399    /// process is reaped or this handle is dropped.
400    roster_guard: Option<crate::child_roster::RosterGuard>,
401}
402
403impl SupervisedChild {
404    fn registered_since_spawn(&self) -> bool {
405        self.registration_watch
406            .as_ref()
407            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
408    }
409
410    fn id(&self) -> Option<u32> {
411        Some(self.pid)
412    }
413
414    fn process_identity(&self) -> Option<ProcessIdentity> {
415        self.process_identity
416    }
417
418    async fn wait(&mut self) -> io::Result<ExitStatus> {
419        #[cfg(target_os = "macos")]
420        self.confirm_privacy_exec().await;
421        // The roster entry is NOT released here. A daemon shutdown waits for the
422        // roster to empty and then exits the process, so releasing at the reap
423        // let it exit before the exit handler wrote this child's terminal record
424        // (the stderr drain and snapshot update sit in between), and the
425        // shutdown's own `daemon_shutdown` record was intermittently lost. The
426        // caller releases it after recording the exit (`release_roster`), and
427        // dropping the handle releases it too.
428        let result = self.child.wait().await;
429        #[cfg(target_os = "linux")]
430        if result.is_ok() && self.child.container().is_none() {
431            if let Some(placement) = self.cgroup_placement.as_ref() {
432                // Ordinary children still own a per-spawn cgroup: returning
433                // from wait must leave it empty and removed, as before carry.
434                // Keep cleanup ownership across cancellation until it finishes.
435                cleanup_reaped_cgroup(placement, &self.module_id).await;
436                self.cgroup_placement = None;
437            }
438        }
439        result
440    }
441
442    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
443        if self.child.container().is_some() {
444            if !carry {
445                self.child.terminate_container().await;
446            }
447        } else {
448            #[cfg(target_os = "linux")]
449            if let Some(placement) = self.cgroup_placement.as_ref() {
450                cleanup_reaped_cgroup(placement, &self.module_id).await;
451                self.cgroup_placement = None;
452            }
453        }
454        self.child.reap().await?;
455        self.release_roster();
456        Ok(())
457    }
458
459    #[cfg(target_os = "macos")]
460    async fn confirm_privacy_exec(&mut self) {
461        let Some(pending) = &mut self.privacy_exec else {
462            return;
463        };
464        let result = tokio::time::timeout_at(pending.deadline, async {
465            let mut record = Vec::new();
466            loop {
467                let mut ready = pending.reader.readable().await?;
468                let read = ready.try_io(|reader| {
469                    use std::io::Read;
470                    let mut reader = reader.get_ref();
471                    let mut buffer = [0u8; 256];
472                    reader.read(&mut buffer).map(|count| (count, buffer))
473                });
474                match read {
475                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
476                    Ok(Ok((count, buffer))) => {
477                        if record.len() + count > 1024 {
478                            return Err(io::Error::other(
479                                "privacy exec refusal record is too long",
480                            ));
481                        }
482                        record.extend_from_slice(&buffer[..count]);
483                    }
484                    Ok(Err(error)) => return Err(error),
485                    Err(_) => continue,
486                }
487            }
488        })
489        .await;
490        #[cfg(test)]
491        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
492            use tokio::io::{AsyncReadExt, AsyncWriteExt};
493            if stream.is_none() {
494                *stream = Some(
495                    tokio::net::TcpStream::connect(address.as_str())
496                        .await
497                        .unwrap(),
498                );
499                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
500                stream
501                    .as_mut()
502                    .unwrap()
503                    .write_all(&self.pid.to_ne_bytes())
504                    .await
505                    .unwrap();
506            }
507            let mut reply = [0];
508            stream
509                .as_mut()
510                .unwrap()
511                .read_exact(&mut reply)
512                .await
513                .unwrap();
514            pending.confirmation_pause = None;
515            if &reply == b"F" {
516                Ok(Err(io::Error::other("injected late exec refusal")))
517            } else {
518                assert_eq!(&reply, b"X");
519                result
520            }
521        } else {
522            result
523        };
524        // Keep the reader in self across await: select cancellation must not
525        // discard the handshake or reset its original five-second deadline.
526        let pending = self.privacy_exec.as_ref().expect("pending exec");
527        let cause = match result {
528            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
529            Ok(Err(error)) => Some(format!(
530                "privacy identity exec acknowledgement failed: {error}"
531            )),
532            Ok(Ok(record)) if !record.is_empty() => Some(
533                std::str::from_utf8(&record)
534                    .ok()
535                    .and_then(|record| {
536                        record
537                            .trim()
538                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
539                    })
540                    .filter(|cause| !cause.is_empty())
541                    .unwrap_or("invalid privacy exec refusal record")
542                    .to_string(),
543            ),
544            Ok(Ok(_)) => match self.child.try_wait() {
545                // Empty EOF is the exec acknowledgement. A real module may exit
546                // immediately, including with a reserved trampoline status; no
547                // image is admitted, and its ordinary exit contract stays intact.
548                Ok(Some(_status)) => None,
549                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
550                Ok(None) => {
551                    let image = observe_spawned_image(self.pid);
552                    if let Some(image) = image.filter(|image| {
553                        image.executable.is_some()
554                            && image.executable != pending.trampoline
555                            && (image.executable == pending.expected || pending.script)
556                    }) {
557                        if let Err(error) = self
558                            .roster_guard
559                            .as_ref()
560                            .map(|guard| guard.confirm_executable(image))
561                            .transpose()
562                        {
563                            Some(format!(
564                                "could not durably record confirmed module image: {error}"
565                            ))
566                        } else {
567                            let _ = self.report_ready.set(());
568                            info!(module_id = %pending.module_id, pid = self.pid,
569                                "module spawned with own privacy identity (responsibility disclaimed)");
570                            None
571                        }
572                    } else if image.is_none()
573                        || image.is_some_and(|image| image.executable.is_none())
574                    {
575                        // A process can exit between try_wait and the kernel
576                        // image read. Empty EOF already acknowledged exec, so
577                        // preserve that module's ordinary exit rather than
578                        // mislabel a disappearing image as trampoline refusal.
579                        // Keep pending in self across await so cancellation does
580                        // not discard validation or reset its original deadline.
581                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
582                            Ok(Ok(_status)) => None,
583                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
584                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
585                        }
586                    } else {
587                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
588                    }
589                }
590            },
591        };
592        let pending = self.privacy_exec.take().expect("pending exec");
593        // Exec confirmation describes launch success, not roster retirement.
594        // Keep even a reaped child recorded until its terminal history exists.
595        self.exec.0.finish(if cause.is_some() {
596            crate::activity::ExecState::Refused
597        } else {
598            crate::activity::ExecState::CommittedExec
599        });
600        if let Some(cause) = cause {
601            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
602            self.spawn_failure = Some(cause);
603            // No image is admitted on failure. Reach the entire fresh process
604            // group, including a module which spawned a helper before refusal.
605            if let Some(container) = self.child.container() {
606                container.hard_kill();
607            } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
608                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
609            }
610            let _ = self.child.start_kill();
611        }
612    }
613
614    /// Releases the leader's roster entry after recording and finalizing its
615    /// exit. A carried container retains its separate roster entry and, for a
616    /// Unix process group, its unreaped first leader that reserves the group ID.
617    fn release_roster(&mut self) {
618        self.roster_guard = None;
619    }
620
621    /// Kill the child and, where containment is available, its process tree.
622    ///
623    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
624    /// helper process leaked the helper — the Synapse embedding module's CUDA
625    /// worker holds the GPU allocation, so the leak cost VRAM until the next
626    /// restart of something else. Terminating the job reaches grandchildren that
627    /// a tree walk cannot, including one whose parent has already exited and
628    /// been reparented away.
629    ///
630    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
631    /// direct-child kill still decides the outcome, so containment can never
632    /// change whether a module is reported as stopped.
633    fn start_kill(&mut self) -> io::Result<()> {
634        if let Some(container) = self.child.container() {
635            container.hard_kill();
636            return self.kill_leader();
637        }
638        #[cfg(windows)]
639        if let Some(job) = &self.job {
640            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
641                debug!(
642                    error = %error,
643                    "job termination failed; the direct-child kill still owns the outcome"
644                );
645            }
646        }
647        #[cfg(target_os = "linux")]
648        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
649        self.child.start_kill()
650    }
651
652    fn kill_leader(&mut self) -> io::Result<()> {
653        self.child.start_kill()
654    }
655
656    async fn drain_stderr(&mut self, module_id: &str) {
657        if let Some(pump) = self.stdout_pump.take() {
658            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
659        }
660
661        let Some(pump) = self.stderr_pump.take() else {
662            return;
663        };
664        settle_stderr_pump(
665            module_id,
666            &self.stderr_ring,
667            pump,
668            STDERR_PUMP_DRAIN_TIMEOUT,
669        )
670        .await;
671    }
672}
673
674/// Bound the restart's wait, not the lifetime of the exited child's stdout
675/// reader. Slow pipe reads or capture writes must not turn the bound into lost
676/// output; dropping the handle lets the reader finish at EOF, like stderr.
677async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
678    match timeout(bound, &mut pump).await {
679        Ok(Ok(())) => {}
680        Ok(Err(error)) => {
681            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
682        }
683        Err(_) => {
684            drop(pump);
685            warn!(
686                module_id,
687                waited = ?bound,
688                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
689            );
690        }
691    }
692}
693
694/// The reader task for one process's stderr, with the ring generation its
695/// lines are attributed to.
696struct StderrPump {
697    task: JoinHandle<()>,
698    generation: u64,
699}
700
701/// Retire an exited process's stderr reader and wait up to `bound` for it to
702/// reach EOF. A reader still running at the bound is detached, not stopped: it
703/// keeps filling the exited process's section of the ring until its pipe
704/// closes, and the tail reads `Incomplete` until then. See
705/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
706async fn settle_stderr_pump(
707    module_id: &str,
708    ring: &Arc<Mutex<StderrRing>>,
709    pump: StderrPump,
710    bound: Duration,
711) {
712    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
713    let StderrPump {
714        mut task,
715        generation,
716    } = pump;
717    lock().retire_pump(generation);
718    match timeout(bound, &mut task).await {
719        Ok(Ok(())) => {}
720        Ok(Err(err)) => {
721            let mut ring = lock();
722            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
723            ring.finish_pump(generation);
724            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
725        }
726        Err(_) => {
727            // Dropping the handle detaches the task; it ends at EOF on its pipe.
728            drop(task);
729            lock().mark_pump_late(
730                generation,
731                format!(
732                    "stderr of the exited process had not reached EOF {bound:?} after it was \
733                     retired (a descendant may still hold the pipe open); lines it still \
734                     writes are kept in that process's section"
735                ),
736            );
737            warn!(
738                module_id,
739                waited = ?bound,
740                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
741            );
742        }
743    }
744}
745
746fn registration_release_events() -> &'static watch::Sender<u64> {
747    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
748    EVENTS.get_or_init(|| {
749        let (sender, _receiver) = watch::channel(0);
750        sender
751    })
752}
753
754pub(crate) fn notify_registration_release() {
755    let events = registration_release_events();
756    let next_generation = (*events.borrow()).wrapping_add(1);
757    events.send_replace(next_generation);
758}
759
760/// How to launch one singleton module process.
761#[derive(Debug, Clone, PartialEq, Eq)]
762pub struct ModuleSpec {
763    pub module_id: String,
764    pub program: PathBuf,
765    pub args: Vec<String>,
766    pub env: Vec<(String, String)>,
767    /// When true this is a reserved module: each spawn gets a fresh one-time launch
768    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
769    /// process can register this module_id (a security-boundary module like the
770    /// credential vault must not be impersonable while it is down/restarting).
771    pub reserved: bool,
772    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
773    /// Prefixes come from daemon config and must end in `:` before they reach the
774    /// supervisor; the owner module's current spawn nonce authorizes claims under
775    /// each prefix.
776    pub reserved_prefixes: Vec<String>,
777    /// The wire protocol this module speaks, as DECLARED in daemon config.
778    ///
779    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
780    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
781    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
782    /// and NO launch nonce, and a clean exit the daemon did not request is
783    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
784    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
785    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
786    /// because a process ignores an environment variable it does not read.
787    ///
788    /// The argument is the part that cannot be "harmless to a process that
789    /// ignores it": a stock binary exits on an unknown flag before it listens
790    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
791    /// first conformance run against this mode found it. The nonce is withheld
792    /// because a process that will never present it gains nothing from holding
793    /// it, and a secret in the environment of a process that does not need it is
794    /// a leak surface for no benefit.
795    pub protocol: ModuleProtocol,
796    /// Whether two processes of this module may run at once, which is what a
797    /// blue/green swap does for the length of its overlap. Declared in daemon
798    /// config because the daemon must be able to answer it while the module is
799    /// down, and so a module cannot talk itself into it after registering.
800    pub overlap: ModuleOverlap,
801    /// Keep module-started helpers in the same container across replacements.
802    /// This is a daemon-config policy, fixed for the lifetime of a container;
803    /// stopping the module or losing its supervisor still destroys the container.
804    pub carry_containment: bool,
805}
806
807/// Only the configured launch inputs enter this value; spawned connection paths,
808/// nonces and inherited environment never do. Its bytes remain daemon-private.
809#[derive(Clone, PartialEq, Eq)]
810pub(crate) struct LaunchSpecDigest([u8; 32]);
811
812impl fmt::Debug for LaunchSpecDigest {
813    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
814        f.write_str("LaunchSpecDigest([redacted])")
815    }
816}
817
818impl LaunchSpecDigest {
819    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
820        fn field(hash: &mut Sha256, bytes: &[u8]) {
821            hash.update((bytes.len() as u64).to_le_bytes());
822            hash.update(bytes);
823        }
824        let mut hash = Sha256::new();
825        if let Some(program) = spec.program.to_str() {
826            field(&mut hash, b"utf8-program");
827            field(&mut hash, program.as_bytes());
828        } else {
829            // The daemon config is UTF-8, but in-process callers may provide
830            // native paths: lossy conversion would collapse distinct programs.
831            #[cfg(unix)]
832            {
833                use std::os::unix::ffi::OsStrExt;
834                field(&mut hash, b"unix-program");
835                field(&mut hash, spec.program.as_os_str().as_bytes());
836            }
837            #[cfg(windows)]
838            {
839                use std::os::windows::ffi::OsStrExt;
840                field(&mut hash, b"windows-program");
841                let bytes = spec
842                    .program
843                    .as_os_str()
844                    .encode_wide()
845                    .flat_map(u16::to_le_bytes)
846                    .collect::<Vec<_>>();
847                field(&mut hash, &bytes);
848            }
849            #[cfg(not(any(unix, windows)))]
850            field(&mut hash, spec.program.to_string_lossy().as_bytes());
851        }
852        hash.update((spec.args.len() as u64).to_le_bytes());
853        for arg in &spec.args {
854            field(&mut hash, arg.as_bytes());
855        }
856        let mut env = spec.env.iter().collect::<Vec<_>>();
857        env.sort_unstable();
858        hash.update((env.len() as u64).to_le_bytes());
859        for (key, value) in env {
860            field(&mut hash, key.as_bytes());
861            field(&mut hash, value.as_bytes());
862        }
863        field(
864            &mut hash,
865            match spec.protocol {
866                ModuleProtocol::Subc => b"subc",
867                ModuleProtocol::None => b"none",
868            },
869        );
870        Self(hash.finalize().into())
871    }
872}
873
874/// Whether a module tolerates a second process of itself running alongside.
875///
876/// Most modules are single-writer on their store (a WAL, a capture log, a
877/// resident index behind a writer barrier), and two processes on one store
878/// corrupt it. So a swap, which overlaps the old and new process by design,
879/// is refused unless the module's config opts in with `overlap: "safe"`.
880#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
881pub enum ModuleOverlap {
882    /// Never run two processes of this module at once. The default.
883    #[default]
884    Exclusive,
885    /// The module has said a second process of itself is harmless for the
886    /// length of a swap.
887    ///
888    /// Declare it only if a second instance can run for a few seconds without
889    /// touching ANY single-writer store: every database, WAL, index, projector
890    /// and scheduled job the module owns. A lease on part of that state is not
891    /// enough. broca's session lease guards WAL appends while its run index, its
892    /// store projector and its archive fold timer (which unlinks live WAL files)
893    /// stay single-writer, so broca is exclusive despite holding a lease. The
894    /// refusal only fires after this has been decided, so the decision is the
895    /// check.
896    Safe,
897}
898
899impl ModuleOverlap {
900    pub fn as_str(self) -> &'static str {
901        match self {
902            Self::Exclusive => "exclusive",
903            Self::Safe => "safe",
904        }
905    }
906}
907
908/// Environment variable telling a spawned module which case it was started
909/// for, before it sends HELLO. Only a swap candidate carries it, as
910/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
911///
912/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
913/// longer because nobody waits on it, while a plain restart must flip ready
914/// quickly because callers see `module_warming` until it does. Absence means
915/// plain restart, the safe reading. The daemon trusts nothing about it; the
916/// candidate is proven by its launch nonce at HELLO.
917pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
918/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
919pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
920/// How long a swap waits for its candidate to register and declare itself
921/// ready when the operator does not say. A module warming as a swap candidate
922/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
923/// daemon allows that plus time to start the process and send HELLO.
924pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
925
926/// Bounded restart policy for crash exits.
927///
928/// `max_restarts` is the number of replacement processes allowed after the
929/// initial spawn WITHIN `window`. After that many crash restarts inside one
930/// window the module enters [`ModuleState::Failed`] and the supervisor stops
931/// the crash loop.
932///
933/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
934/// and that only survived because crashes were rare: a module that crashed
935/// three times across a week was disabled forever by crashes that had nothing
936/// to do with each other. That stopped being survivable once modules began
937/// exiting non-zero whenever the daemon's connection to them drops, because
938/// then every daemon-side connection drop spends a unit of the same budget and
939/// one flappy hour permanently stops a healthy module. Restarts older than
940/// `window` release their slot, so a module that crashed twice yesterday has a
941/// full budget today, while a genuine crash loop -- which is fast by
942/// definition -- still reaches the cap and stops.
943#[derive(Debug, Clone, Copy, PartialEq, Eq)]
944pub struct RestartPolicy {
945    pub max_restarts: u32,
946    /// Base delay before a crash replacement. The actual delay escalates with
947    /// the number of recent crash replacements and is capped by `max_backoff`.
948    pub backoff: Duration,
949    /// Maximum delay before a crash replacement.
950    pub max_backoff: Duration,
951    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
952    /// budget effectively infinite (nothing is ever in-window), which is why
953    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
954    pub window: Duration,
955}
956
957impl RestartPolicy {
958    /// A policy with the default crash window. Callers that care about the
959    /// window say so with [`Self::with_window`]; the ones that do not are
960    /// asking for the standard rate limit, not for no limit.
961    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
962        Self {
963            max_restarts,
964            backoff,
965            max_backoff: DEFAULT_MAX_BACKOFF,
966            window: DEFAULT_RESTART_WINDOW,
967        }
968    }
969
970    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
971        self.max_backoff = max_backoff;
972        self
973    }
974
975    pub fn with_window(mut self, window: Duration) -> Self {
976        self.window = window;
977        self
978    }
979
980    /// Calculate the capped exponential delay for the next crash replacement.
981    /// `restart_in_window` is zero for the first replacement after an operator
982    /// action (restart, reload, re-enable) cleared the crash ring, or after all
983    /// older crash replacements have aged out of the window.
984    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
985        if self.backoff.is_zero() || self.max_backoff.is_zero() {
986            return Duration::ZERO;
987        }
988
989        let mut delay = self.backoff;
990        for _ in 0..restart_in_window {
991            if delay >= self.max_backoff {
992                return self.max_backoff;
993            }
994            delay = delay
995                .checked_mul(10)
996                .unwrap_or(self.max_backoff)
997                .min(self.max_backoff);
998        }
999        delay.min(self.max_backoff)
1000    }
1001
1002    /// The one sentence that explains a budget-exhausted stop, used for both the
1003    /// log line and the terminal record so the two cannot drift. It names the
1004    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1005    /// exactly what this budget is not.
1006    fn budget_exhausted_detail(&self) -> String {
1007        format!(
1008            "crash budget exhausted: max_restarts={} within window_secs={}",
1009            self.max_restarts,
1010            self.window.as_secs()
1011        )
1012    }
1013}
1014
1015impl Default for RestartPolicy {
1016    fn default() -> Self {
1017        Self {
1018            max_restarts: DEFAULT_MAX_RESTARTS,
1019            backoff: DEFAULT_BACKOFF,
1020            max_backoff: DEFAULT_MAX_BACKOFF,
1021            window: DEFAULT_RESTART_WINDOW,
1022        }
1023    }
1024}
1025
1026#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1027struct CrashRestartSchedule {
1028    restart_in_window: u32,
1029    delay: Duration,
1030}
1031
1032/// Whether the daemon itself will bring this module back after the exit being
1033/// handled: it is enabled AND its in-window crash restarts are below the cap.
1034///
1035/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1036/// the window are dropped here rather than by a timer, so the count is right
1037/// the moment somebody asks and no bookkeeping runs for idle modules.
1038fn daemon_will_restart(
1039    state: &mut SupervisorSnapshot,
1040    policy: &RestartPolicy,
1041    now: Instant,
1042) -> bool {
1043    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1044}
1045
1046const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1047const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1048const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1049const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1050
1051#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1052pub enum HealthAction {
1053    Report,
1054    Restart,
1055    Alert,
1056}
1057
1058impl fmt::Display for HealthAction {
1059    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1060        f.write_str(match self {
1061            Self::Report => "report",
1062            Self::Restart => "restart",
1063            Self::Alert => "alert",
1064        })
1065    }
1066}
1067
1068#[derive(Debug, Clone, PartialEq, Eq)]
1069pub struct HealthConfig {
1070    /// Optional loopback HTTP endpoint for a managed non-wire process.
1071    /// Changing it applies live on rescan; the process protocol changes only
1072    /// at its next spawn.
1073    pub http: Option<String>,
1074    pub cadence: Duration,
1075    pub deadline: Duration,
1076    pub failure_threshold: u32,
1077    pub on_degraded: HealthAction,
1078    pub on_failing: HealthAction,
1079    pub critical: bool,
1080}
1081
1082impl Default for HealthConfig {
1083    fn default() -> Self {
1084        Self {
1085            http: None,
1086            cadence: DEFAULT_HEALTH_CADENCE,
1087            deadline: DEFAULT_HEALTH_DEADLINE,
1088            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1089            on_degraded: HealthAction::Report,
1090            on_failing: HealthAction::Report,
1091            critical: false,
1092        }
1093    }
1094}
1095
1096/// The supervisor's view of one module's health, relayed to clients over
1097/// channel-0 and rendered by `ck health`.
1098///
1099/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1100/// stated here rather than only at the wire type a consumer reads. A reader can
1101/// look up what `None` means; only a writer can silently change it, and the
1102/// writer has no reason to go looking at a downstream contract before editing.
1103///
1104/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1105/// back to `None` on re-registration precisely so a respawned module does not
1106/// carry its predecessor's timestamp — so an old value and an absent one call for
1107/// opposite readings, and anything that defaulted this to a number would make a
1108/// never-probed module indistinguishable from one probed at the epoch.
1109///
1110/// `detail` and `metrics` are `None` when the module published none on this
1111/// probe, which does not mean it reported nothing wrong — it is also the shape
1112/// when the probe never reached it. `last_probe_ms` is what separates those.
1113#[derive(Debug, Clone, PartialEq)]
1114pub struct ModuleHealthStatus {
1115    pub status: SupervisorHealthStatus,
1116    pub last_probe_ms: Option<u64>,
1117    pub detail: Option<String>,
1118    pub metrics: Option<Value>,
1119    pub consecutive_failures: u32,
1120    /// Number of replies received after a recurring health probe's deadline.
1121    /// Unlike a timeout, every increment proves the module was alive.
1122    pub late_answer_count: u64,
1123    /// End-to-end latency of the newest late reply, measured from probe start.
1124    pub last_late_answer_latency_ms: Option<u64>,
1125    pub last_action: Option<String>,
1126    /// Set together with `last_action`; the pair moves as one, and both being
1127    /// absent means no escalation has ever been taken rather than that the last
1128    /// one succeeded.
1129    pub last_action_ms: Option<u64>,
1130}
1131
1132impl Default for ModuleHealthStatus {
1133    fn default() -> Self {
1134        Self {
1135            status: SupervisorHealthStatus::Unknown,
1136            last_probe_ms: None,
1137            detail: None,
1138            metrics: None,
1139            consecutive_failures: 0,
1140            late_answer_count: 0,
1141            last_late_answer_latency_ms: None,
1142            last_action: None,
1143            last_action_ms: None,
1144        }
1145    }
1146}
1147
1148/// Typed lifecycle state for a supervised module.
1149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub enum ModuleState {
1151    Starting,
1152    Running,
1153    Unresponsive,
1154    Restarting,
1155    Draining,
1156    Stopped,
1157    Failed,
1158    Disabled,
1159}
1160
1161impl fmt::Display for ModuleState {
1162    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1163        f.write_str(match self {
1164            Self::Starting => "starting",
1165            Self::Running => "running",
1166            Self::Unresponsive => "unresponsive",
1167            Self::Restarting => "restarting",
1168            Self::Draining => "draining",
1169            Self::Stopped => "stopped",
1170            Self::Failed => "failed",
1171            Self::Disabled => "disabled",
1172        })
1173    }
1174}
1175
1176/// Supervisor classification of a child-process exit.
1177#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1178pub enum ExitKind {
1179    Clean,
1180    Crash,
1181    DeliberateSeverance,
1182}
1183
1184impl From<ExitKind> for TerminalExitKind {
1185    fn from(kind: ExitKind) -> Self {
1186        match kind {
1187            ExitKind::Clean => Self::Clean,
1188            ExitKind::Crash => Self::Crash,
1189            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1190        }
1191    }
1192}
1193
1194/// Exact process identity retained when a supervised module registers its
1195/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1196#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1197pub(crate) struct ProcessIdentity {
1198    pub(crate) pid: u32,
1199    pub(crate) start_time: u64,
1200}
1201
1202/// Last observed child exit, if any.
1203#[derive(Debug, Clone, PartialEq, Eq)]
1204pub struct ExitReport {
1205    pub kind: ExitKind,
1206    pub code: Option<i32>,
1207    pub signal: Option<i32>,
1208    pub at_ms: u64,
1209}
1210
1211/// Point-in-time module status answerable by subc without forwarding to the
1212/// module process.
1213#[derive(Debug, Clone, PartialEq)]
1214pub struct ModuleStatus {
1215    pub module_id: String,
1216    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1217    pub waiting_for_previous_pid: Option<u32>,
1218    /// Why the previous instance still prevents a new start.
1219    pub waiting_for_previous_detail: Option<String>,
1220    /// The daemon's Windows per-spawn record: `pipe` after the process it
1221    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1222    /// List/status reports use this record, not the source the module sends
1223    /// in its registration message's manifest.provenance.launch_nonce_source.
1224    pub launch_nonce_source: Option<String>,
1225    pub state: ModuleState,
1226    pub enabled: bool,
1227    pub process_alive: bool,
1228    pub registration_active: bool,
1229    /// The module's declared wire protocol, carried beside `live` because it is
1230    /// what makes `live` readable: the two fields answer one question together.
1231    /// While a process is alive this is its launch declaration, not a later
1232    /// pending-reload edit. When down it is the configured next launch protocol.
1233    pub protocol: ModuleProtocol,
1234    /// Whether the module is serving, under the strongest definition the daemon
1235    /// can assert for its protocol.
1236    ///
1237    /// A subc module must also be REGISTERED: its process being alive says
1238    /// nothing about whether it can take a request. A `protocol: "none"` module
1239    /// never registers, so that term is dropped and this falls back to "enabled,
1240    /// running, and the process the daemon launched is alive" -- which is all
1241    /// the daemon observes about a process that speaks no subc wire. It stays a
1242    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1243    /// rather than printing it bare.
1244    pub live: bool,
1245    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1246    /// restarts have already released their slot, so this count can go down
1247    /// without anybody touching the module.
1248    pub restart_count: u32,
1249    /// Replacement processes spawned over this module's entire supervisor lifetime;
1250    /// unlike `restart_count`, this value is never reset by an operator action
1251    /// and never falls out of a window.
1252    pub lifetime_restarts: u32,
1253    pub spawn_generation: u64,
1254    /// The budget `restart_count` is spent against. Carried alongside the count
1255    /// because the count alone does not say how close the module is to being
1256    /// disabled, and reporting one without the other is what makes an
1257    /// about-to-be-retired module look ordinary.
1258    pub max_restarts: u32,
1259    /// The span `restart_count` is counted over. Carried with the pair above for
1260    /// the same reason they are carried together: "2 of 3" means one thing for a
1261    /// ten-minute window and something else entirely for a lifetime.
1262    pub restart_window: Duration,
1263    /// Effective drain and restart timing policy used by this running module.
1264    /// These values are carried together with the restart budget so status
1265    /// readers can compare configured intent with what the supervisor applied.
1266    pub drain_timeout: Duration,
1267    pub restart_backoff: Duration,
1268    pub restart_max_backoff: Duration,
1269    /// The module's process. On macOS this stays absent while the `ck-subc`
1270    /// launch trampoline is still running in that pid, and appears once the
1271    /// exec acknowledgement confirms the module image has replaced it. Launch
1272    /// time and the supervisor's own hold on the process are unaffected.
1273    pub pid: Option<u32>,
1274    pub spawned_at_ms: Option<u64>,
1275    pub spawned_from: Option<PathBuf>,
1276    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1277    pub process_start_time: Option<u64>,
1278    pub last_exit: Option<ExitReport>,
1279    pub health: ModuleHealthStatus,
1280}
1281
1282#[cfg(test)]
1283#[derive(Debug, Clone, Copy, PartialEq)]
1284struct ActorSelectCheckpoint {
1285    generation: u64,
1286    turn: u64,
1287    registered_connection: Option<ConnectionId>,
1288    next_probe_at: Option<Instant>,
1289    wake_after: Option<Duration>,
1290}
1291
1292#[derive(Debug, Clone, PartialEq)]
1293struct SupervisorSnapshot {
1294    subtree_target: crate::subtree_resources::Target,
1295    failed_retry: FailedRetry,
1296    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1297    #[cfg(windows)]
1298    pipe_consumed: Option<Arc<OnceLock<()>>>,
1299    /// Counts running-child loop turns, not executor polls of a parked wait.
1300    #[cfg(test)]
1301    actor_turns: u64,
1302    /// Running select after exec confirmation and with no unconsumed registry
1303    /// event. Recording a spawn or starting a loop turn is not this barrier.
1304    #[cfg(test)]
1305    actor_select: Option<ActorSelectCheckpoint>,
1306    state: ModuleState,
1307    enabled: bool,
1308    process_alive: bool,
1309    spawned_protocol: Option<ModuleProtocol>,
1310    spawn_failure: Option<String>,
1311    waiting_for_previous_pid: Option<u32>,
1312    waiting_for_previous_detail: Option<String>,
1313    /// When each crash restart was spent, oldest first. This IS the crash
1314    /// budget: its in-window length is the count an operator sees and the count
1315    /// the restart decision is made against, so there is no second counter that
1316    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1317    /// operator actions that used to zero the old lifetime counter.
1318    crash_restarts: VecDeque<Instant>,
1319    lifetime_restarts: u32,
1320    /// Successful child spawns in this daemon incarnation.
1321    ///
1322    /// `lifetime_restarts` was considered and rejected: it starts at zero
1323    /// (line 640), successful initial/operator spawns in `set_running` do not
1324    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1325    /// increments before a successful replacement exists (lines 604, 3846,
1326    /// and 3921), so a failed spawn can consume it. This counter moves only
1327    /// when a live PID is accepted below.
1328    spawn_generation: u64,
1329    pid: Option<u32>,
1330    #[cfg(target_os = "macos")]
1331    report_ready: Option<Arc<OnceLock<()>>>,
1332    /// Last reaped child, retained after current process facts are cleared.
1333    reaped_pid: Option<u32>,
1334    /// Whether a replacement has already been scheduled for the current process.
1335    respawn_pending: bool,
1336    /// A second restart is waiting for the replacement already scheduled.
1337    coalesced_restart_pending: bool,
1338    spawned_at_ms: Option<u64>,
1339    spawned_from: Option<PathBuf>,
1340    launch_spec_digest: Option<LaunchSpecDigest>,
1341    spawned_file_identity: Option<SpawnedFileIdentity>,
1342    process_start_time: Option<u64>,
1343    deliberate_severance: Option<ProcessIdentity>,
1344    last_exit: Option<ExitReport>,
1345    /// Diagnostic attached to the next drain's terminal record, if any.
1346    drain_disposition_detail: Option<String>,
1347    health: ModuleHealthStatus,
1348    last_probe_evidence: Option<HealthProbeEvidence>,
1349    /// Whether the current process was started as a swap candidate and so
1350    /// lives in the module's alternate cgroup. The next swap's candidate takes
1351    /// the other one, so the two processes of a swap never share a cgroup. A
1352    /// plain spawn always uses the primary cgroup.
1353    in_alternate_slot: bool,
1354    /// Whether the current `Draining` state ends in a replacement process
1355    /// (restart, reload, health restart) rather than a stop. Only meaningful
1356    /// while `state` is `Draining`; every entry into that state rewrites it.
1357    /// It is what lets route.open answer the retryable `module_reloading` to a
1358    /// consumer that reaches a still-registered process mid-restart, instead of
1359    /// the `supervisor_not_live` a stop or disable deserves.
1360    draining_to_replace: bool,
1361    /// Whether a configuration update has been applied since the current
1362    /// process was spawned, so that process runs an older spec than the one
1363    /// the supervisor now holds. A queued restart is only coalesced into a
1364    /// fresher process when this is false: a restart requested to pick up a
1365    /// new configuration must not be satisfied by a process that predates it.
1366    configuration_updated_since_spawn: bool,
1367}
1368
1369impl SupervisorSnapshot {
1370    /// The pid that status, provenance and resource readings may report. While
1371    /// the launch trampoline still runs in the pid, reading its executable or
1372    /// resource use would describe `ck-subc`, not the module, so none is
1373    /// reported until the exec acknowledgement confirms the module image.
1374    fn reported_pid(&self) -> Option<u32> {
1375        #[cfg(target_os = "macos")]
1376        if self
1377            .report_ready
1378            .as_ref()
1379            .is_some_and(|ready| ready.get().is_none())
1380        {
1381            return None;
1382        }
1383        self.pid
1384    }
1385
1386    fn starting() -> Self {
1387        Self::new(ModuleState::Starting, true)
1388    }
1389
1390    fn disabled() -> Self {
1391        Self::new(ModuleState::Disabled, false)
1392    }
1393
1394    /// Crash restarts still inside `window`, having dropped the ones that are
1395    /// not. Pruning on read is what makes the budget a rate: an instant older
1396    /// than the window stops holding a slot the moment anybody counts.
1397    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1398        while let Some(oldest) = self.crash_restarts.front() {
1399            if now.duration_since(*oldest) > window {
1400                self.crash_restarts.pop_front();
1401            } else {
1402                break;
1403            }
1404        }
1405        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1406    }
1407
1408    /// Spend one unit of the crash budget and record the restart in the ledger.
1409    ///
1410    /// The ring is bounded by the cap because more than `max_restarts` in-window
1411    /// instants can never be reached (the caller refuses the restart first), so
1412    /// anything beyond that is an unbounded queue waiting to happen.
1413    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1414        self.crash_restarts.push_back(now);
1415        while self.crash_restarts.len() > policy.max_restarts as usize {
1416            self.crash_restarts.pop_front();
1417        }
1418        self.lifetime_restarts += 1;
1419    }
1420
1421    /// Reserve one crash-restart slot and calculate the delay before respawning.
1422    /// The count is captured before recording this restart, so the first retry
1423    /// uses the base delay and each later in-window retry escalates once.
1424    fn next_crash_restart(
1425        &mut self,
1426        policy: &RestartPolicy,
1427        now: Instant,
1428    ) -> Option<CrashRestartSchedule> {
1429        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1430        if restart_in_window >= policy.max_restarts {
1431            return None;
1432        }
1433        self.record_crash_restart(policy, now);
1434        Some(CrashRestartSchedule {
1435            restart_in_window,
1436            delay: policy.delay_for_restart(restart_in_window),
1437        })
1438    }
1439
1440    /// Give the module its full budget back, as an operator restart, reload, or
1441    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1442    /// ledger of what actually happened, and an operator action does not unmake
1443    /// the crashes.
1444    fn clear_crash_restarts(&mut self) {
1445        self.crash_restarts.clear();
1446    }
1447
1448    fn new(state: ModuleState, enabled: bool) -> Self {
1449        Self {
1450            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1451            failed_retry: FailedRetry::default(),
1452            exec_outcome: None,
1453            #[cfg(test)]
1454            actor_turns: 0,
1455            #[cfg(test)]
1456            actor_select: None,
1457            state,
1458            enabled,
1459            process_alive: false,
1460            #[cfg(windows)]
1461            pipe_consumed: None,
1462            spawned_protocol: None,
1463            spawn_failure: None,
1464            waiting_for_previous_pid: None,
1465            waiting_for_previous_detail: None,
1466            crash_restarts: VecDeque::new(),
1467            lifetime_restarts: 0,
1468            spawn_generation: 0,
1469            pid: None,
1470            #[cfg(target_os = "macos")]
1471            report_ready: None,
1472            reaped_pid: None,
1473            respawn_pending: false,
1474            coalesced_restart_pending: false,
1475            spawned_at_ms: None,
1476            spawned_from: None,
1477            launch_spec_digest: None,
1478            spawned_file_identity: None,
1479            process_start_time: None,
1480            deliberate_severance: None,
1481            last_exit: None,
1482            drain_disposition_detail: None,
1483            health: ModuleHealthStatus::default(),
1484            last_probe_evidence: None,
1485            in_alternate_slot: false,
1486            draining_to_replace: false,
1487            configuration_updated_since_spawn: false,
1488        }
1489    }
1490}
1491
1492type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1493
1494type SpawnSubscriberKey = (ConnectionId, u64);
1495
1496#[derive(Debug)]
1497struct SpawnSubscriber {
1498    version: u8,
1499    frames: mpsc::Sender<Frame>,
1500    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1501    /// from which event. The full frame channel cannot carry that news, so it
1502    /// travels beside it; see `SpawnEventFeed::subscribe`.
1503    lagged: Option<oneshot::Sender<SpawnCursor>>,
1504}
1505
1506#[derive(Debug)]
1507struct SpawnEventState {
1508    daemon_incarnation: String,
1509    seq: u64,
1510    capacity: usize,
1511    live: HashMap<String, LiveSpawn>,
1512    generations: HashMap<String, u64>,
1513    events: VecDeque<SpawnEvent>,
1514    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1515}
1516
1517impl Default for SpawnEventState {
1518    fn default() -> Self {
1519        Self {
1520            daemon_incarnation: "unconfigured".to_string(),
1521            seq: 0,
1522            capacity: SPAWN_EVENT_RING_CAPACITY,
1523            live: HashMap::new(),
1524            generations: HashMap::new(),
1525            events: VecDeque::new(),
1526            subscribers: HashMap::new(),
1527        }
1528    }
1529}
1530
1531#[derive(Debug, Clone, Default)]
1532struct SpawnEventFeed(
1533    Arc<Mutex<SpawnEventState>>,
1534    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1535);
1536
1537#[derive(Debug, Clone, PartialEq, Eq)]
1538pub(crate) enum SpawnSubscribeRefusal {
1539    ForeignIncarnation { current: String },
1540    TooOld { oldest: SpawnCursor },
1541    Frame(String),
1542}
1543
1544impl SpawnEventFeed {
1545    fn configure_incarnation(&self, daemon_incarnation: String) {
1546        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1547        state.daemon_incarnation = daemon_incarnation;
1548        state.seq = 0;
1549        state.live.clear();
1550        state.generations.clear();
1551        state.events.clear();
1552        state.subscribers.clear();
1553    }
1554
1555    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1556        SpawnCursor {
1557            daemon_incarnation: state.daemon_incarnation.clone(),
1558            seq: state.seq,
1559        }
1560    }
1561
1562    fn snapshot(&self) -> SpawnSnapshot {
1563        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1564        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1565        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1566        SpawnSnapshot {
1567            cursor: Self::cursor(&state),
1568            ring_bound: state.capacity as u64,
1569            live,
1570        }
1571    }
1572
1573    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1574        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1575        let generation = state
1576            .generations
1577            .get(module_id)
1578            .copied()
1579            .unwrap_or(0)
1580            .checked_add(1)
1581            .expect("spawn generation exhausted");
1582        state.generations.insert(module_id.to_string(), generation);
1583        let live = LiveSpawn {
1584            module_id: module_id.to_string(),
1585            spawn_generation: generation,
1586            pid,
1587            spawned_at_ms,
1588        };
1589        state.live.insert(module_id.to_string(), live);
1590        Self::emit_locked(
1591            &mut state,
1592            SpawnEventKind::Spawned,
1593            module_id.to_string(),
1594            generation,
1595            pid,
1596            None,
1597            None,
1598        );
1599        generation
1600    }
1601
1602    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1603        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1604        let Some(live) = state.live.remove(module_id) else {
1605            warn!(
1606                module_id,
1607                "terminal record had no live spawn event identity"
1608            );
1609            return;
1610        };
1611        Self::emit_locked(
1612            &mut state,
1613            SpawnEventKind::Exited,
1614            module_id.to_string(),
1615            live.spawn_generation,
1616            live.pid,
1617            exit_code,
1618            exit_signal,
1619        );
1620    }
1621
1622    /// Report the exit of a process that a swap has already replaced.
1623    ///
1624    /// `emit_exited` removes the module's live entry, which after a swap's
1625    /// cutover describes the promoted candidate, not the old process now
1626    /// exiting. This emits the old generation's exit and leaves the live entry
1627    /// alone unless it still names that generation.
1628    fn emit_superseded_exited(
1629        &self,
1630        module_id: &str,
1631        spawn_generation: u64,
1632        pid: u32,
1633        exit_code: Option<i32>,
1634        exit_signal: Option<i32>,
1635    ) {
1636        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1637        if state
1638            .live
1639            .get(module_id)
1640            .is_some_and(|live| live.spawn_generation == spawn_generation)
1641        {
1642            state.live.remove(module_id);
1643        }
1644        Self::emit_locked(
1645            &mut state,
1646            SpawnEventKind::Exited,
1647            module_id.to_string(),
1648            spawn_generation,
1649            pid,
1650            exit_code,
1651            exit_signal,
1652        );
1653    }
1654
1655    #[allow(clippy::too_many_arguments)]
1656    fn emit_locked(
1657        state: &mut SpawnEventState,
1658        kind: SpawnEventKind,
1659        module_id: String,
1660        spawn_generation: u64,
1661        pid: u32,
1662        exit_code: Option<i32>,
1663        exit_signal: Option<i32>,
1664    ) {
1665        state.seq = state
1666            .seq
1667            .checked_add(1)
1668            .expect("spawn event sequence exhausted");
1669        let event = SpawnEvent {
1670            cursor: Self::cursor(state),
1671            kind,
1672            module_id,
1673            spawn_generation,
1674            pid,
1675            exit_code,
1676            exit_signal,
1677        };
1678        state.events.push_back(event.clone());
1679        while state.events.len() > state.capacity {
1680            state.events.pop_front();
1681        }
1682        let body = match serde_json::to_vec(&event) {
1683            Ok(body) => body,
1684            Err(error) => {
1685                error!(%error, "failed to serialize supervisor spawn event");
1686                return;
1687            }
1688        };
1689        state.subscribers.retain(|(connection_id, corr), subscriber| {
1690            let frame = Frame::build_with_version(
1691                subscriber.version,
1692                FrameType::StreamData,
1693                control_flags(),
1694                0,
1695                0,
1696                *corr,
1697                body.clone(),
1698            );
1699            match frame {
1700                Ok(frame) => {
1701                    if subscriber.frames.try_send(frame).is_ok() {
1702                        true
1703                    } else {
1704                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1705                        if let Some(lagged) = subscriber.lagged.take() {
1706                            let _ = lagged.send(event.cursor.clone());
1707                        }
1708                        false
1709                    }
1710                }
1711                Err(error) => {
1712                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1713                    false
1714                }
1715            }
1716        });
1717    }
1718
1719    fn subscribe(
1720        &self,
1721        connection_id: ConnectionId,
1722        corr: u64,
1723        version: u8,
1724        since: Option<SpawnCursor>,
1725        sink: FrameSink,
1726    ) -> Result<(), SpawnSubscribeRefusal> {
1727        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1728        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1729        {
1730            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1731            let replay = if let Some(since) = since {
1732                if since.daemon_incarnation != state.daemon_incarnation {
1733                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1734                        current: state.daemon_incarnation.clone(),
1735                    });
1736                }
1737                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1738                    if since.seq < oldest.seq.saturating_sub(1) {
1739                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1740                    }
1741                }
1742                state
1743                    .events
1744                    .iter()
1745                    .filter(|event| event.cursor.seq > since.seq)
1746                    .cloned()
1747                    .collect::<Vec<_>>()
1748            } else {
1749                Vec::new()
1750            };
1751            for event in replay {
1752                let body = serde_json::to_vec(&event)
1753                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1754                let frame = Frame::build_with_version(
1755                    version,
1756                    FrameType::StreamData,
1757                    control_flags(),
1758                    0,
1759                    0,
1760                    corr,
1761                    body,
1762                )
1763                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1764                frames
1765                    .try_send(frame)
1766                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1767            }
1768            state.subscribers.insert(
1769                (connection_id, corr),
1770                SpawnSubscriber {
1771                    version,
1772                    frames,
1773                    lagged: Some(lagged),
1774                },
1775            );
1776        }
1777        // The lagged terminal is sent here, by the forwarder, rather than by
1778        // the emitter: at the moment of the drop the subscriber's own channel
1779        // is full, and writing to the connection sink directly from the emitter
1780        // would put the Error AHEAD of the events still queued in that channel
1781        // (and the emitter holds the feed lock, so it cannot await the sink).
1782        // Dropping the subscriber drops the only sender, so `recv` drains every
1783        // queued event and then returns `None`; only then is the Error sent, so
1784        // the client sees each event it can keep, then the reason it was cut.
1785        // Cancel and connection removal drop the oneshot unsent, so they end
1786        // the stream with no Error.
1787        tokio::spawn(async move {
1788            while let Some(frame) = receiver.recv().await {
1789                if sink.send(frame).await.is_err() {
1790                    return;
1791                }
1792            }
1793            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1794                return;
1795            };
1796            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1797                Ok(frame) => {
1798                    let _ = sink.send(frame).await;
1799                }
1800                Err(error) => {
1801                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1802                }
1803            }
1804        });
1805        Ok(())
1806    }
1807
1808    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1809        let Some(subscriber) = self
1810            .0
1811            .lock()
1812            .unwrap_or_else(|p| p.into_inner())
1813            .subscribers
1814            .remove(&(connection_id, corr))
1815        else {
1816            return false;
1817        };
1818        if let Ok(frame) = Frame::build_with_version(
1819            subscriber.version,
1820            FrameType::StreamEnd,
1821            control_flags(),
1822            0,
1823            0,
1824            corr,
1825            Vec::new(),
1826        ) {
1827            tokio::spawn(async move {
1828                let _ = subscriber.frames.send(frame).await;
1829            });
1830        }
1831        true
1832    }
1833
1834    fn remove_connection(&self, connection_id: ConnectionId) {
1835        self.0
1836            .lock()
1837            .unwrap_or_else(|p| p.into_inner())
1838            .subscribers
1839            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1840    }
1841
1842    #[cfg(any(test, feature = "test-support"))]
1843    fn set_capacity(&self, capacity: usize) {
1844        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1845    }
1846
1847    #[cfg(any(test, feature = "test-support"))]
1848    fn subscriber_count(&self) -> usize {
1849        self.0
1850            .lock()
1851            .unwrap_or_else(|p| p.into_inner())
1852            .subscribers
1853            .len()
1854    }
1855}
1856
1857/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1858/// The terminal Error a lagged spawn subscriber receives after its queued events.
1859fn spawn_subscriber_lagged_frame(
1860    version: u8,
1861    corr: u64,
1862    first_undelivered: SpawnCursor,
1863) -> Result<Frame, String> {
1864    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1865    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1866        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1867        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1868            .to_string(),
1869        detail: Some(serde_json::json!({
1870            "first_undelivered_cursor": first_undelivered
1871        })),
1872    })
1873    .map_err(|error| error.to_string())?;
1874    Frame::build_with_version(
1875        version,
1876        FrameType::Error,
1877        crate::router::daemon_error_flags(),
1878        0,
1879        0,
1880        corr,
1881        body,
1882    )
1883    .map_err(|error| error.to_string())
1884}
1885
1886pub trait ModuleProcessLiveness: Send + Sync {
1887    fn process_live(&self, module_id: &str) -> Option<bool>;
1888
1889    /// Whether the supervisor is replacing this module's process right now: an
1890    /// operator restart or reload, a health restart, or a crash respawn whose
1891    /// backoff is running. A module in that state is not live, but a consumer
1892    /// refused now should retry shortly rather than treat the target as gone.
1893    /// Stopped, failed, and disabled modules are not replacing.
1894    fn process_replacing(&self, _module_id: &str) -> bool {
1895        false
1896    }
1897}
1898
1899/// Shared process-liveness registry keyed by supervised `module_id`.
1900#[derive(Debug, Clone, Default)]
1901pub struct SupervisorProcessLiveness {
1902    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1903}
1904
1905impl SupervisorProcessLiveness {
1906    pub fn new() -> Self {
1907        Self::default()
1908    }
1909
1910    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1911        let mut snapshots = self
1912            .snapshots
1913            .lock()
1914            .unwrap_or_else(|poisoned| poisoned.into_inner());
1915        snapshots.insert(module_id, snapshot);
1916    }
1917
1918    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1919        let mut snapshots = self
1920            .snapshots
1921            .lock()
1922            .unwrap_or_else(|poisoned| poisoned.into_inner());
1923        let is_current = snapshots
1924            .get(module_id)
1925            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1926            .unwrap_or(false);
1927        if is_current {
1928            snapshots.remove(module_id);
1929        }
1930    }
1931}
1932
1933impl ModuleProcessLiveness for SupervisorProcessLiveness {
1934    fn process_live(&self, module_id: &str) -> Option<bool> {
1935        let snapshot = {
1936            let snapshots = self
1937                .snapshots
1938                .lock()
1939                .unwrap_or_else(|poisoned| poisoned.into_inner());
1940            snapshots.get(module_id).cloned()
1941        }?;
1942        let snapshot = snapshot
1943            .lock()
1944            .unwrap_or_else(|poisoned| poisoned.into_inner());
1945        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1946    }
1947
1948    fn process_replacing(&self, module_id: &str) -> bool {
1949        let Some(snapshot) = self
1950            .snapshots
1951            .lock()
1952            .unwrap_or_else(|poisoned| poisoned.into_inner())
1953            .get(module_id)
1954            .cloned()
1955        else {
1956            return false;
1957        };
1958        let snapshot = snapshot
1959            .lock()
1960            .unwrap_or_else(|poisoned| poisoned.into_inner());
1961        snapshot.enabled
1962            && match snapshot.state {
1963                ModuleState::Restarting => true,
1964                ModuleState::Draining => snapshot.draining_to_replace,
1965                ModuleState::Starting
1966                | ModuleState::Running
1967                | ModuleState::Unresponsive
1968                | ModuleState::Stopped
1969                | ModuleState::Failed
1970                | ModuleState::Disabled => false,
1971            }
1972    }
1973}
1974
1975#[cfg(test)]
1976#[derive(Debug, Default)]
1977struct ReloadExitRecordGate {
1978    reached: tokio::sync::Notify,
1979    resume: tokio::sync::Notify,
1980}
1981
1982#[derive(Debug, Clone, Copy)]
1983enum RespawnKind {
1984    Spawn,
1985    Reload,
1986}
1987
1988#[derive(Debug, Clone, Copy)]
1989struct PendingRespawn {
1990    deadline: Instant,
1991    kind: RespawnKind,
1992    daemon_initiated: bool,
1993}
1994
1995type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1996
1997#[derive(Debug, Clone)]
1998struct SupervisorRuntimeConfig {
1999    registry: Arc<Registry>,
2000    /// Retained across an operation's deferred backoff, until its child starts.
2001    recovery_is_daemon: bool,
2002    start_requester: Option<Principal>,
2003    pending_activity_restart: bool,
2004    provenance_probe: ExecutableIdentityProbe,
2005    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2006    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2007    /// A reload acknowledges completion only after its replacement registers.
2008    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2009    restart_policy: RestartPolicy,
2010    register_timeout: Duration,
2011    /// This module's RESOLVED drain budget: per-module config when present,
2012    /// else `default_drain_timeout`.
2013    drain_timeout: Duration,
2014    /// Shared with the status handle so the attested value changes atomically
2015    /// when a rescan updates the running drain policy.
2016    effective_drain_timeout: Arc<Mutex<Duration>>,
2017    /// The supervisor-wide fallback, kept so a configuration update that
2018    /// REMOVES the per-module override can re-resolve to it.
2019    default_drain_timeout: Duration,
2020    health: HealthConfig,
2021    host_health: HostHealth,
2022    connection_file_path: Option<PathBuf>,
2023    capture_logs_dir: Option<PathBuf>,
2024    forwarding: Option<Arc<ForwardingTable>>,
2025    /// The shared handle, so every spawn path (initial, restart, reload) records the
2026    /// reserved-module launch nonce the HELLO verifier checks against.
2027    supervisor_handle: Option<SupervisorHandle>,
2028    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2029    /// status queries.
2030    ///
2031    /// One ring per module, held across every respawn. The lines explaining an exit
2032    /// are written BEFORE that exit, so a ring recreated per process would be empty
2033    /// exactly when it is asked for.
2034    stderr_ring: Arc<Mutex<StderrRing>>,
2035    terminal_ring: Arc<Mutex<TerminalRing>>,
2036    spawn_events: SpawnEventFeed,
2037    child_roster: ChildRoster,
2038    #[cfg(target_os = "linux")]
2039    cgroup_placement: Option<subc_cgroup::Placement>,
2040    #[cfg(test)]
2041    test_seed_stale_facts_before_enable_spawn: bool,
2042    #[cfg(test)]
2043    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2044}
2045
2046impl SupervisorRuntimeConfig {
2047    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2048        FailureContext {
2049            daemon_initiated,
2050            shutdown: self.child_roster.shutdown_flag(),
2051        }
2052    }
2053}
2054
2055#[derive(Debug, Clone, PartialEq, Eq)]
2056struct SupervisedConfiguration {
2057    spec: ModuleSpec,
2058    health: HealthConfig,
2059}
2060
2061/// Shared daemon lookup table for supervised module handles.
2062///
2063/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2064/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2065/// launch nonces recorded at spawn are checked by the same daemon instance.
2066#[derive(Debug, Clone, Default)]
2067pub struct SupervisorHandle {
2068    host_health: HostHealth,
2069    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2070    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2071    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2072    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2073    /// Module ids the supervisor has taken on. An id is added BEFORE the
2074    /// module's first process is spawned and removed only when the module
2075    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2076    /// the keys of `modules`.
2077    ///
2078    /// `modules` cannot answer "is this module configured?" on its own: a
2079    /// [`SupervisedModule`] only exists once its process has been spawned, and
2080    /// a fast child can connect, register, sync its scopes and ask about them
2081    /// before the supervisor has inserted it. Answering "not configured" in that
2082    /// gap makes scope admission refuse with the terminal "will never sync"
2083    /// instead of the retryable "has not synced yet".
2084    configured_ids: Arc<Mutex<HashSet<String>>>,
2085    spawn_events: SpawnEventFeed,
2086    /// The current expected launch nonce for each reserved module_id. Set when the
2087    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2088    /// non-reserved module never has an entry here and is never nonce-checked.
2089    /// Reserved module ids and the nonce that authorizes their next HELLO.
2090    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2091    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2092    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2093    /// had NO entry and admitted anyone: the reservation protected the nonce
2094    /// holder, not the NAME (found live by CKCRED's canary probe registering
2095    /// against a reserved scratch id).
2096    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2097    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2098    ///
2099    /// This is deliberately in-memory only: subc is state-free across daemon
2100    /// restarts, and the tombstone only explains the hours-after-removal window
2101    /// while this executing daemon is still alive. Do not persist it in a store.
2102    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2103    /// The current launch nonce for every supervised spawn. This is separate from
2104    /// reserved_nonces because consumer route.open attestation applies to all spawned
2105    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2106    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2107    /// Reserved namespace prefixes mapped to the supervised owner module whose
2108    /// current spawn nonce authorizes HELLO claims below the prefix.
2109    ///
2110    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2111    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2112    /// accidental collisions and lower-trust processes from squatting protected
2113    /// namespaces.
2114    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2115    /// Blue/green swaps in progress, by module id. An entry exists from just
2116    /// before the candidate process is spawned until the swap has failed, or
2117    /// has cut over and the old process is gone. While it exists, HELLO for the
2118    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2119    /// consumer attestation accepts both processes' nonces.
2120    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2121    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2122    promotion_observer: PromotionObserverSlot,
2123    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2124    /// this daemon-wide ordering, a rescan could retire or update a module while a
2125    /// concurrent reload still held its old handle and launch specification.
2126    operation_lock: Arc<AsyncMutex<()>>,
2127}
2128
2129/// Told when a swap has promoted its candidate to be the module's active
2130/// registration.
2131///
2132/// An ordinary HELLO runs the control plane's registration side effects (the
2133/// capability cache, the deny census, the requirement recompute) as it
2134/// registers. A swap candidate's HELLO does not, because it is not routable;
2135/// promotion is when those must run instead, and promotion happens in the
2136/// supervisor, which has no other way into the control handler.
2137pub(crate) trait SwapPromotionObserver: Send + Sync {
2138    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2139}
2140
2141/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2142/// control handler) owns this handle, so a strong reference back would be a
2143/// cycle that keeps both alive.
2144#[derive(Clone, Default)]
2145struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2146
2147impl fmt::Debug for PromotionObserverSlot {
2148    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2149        f.write_str("PromotionObserverSlot")
2150    }
2151}
2152
2153/// The nonces of one open swap.
2154#[derive(Debug, Clone)]
2155struct OpenSwap {
2156    /// The launch nonce minted for the candidate process. It is the swap
2157    /// token: the only thing that admits a HELLO into the candidate slot.
2158    candidate_nonce: String,
2159    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2160    /// here because cutover moves the module's recorded spawn nonce to the
2161    /// candidate while the incumbent is still draining and its consumers are
2162    /// still attesting with this one.
2163    incumbent_nonce: Option<String>,
2164    /// Set once a HELLO has been admitted with the swap token, so the token
2165    /// admits one registration and cannot be replayed after cutover empties
2166    /// the candidate slot.
2167    candidate_admitted: bool,
2168}
2169
2170/// What the swap gate says about a HELLO. See
2171/// [`SupervisorHandle::swap_hello_admission`].
2172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2173pub(crate) enum SwapHelloAdmission {
2174    /// No swap is open for the id (or the HELLO carries the incumbent's own
2175    /// nonce); the ordinary gates decide.
2176    NotSwapping,
2177    /// The HELLO carries the swap token: register it into the candidate slot.
2178    Candidate,
2179    /// A swap is open and the HELLO carries a nonce the supervisor did not
2180    /// mint for this id, no nonce, or a token already used.
2181    Refused,
2182}
2183
2184#[derive(Debug, Clone, PartialEq, Eq)]
2185pub(crate) enum ReservedHelloRejection {
2186    Exact {
2187        module_id: String,
2188    },
2189    Prefix {
2190        prefix: String,
2191        owner_module_id: String,
2192    },
2193}
2194
2195/// Identity facts established by the reserved-module policy for one HELLO.
2196#[derive(Debug, Default)]
2197pub(crate) struct ReservedHelloVerification {
2198    pub(crate) nonce_verified: bool,
2199    pub(crate) prefix_owner: Option<String>,
2200}
2201
2202#[derive(Debug)]
2203struct DaemonRestartIntent {
2204    expires: Option<Duration>,
2205    window: Duration,
2206}
2207
2208impl Default for DaemonRestartIntent {
2209    fn default() -> Self {
2210        Self {
2211            expires: None,
2212            window: Duration::from_secs(120),
2213        }
2214    }
2215}
2216
2217impl SupervisorHandle {
2218    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2219        let mut intent = self
2220            .restart_intent
2221            .lock()
2222            .unwrap_or_else(|p| p.into_inner());
2223        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2224        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2225    }
2226
2227    #[cfg(feature = "test-support")]
2228    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2229        self.restart_intent
2230            .lock()
2231            .unwrap_or_else(|p| p.into_inner())
2232            .window = window;
2233    }
2234
2235    #[cfg(unix)]
2236    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2237        let expiry = self
2238            .restart_intent
2239            .lock()
2240            .unwrap_or_else(|p| p.into_inner())
2241            .expires
2242            .take();
2243        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2244            RouteCloseReason::Restart
2245        } else {
2246            RouteCloseReason::Disable
2247        }
2248    }
2249
2250    pub fn new() -> Self {
2251        Self::default()
2252    }
2253
2254    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2255        self.spawn_events.snapshot()
2256    }
2257
2258    pub(crate) fn subscribe_spawns(
2259        &self,
2260        connection_id: ConnectionId,
2261        corr: u64,
2262        version: u8,
2263        since: Option<SpawnCursor>,
2264        sink: FrameSink,
2265    ) -> Result<(), SpawnSubscribeRefusal> {
2266        self.spawn_events
2267            .subscribe(connection_id, corr, version, since, sink)
2268    }
2269
2270    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2271        self.spawn_events.cancel(connection_id, corr)
2272    }
2273
2274    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2275        self.spawn_events.remove_connection(connection_id);
2276    }
2277
2278    #[cfg(any(test, feature = "test-support"))]
2279    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2280        assert!(capacity > 0, "spawn event capacity must be non-zero");
2281        self.spawn_events.set_capacity(capacity);
2282    }
2283
2284    #[cfg(any(test, feature = "test-support"))]
2285    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2286        self.spawn_events.subscriber_count()
2287    }
2288
2289    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2290    /// a respawn invalidates stale consumer identities.
2291    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2292        self.spawn_nonces
2293            .lock()
2294            .unwrap_or_else(|poisoned| poisoned.into_inner())
2295            .insert(module_id.to_string(), nonce);
2296    }
2297
2298    /// Record the launch nonce expected from the next HELLO for a reserved module,
2299    /// replacing any prior nonce (a respawn invalidates the previous one).
2300    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2301        self.reserved_nonces
2302            .lock()
2303            .unwrap_or_else(|poisoned| poisoned.into_inner())
2304            .insert(module_id.to_string(), Some(nonce));
2305    }
2306
2307    /// Record namespace prefixes owned by a supervised module.
2308    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2309        let mut owners = self
2310            .reserved_prefix_owners
2311            .lock()
2312            .unwrap_or_else(|poisoned| poisoned.into_inner());
2313        owners.retain(|_, owner| owner != owner_module_id);
2314        for prefix in prefixes {
2315            owners.insert(prefix.clone(), owner_module_id.to_string());
2316        }
2317    }
2318
2319    /// The launch nonce most recently minted for a module's spawn, if any.
2320    #[cfg(test)]
2321    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2322        self.spawn_nonces
2323            .lock()
2324            .unwrap_or_else(|poisoned| poisoned.into_inner())
2325            .get(module_id)
2326            .cloned()
2327    }
2328
2329    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2330        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2331        let spawn_nonce = self
2332            .spawn_nonces
2333            .lock()
2334            .unwrap_or_else(|poisoned| poisoned.into_inner())
2335            .get(&spec.module_id)
2336            .cloned();
2337        let mut reserved_nonces = self
2338            .reserved_nonces
2339            .lock()
2340            .unwrap_or_else(|poisoned| poisoned.into_inner());
2341        if spec.reserved {
2342            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2343            // reserved name whose module has never spawned has no legitimate
2344            // holder, and the entry's absence is what used to leave the name
2345            // open to the first claimant.
2346            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2347        }
2348        drop(reserved_nonces);
2349        // A later unreserved declaration must not silently unreserve an id that
2350        // was retained after its reserved configuration was removed. The explicit
2351        // release ceremony is the only operation that retires that gate.
2352        self.removal_tombstones
2353            .lock()
2354            .unwrap_or_else(|poisoned| poisoned.into_inner())
2355            .remove(&spec.module_id);
2356    }
2357
2358    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2359    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2360    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2361    /// with no matching prefix are always authorized.
2362    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2363        self.reserved_hello_verification(module_id, presented)
2364            .is_ok()
2365    }
2366
2367    /// Whether HELLO passed a nonce check that was actually required by reserved
2368    /// module policy, or the refusal explaining why a required check failed.
2369    pub(crate) fn reserved_hello_verification(
2370        &self,
2371        module_id: &str,
2372        presented: Option<&str>,
2373    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2374        let nonces = self
2375            .reserved_nonces
2376            .lock()
2377            .unwrap_or_else(|poisoned| poisoned.into_inner());
2378        if let Some(expected) = nonces.get(module_id) {
2379            // `None` = reserved with no legitimate holder: refuse every
2380            // presentation, because no process can hold a nonce that was never
2381            // minted. Only a real minted nonce admits, in constant time.
2382            let authorized = match expected {
2383                Some(expected) => {
2384                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2385                }
2386                None => false,
2387            };
2388            if authorized {
2389                return Ok(ReservedHelloVerification {
2390                    nonce_verified: true,
2391                    prefix_owner: None,
2392                });
2393            }
2394            return Err(ReservedHelloRejection::Exact {
2395                module_id: module_id.to_string(),
2396            });
2397        }
2398        drop(nonces);
2399
2400        let matched_prefix = self
2401            .reserved_prefix_owners
2402            .lock()
2403            .unwrap_or_else(|poisoned| poisoned.into_inner())
2404            .iter()
2405            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2406            .max_by_key(|(prefix, _)| prefix.len())
2407            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2408        let Some((prefix, owner_module_id)) = matched_prefix else {
2409            return Ok(ReservedHelloVerification::default());
2410        };
2411
2412        let authorized = presented.is_some_and(|presented| {
2413            self.spawn_nonces
2414                .lock()
2415                .unwrap_or_else(|poisoned| poisoned.into_inner())
2416                .get(&owner_module_id)
2417                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2418                // While the owner is being swapped, children started by
2419                // either of its two processes hold that process's nonce.
2420                || self.swap_nonce_matches(&owner_module_id, presented)
2421        });
2422        if authorized {
2423            Ok(ReservedHelloVerification {
2424                nonce_verified: true,
2425                prefix_owner: Some(owner_module_id),
2426            })
2427        } else {
2428            Err(ReservedHelloRejection::Prefix {
2429                prefix,
2430                owner_module_id,
2431            })
2432        }
2433    }
2434
2435    /// Whether a consumer connection proved it came from a daemon-spawned module.
2436    ///
2437    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2438    /// accepted only for module ids the supervisor has spawned.
2439    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2440        if presented.is_empty() {
2441            return false;
2442        }
2443        let nonces = self
2444            .spawn_nonces
2445            .lock()
2446            .unwrap_or_else(|poisoned| poisoned.into_inner());
2447        let current = nonces
2448            .get(module_id)
2449            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2450        drop(nonces);
2451        // During a swap two processes of the module are alive, and a consumer
2452        // started by either one presents that process's nonce. Accepting only
2453        // the recorded one would fail the incumbent's consumers for the whole
2454        // overlap once cutover moves the record to the candidate.
2455        current || self.swap_nonce_matches(module_id, presented)
2456    }
2457
2458    /// Whether `presented` is either nonce of an open swap for `module_id`.
2459    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2460        let swaps = self
2461            .swaps
2462            .lock()
2463            .unwrap_or_else(|poisoned| poisoned.into_inner());
2464        swaps.get(module_id).is_some_and(|swap| {
2465            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2466                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2467                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2468                })
2469        })
2470    }
2471
2472    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2473    /// Called before the candidate process exists.
2474    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2475        let incumbent_nonce = self
2476            .spawn_nonces
2477            .lock()
2478            .unwrap_or_else(|poisoned| poisoned.into_inner())
2479            .get(module_id)
2480            .cloned();
2481        self.swaps
2482            .lock()
2483            .unwrap_or_else(|poisoned| poisoned.into_inner())
2484            .insert(
2485                module_id.to_string(),
2486                OpenSwap {
2487                    candidate_nonce,
2488                    incumbent_nonce,
2489                    candidate_admitted: false,
2490                },
2491            );
2492    }
2493
2494    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2495    /// the module's recorded one.
2496    pub(crate) fn close_swap(&self, module_id: &str) {
2497        self.swaps
2498            .lock()
2499            .unwrap_or_else(|poisoned| poisoned.into_inner())
2500            .remove(module_id);
2501    }
2502
2503    /// Install the observer told about swap promotions, replacing any earlier
2504    /// one.
2505    pub(crate) fn set_swap_promotion_observer(
2506        &self,
2507        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2508    ) {
2509        *self
2510            .promotion_observer
2511            .0
2512            .lock()
2513            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2514    }
2515
2516    /// Tell the installed observer, if it is still alive, that a swap promoted
2517    /// `registration`.
2518    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2519        let observer = self
2520            .promotion_observer
2521            .0
2522            .lock()
2523            .unwrap_or_else(|poisoned| poisoned.into_inner())
2524            .as_ref()
2525            .and_then(std::sync::Weak::upgrade);
2526        if let Some(observer) = observer {
2527            observer.swap_promoted(registration);
2528        }
2529    }
2530
2531    /// Whether a swap is open for `module_id`.
2532    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2533        self.swaps
2534            .lock()
2535            .unwrap_or_else(|poisoned| poisoned.into_inner())
2536            .contains_key(module_id)
2537    }
2538
2539    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2540    /// respawn would, once cutover has made the candidate the module's process.
2541    /// The swap stays open so the incumbent's nonce keeps attesting until the
2542    /// incumbent has drained and exited.
2543    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2544        let candidate_nonce = self
2545            .swaps
2546            .lock()
2547            .unwrap_or_else(|poisoned| poisoned.into_inner())
2548            .get(module_id)
2549            .map(|swap| swap.candidate_nonce.clone());
2550        let Some(nonce) = candidate_nonce else {
2551            return;
2552        };
2553        self.set_spawn_nonce(module_id, nonce.clone());
2554        if reserved {
2555            self.set_reserved_nonce(module_id, nonce);
2556        }
2557    }
2558
2559    /// The swap gate for a HELLO claiming `module_id`.
2560    ///
2561    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2562    /// presents the candidate nonce, which the reserved gate (holding the
2563    /// incumbent's nonce) would refuse as `reserved_module` before swap
2564    /// admission was ever reached. And it applies to unreserved ids too: for an
2565    /// unreserved id the only thing that ever stopped a second process claiming
2566    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2567    /// refusal a swap lifts for its candidate.
2568    ///
2569    /// The incumbent's own nonce falls through to the ordinary gates, which
2570    /// treat it as they always have (a live incumbent is refused as a
2571    /// duplicate). Anything else while a swap is open is refused, including an
2572    /// absent nonce.
2573    pub(crate) fn swap_hello_admission(
2574        &self,
2575        module_id: &str,
2576        presented: Option<&str>,
2577    ) -> SwapHelloAdmission {
2578        let swaps = self
2579            .swaps
2580            .lock()
2581            .unwrap_or_else(|poisoned| poisoned.into_inner());
2582        let Some(swap) = swaps.get(module_id) else {
2583            return SwapHelloAdmission::NotSwapping;
2584        };
2585        let Some(presented) = presented else {
2586            return SwapHelloAdmission::Refused;
2587        };
2588        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2589            return if swap.candidate_admitted {
2590                SwapHelloAdmission::Refused
2591            } else {
2592                SwapHelloAdmission::Candidate
2593            };
2594        }
2595        if swap
2596            .incumbent_nonce
2597            .as_deref()
2598            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2599        {
2600            return SwapHelloAdmission::NotSwapping;
2601        }
2602        SwapHelloAdmission::Refused
2603    }
2604
2605    /// Record that the swap token has registered a candidate, so it admits no
2606    /// second HELLO.
2607    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2608        if let Some(swap) = self
2609            .swaps
2610            .lock()
2611            .unwrap_or_else(|poisoned| poisoned.into_inner())
2612            .get_mut(module_id)
2613        {
2614            swap.candidate_admitted = true;
2615        }
2616    }
2617
2618    /// Test/support lookup for the current launch nonce of a supervised spawn.
2619    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2620        self.spawn_nonces
2621            .lock()
2622            .unwrap_or_else(|poisoned| poisoned.into_inner())
2623            .get(module_id)
2624            .cloned()
2625    }
2626
2627    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2628    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2629        self.reserved_nonces
2630            .lock()
2631            .unwrap_or_else(|poisoned| poisoned.into_inner())
2632            .get(module_id)
2633            .cloned()
2634            .flatten()
2635    }
2636
2637    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2638        // Normally already marked before the process was spawned; marking here
2639        // too keeps `configured_ids` a superset of the roster for any caller
2640        // that inserts a module directly.
2641        self.mark_configured(module.module_id());
2642        let mut modules = self
2643            .modules
2644            .lock()
2645            .unwrap_or_else(|poisoned| poisoned.into_inner());
2646        modules.insert(module.module_id().to_string(), module)
2647    }
2648
2649    /// Record that the supervisor has taken on `module_id`. Called before the
2650    /// module's first process is spawned, so that by the time that process can
2651    /// register, [`Self::is_configured`] already answers true.
2652    fn mark_configured(&self, module_id: &str) {
2653        self.configured_ids
2654            .lock()
2655            .unwrap_or_else(|poisoned| poisoned.into_inner())
2656            .insert(module_id.to_string());
2657    }
2658
2659    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2660    /// before it was ever put on the roster. A module already on the roster
2661    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2662    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2663        let modules = self
2664            .modules
2665            .lock()
2666            .unwrap_or_else(|poisoned| poisoned.into_inner());
2667        if !modules.contains_key(module_id) {
2668            self.configured_ids
2669                .lock()
2670                .unwrap_or_else(|poisoned| poisoned.into_inner())
2671                .remove(module_id);
2672        }
2673    }
2674
2675    /// Whether `module_id` is a module this daemon supervises: on the roster,
2676    /// or about to be (its process is being spawned right now).
2677    ///
2678    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2679    /// for scopes: a supervised module's process can register and sync before
2680    /// [`Self::get`] can return it, and in that window it is still a module
2681    /// that will sync, not one that never will.
2682    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2683        self.configured_ids
2684            .lock()
2685            .unwrap_or_else(|poisoned| poisoned.into_inner())
2686            .contains(module_id)
2687    }
2688
2689    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2690        let modules = self
2691            .modules
2692            .lock()
2693            .unwrap_or_else(|poisoned| poisoned.into_inner());
2694        modules.get(module_id).cloned()
2695    }
2696
2697    pub(crate) fn record_late_health_answer(
2698        &self,
2699        module_id: &str,
2700        latency_ms: u64,
2701    ) -> Result<bool, SuperviseError> {
2702        let Some(module) = self.get(module_id) else {
2703            return Ok(false);
2704        };
2705        self.host_health.record_late_module(module_id);
2706        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2707            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2708            state.health.last_late_answer_latency_ms = Some(latency_ms);
2709            // A late answer is an answer: the module served the probe, just past
2710            // the deadline. Leaving the miss streak in place while logging
2711            // "proves the module is alive" is how a CPU-starved module that
2712            // answers every probe a few seconds late still marches to the
2713            // threshold and gets killed — the exact kill class `NoAnswer` is
2714            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2715            // is degradation, and degradation reports; it does not restart.
2716            state.health.consecutive_failures = 0;
2717        })?;
2718        Ok(true)
2719    }
2720
2721    pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
2722        self.host_health.spawn_monitor()
2723    }
2724
2725    /// Arm the one-shot marker for the module process that this caller
2726    /// deliberately initiated severance against. Generic connection teardown
2727    /// must not call this:
2728    /// a surviving process would otherwise retain an exemption for a later
2729    /// genuine crash.
2730    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2731        let Some(module) = self.get(module_id) else {
2732            return Ok(false);
2733        };
2734        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2735        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2736            return Ok(false);
2737        };
2738        drop(snapshot);
2739        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2740    }
2741
2742    pub fn list(&self) -> Vec<SupervisedModule> {
2743        let modules = self
2744            .modules
2745            .lock()
2746            .unwrap_or_else(|poisoned| poisoned.into_inner());
2747        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2748        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2749        modules
2750    }
2751
2752    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2753        self.activity_images
2754            .lock()
2755            .unwrap_or_else(|p| p.into_inner())
2756            .remove(module_id);
2757        self.spawn_nonces
2758            .lock()
2759            .unwrap_or_else(|poisoned| poisoned.into_inner())
2760            .remove(module_id);
2761        self.close_swap(module_id);
2762        let mut reserved_nonces = self
2763            .reserved_nonces
2764            .lock()
2765            .unwrap_or_else(|poisoned| poisoned.into_inner());
2766        if reserved_nonces.contains_key(module_id) {
2767            // The old nonce must die with the removed process, but the exact-id
2768            // gate remains until an operator explicitly releases it.
2769            reserved_nonces.insert(module_id.to_string(), None);
2770        }
2771        drop(reserved_nonces);
2772        self.reserved_prefix_owners
2773            .lock()
2774            .unwrap_or_else(|poisoned| poisoned.into_inner())
2775            .retain(|_, owner| owner != module_id);
2776        let removed = self
2777            .modules
2778            .lock()
2779            .unwrap_or_else(|poisoned| poisoned.into_inner())
2780            .remove(module_id);
2781        self.configured_ids
2782            .lock()
2783            .unwrap_or_else(|poisoned| poisoned.into_inner())
2784            .remove(module_id);
2785        removed
2786    }
2787
2788    /// Remember a module removed by a non-preview rescan so route.open can
2789    /// distinguish that intentional removal from an unknown id.
2790    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2791        self.removal_tombstones
2792            .lock()
2793            .unwrap_or_else(|poisoned| poisoned.into_inner())
2794            .insert(module_id.to_string(), unix_ms_now());
2795    }
2796
2797    /// Return how long ago a rescan removed this module in milliseconds.
2798    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2799        self.removal_tombstones
2800            .lock()
2801            .unwrap_or_else(|poisoned| poisoned.into_inner())
2802            .get(module_id)
2803            .copied()
2804            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2805    }
2806
2807    /// Retire a reserved-id gate only after its module has left supervision.
2808    ///
2809    /// A retained gate has no live nonce (`None`), so releasing any other entry
2810    /// would weaken a currently configured or otherwise active reservation.
2811    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2812        if self.get(module_id).is_some() {
2813            return false;
2814        }
2815        let mut reserved_nonces = self
2816            .reserved_nonces
2817            .lock()
2818            .unwrap_or_else(|poisoned| poisoned.into_inner());
2819        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2820            return false;
2821        }
2822        reserved_nonces.remove(module_id);
2823        true
2824    }
2825
2826    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2827        Arc::clone(&self.operation_lock)
2828    }
2829}
2830
2831/// Process supervisor for subc-owned singleton modules.
2832#[derive(Debug, Clone)]
2833pub struct Supervisor {
2834    registry: Arc<Registry>,
2835    restart_policy: RestartPolicy,
2836    drain_timeout: Duration,
2837    connection_file_path: Option<PathBuf>,
2838    capture_logs_dir: Option<PathBuf>,
2839    forwarding: Option<Arc<ForwardingTable>>,
2840    process_liveness: Arc<SupervisorProcessLiveness>,
2841    supervisor_handle: Option<SupervisorHandle>,
2842    health: HealthConfig,
2843    host_health: HostHealth,
2844    daemon_start_clock: crate::clock::StartClock,
2845    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2846    spawn_events: SpawnEventFeed,
2847    provenance_probe: ExecutableIdentityProbe,
2848    /// Every process spawned through this supervisor (and its clones) and not
2849    /// yet reaped, so daemon shutdown can end them.
2850    child_roster: ChildRoster,
2851    #[cfg(target_os = "linux")]
2852    cgroup_placement: Option<subc_cgroup::Placement>,
2853    #[cfg(test)]
2854    test_after_first_spawn: AfterFirstSpawnHook,
2855}
2856
2857/// A roster entry whose monitor and first process have not started yet.
2858/// Keeping the command receiver here lets discovery serve status immediately
2859/// while start/stop requests wait until bootstrap has published the connection file.
2860pub(crate) struct RegisteredConfiguredModule {
2861    spec: ModuleSpec,
2862    runtime: SupervisorRuntimeConfig,
2863    module: SupervisedModule,
2864    commands: mpsc::Receiver<SupervisorCommand>,
2865}
2866
2867/// Test-only hook run on the path that takes on a new module, right after its
2868/// first `spawn_child` returns (the process exists and could already be
2869/// registering) and before that process is handed to the module's supervise
2870/// loop and put on the roster. Lets a test observe what a fast child would see
2871/// in that window without racing a real one.
2872#[cfg(test)]
2873#[derive(Clone, Default)]
2874struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2875
2876#[cfg(test)]
2877type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2878
2879#[cfg(test)]
2880impl fmt::Debug for AfterFirstSpawnHook {
2881    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2882        f.write_str("AfterFirstSpawnHook")
2883    }
2884}
2885
2886#[cfg(test)]
2887impl AfterFirstSpawnHook {
2888    fn run(&self, module_id: &str) {
2889        if let Some(hook) = &self.0 {
2890            hook(module_id);
2891        }
2892    }
2893}
2894
2895impl Supervisor {
2896    #[cfg(test)]
2897    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2898        let supervisor = Self::new(registry, policy);
2899        #[cfg(target_os = "macos")]
2900        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2901        supervisor
2902    }
2903    /// Verify the trampoline once when configured. Missing private OS support
2904    /// refuses every macOS launch by name but does not stop the daemon's control
2905    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2906    /// entry point; the library must not exec an arbitrary hosting program.
2907    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2908        let path = path.into();
2909        #[cfg(target_os = "macos")]
2910        {
2911            let result = probe_privacy_trampoline(&path).map(|()| path);
2912            if let Err(cause) = &result {
2913                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2914            }
2915            self.child_roster.set_privacy_trampoline(result);
2916        }
2917        #[cfg(not(target_os = "macos"))]
2918        let _ = path;
2919        self
2920    }
2921    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2922    /// capability-requirement log to read.
2923    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2924        self.child_roster.shutdown_flag()
2925    }
2926
2927    /// The first step of an announced daemon shutdown, before the notice and
2928    /// before any connection is closed.
2929    ///
2930    /// Sets the daemon-shutdown flag first: from here on no module is
2931    /// respawned (crash restart, operator restart, or swap), and every child
2932    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2933    /// the module exits on the EOF this shutdown gives it or is signalled by a
2934    /// service manager that kills the whole cgroup. Then writes the journal's
2935    /// shutdown marker, which records the instant and closes this daemon
2936    /// incarnation's stretch of the journal.
2937    #[cfg(unix)]
2938    pub(crate) fn begin_daemon_shutdown(&self) {
2939        self.child_roster.close();
2940        if let Some(journal) = &self.terminal_journal {
2941            journal.stamp_shutdown();
2942        }
2943    }
2944
2945    /// Announce a cut while established connections can still carry replies.
2946    /// These budgets bound the attempt to deliver notices and the wait for
2947    /// routes to drain; neither guarantees delivery or child completion, and
2948    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2949    #[cfg(unix)]
2950    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2951        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2952        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2953        let Some(forwarding) = &self.forwarding else {
2954            return Ok(());
2955        };
2956        let notice_started = Instant::now();
2957        let modules = forwarding
2958            .begin_daemon_drain()
2959            .map_err(SuperviseError::Forwarding)?;
2960        let deadline_ms =
2961            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2962        let shutdown_reason = self
2963            .supervisor_handle
2964            .as_ref()
2965            .map(SupervisorHandle::take_daemon_shutdown_reason)
2966            .unwrap_or(RouteCloseReason::Disable);
2967        let mut modules_told = 0;
2968        let mut notices = tokio::task::JoinSet::new();
2969        let mut drains = Vec::new();
2970        for connection in forwarding
2971            .module_connections()
2972            .map_err(SuperviseError::Forwarding)?
2973        {
2974            let target = ModuleDrainTarget {
2975                endpoint: connection.endpoint,
2976                sink: connection.sink,
2977                negotiated_ver: connection.negotiated_ver,
2978                abandoned_bindings: Vec::new(),
2979                excluded_subscriptions: 0,
2980            };
2981            notices.spawn(async move {
2982                (
2983                    true,
2984                    send_module_draining(
2985                        &connection.module_id,
2986                        shutdown_reason,
2987                        deadline_ms,
2988                        &target,
2989                        true,
2990                    )
2991                    .await,
2992                )
2993            });
2994        }
2995        for (module_id, endpoint) in modules {
2996            let Some(target) = forwarding
2997                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2998                .map_err(SuperviseError::Forwarding)?
2999            else {
3000                continue;
3001            };
3002            let routes = forwarding
3003                .endpoint_routes(target.endpoint)
3004                .map_err(SuperviseError::Forwarding)?;
3005            // Only an announced restart promises that modules are coming back.
3006            // Client route notices retain their existing reconnect semantics.
3007            let mut recipients = Vec::new();
3008            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3009            for route in routes {
3010                let client = route.goodbye_target;
3011                if let Some((_, channels)) = clients
3012                    .iter_mut()
3013                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3014                {
3015                    channels.push(client.channel);
3016                } else {
3017                    let channel = client.channel;
3018                    clients.push((client, vec![channel]));
3019                }
3020            }
3021            for (client, mut channels) in clients {
3022                channels.sort_unstable();
3023                channels.dedup();
3024                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3025                    module_id: module_id.clone(),
3026                    channels,
3027                    reason: RouteCloseReason::Restart,
3028                })
3029                .expect("route closing serializes");
3030                recipients.push((client.sink, client.negotiated_ver, closing));
3031            }
3032            for (sink, version, body) in recipients {
3033                notices.spawn(async move {
3034                    let frame = Frame::build_with_version(
3035                        version,
3036                        FrameType::Push,
3037                        control_flags(),
3038                        0,
3039                        0,
3040                        0,
3041                        body,
3042                    )
3043                    .expect("bounded lifecycle notice frame builds");
3044                    (false, sink.send_flushed(frame).await.is_ok())
3045                });
3046            }
3047            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3048            drains.push((module_id, target.endpoint, gauges));
3049        }
3050        // A quiet forwarding table is not proof that queued notices reached the
3051        // socket. Wait for writer flush acknowledgements before testing quiescence.
3052        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3053        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3054            match result {
3055                Ok((true, true)) => modules_told += 1,
3056                Ok((false, true)) => {}
3057                result => warn!(?result, "daemon shutdown notice delivery failed"),
3058            }
3059        }
3060        notices.abort_all();
3061        info!(
3062            ?shutdown_reason,
3063            modules_told, "daemon shutdown module notices delivered"
3064        );
3065        let notice_elapsed = notice_started.elapsed();
3066        let drain_started = Instant::now();
3067        let deadline = drain_started + DRAIN_BUDGET;
3068        let mut waits = tokio::task::JoinSet::new();
3069        for (module_id, endpoint, gauges) in drains {
3070            let forwarding = Arc::clone(forwarding);
3071            let mut runtime = self.runtime_config();
3072            runtime.health.cadence = Duration::from_millis(100);
3073            waits.spawn(async move {
3074                let wait_result = wait_for_forwarding_quiescence(
3075                    &forwarding,
3076                    &module_id,
3077                    &runtime,
3078                    endpoint,
3079                    deadline,
3080                    &gauges,
3081                    DrainScope::Endpoint(endpoint),
3082                    &mut DrainGaugeTrace::new(&gauges),
3083                )
3084                .await;
3085                // Store the measurement before the task returns: quiescence
3086                // may be reached just as the deadline stops the join loop from
3087                // collecting results. Unfinished or failed waits report false.
3088                forwarding
3089                    .record_daemon_drain_result(
3090                        endpoint,
3091                        drained_after_quiescence_wait(&wait_result),
3092                    )
3093                    .map_err(SuperviseError::Forwarding)?;
3094                wait_result
3095            });
3096        }
3097        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3098            if !matches!(result, Ok(Ok(true))) {
3099                warn!(?result, "daemon shutdown drain did not reach quiescence");
3100            }
3101        }
3102        // Measure only the forwarding drain, not child teardown or the caller's
3103        // signal delivery and process-reaping delays.
3104        info!(
3105            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3106            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3107            "daemon shutdown forwarding drain complete"
3108        );
3109        Ok(())
3110    }
3111
3112    /// The last step of a graceful daemon shutdown, after the notice and the
3113    /// drain: send every registered module a module GOODBYE, the same planned
3114    /// stop signal `ck module stop` gives, then close every connection so each
3115    /// subc module sees EOF and starts its own teardown, then end every
3116    /// supervised child that has not exited
3117    /// by its own deadline (its drain budget, capped). Modules lead their own
3118    /// process groups, so a
3119    /// service manager's group kill no longer reaches them; without this a
3120    /// child that does not stop on EOF (every `protocol: "none"` child, which
3121    /// has no connection) would outlive the daemon. Every wait is bounded (see
3122    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3123    #[cfg(unix)]
3124    pub(crate) async fn end_children_for_daemon_shutdown(
3125        &self,
3126        already_escalated: bool,
3127        escalate: impl std::future::Future<Output = ()>,
3128    ) {
3129        tokio::pin!(escalate);
3130        let mut escalated = already_escalated;
3131        if let Some(forwarding) = &self.forwarding {
3132            let reason = CloseReason::new(
3133                "daemon_shutdown",
3134                "the daemon is exiting after its shutdown notice and drain",
3135            );
3136            if escalated {
3137                // The operator asked to stop waiting: queue the GOODBYEs but
3138                // do not wait for them to be written.
3139                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3140            } else {
3141                tokio::select! {
3142                    biased;
3143                    _ = escalate.as_mut() => {
3144                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3145                        escalated = true;
3146                    }
3147                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3148                }
3149            }
3150            let closed = forwarding.close_all_connections(&reason);
3151            debug!(closed, "closed established connections for daemon shutdown");
3152        }
3153        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3154        // already completed and must not be polled again; the child shutdown
3155        // wait is told it is escalated and gets a future that never fires.
3156        let escalated_here = escalated && !already_escalated;
3157        let remaining_escalate = async move {
3158            if escalated_here {
3159                std::future::pending::<()>().await;
3160            } else {
3161                escalate.await;
3162            }
3163        };
3164        crate::child_roster::end_children_for_daemon_shutdown(
3165            &self.child_roster,
3166            escalated,
3167            remaining_escalate,
3168        )
3169        .await;
3170    }
3171
3172    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3173        Self {
3174            registry,
3175            restart_policy,
3176            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3177            connection_file_path: None,
3178            capture_logs_dir: None,
3179            forwarding: None,
3180            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3181            supervisor_handle: None,
3182            health: HealthConfig::default(),
3183            host_health: HostHealth::default(),
3184            daemon_start_clock: crate::clock::StartClock::capture(),
3185            terminal_journal: None,
3186            spawn_events: SpawnEventFeed::default(),
3187            provenance_probe: ExecutableIdentityProbe::default(),
3188            child_roster: ChildRoster::default(),
3189            #[cfg(target_os = "linux")]
3190            cgroup_placement: None,
3191            #[cfg(test)]
3192            test_after_first_spawn: AfterFirstSpawnHook::default(),
3193        }
3194    }
3195
3196    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3197        self.drain_timeout = drain_timeout;
3198        self
3199    }
3200
3201    /// The drain budget a module gets when its config entry sets none.
3202    pub(crate) fn default_drain_timeout(&self) -> Duration {
3203        self.drain_timeout
3204    }
3205
3206    pub fn with_process_liveness(
3207        mut self,
3208        process_liveness: Arc<SupervisorProcessLiveness>,
3209    ) -> Self {
3210        self.process_liveness = process_liveness;
3211        self
3212    }
3213
3214    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3215        self.connection_file_path = Some(connection_file_path.into());
3216        self
3217    }
3218
3219    /// Enables daemon-owned capture files for supervised stdout and stderr.
3220    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3221        self.capture_logs_dir = Some(logs_dir.into());
3222        self
3223    }
3224
3225    /// Names this daemon lifetime in spawn events, independently of whether a
3226    /// terminal journal is configured.
3227    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3228        // A millisecond start stamp can repeat after clock rollback or a rapid
3229        // restart. Use the connection file's random daemon_id instead: it already
3230        // identifies this daemon lifetime independently of the wall clock.
3231        self.spawn_events.configure_incarnation(daemon_incarnation);
3232        self
3233    }
3234
3235    /// Enables best-effort history shared by every supervised module. Without
3236    /// it, terminal history is kept only in each module's in-memory ring.
3237    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3238        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3239        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3240            path,
3241            daemon_incarnation,
3242        )));
3243        this
3244    }
3245
3246    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3247        self.child_roster.counters = forwarding.counters();
3248        self.forwarding = Some(forwarding);
3249        self
3250    }
3251
3252    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3253        self.host_health = supervisor_handle.host_health.clone();
3254        self.spawn_events = supervisor_handle.spawn_events.clone();
3255        self.supervisor_handle = Some(supervisor_handle);
3256        self
3257    }
3258
3259    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3260        self.health = health;
3261        self
3262    }
3263
3264    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3265    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3266    /// record is kept.
3267    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3268        self.child_roster.record_to(path.into());
3269        self
3270    }
3271
3272    #[cfg(target_os = "linux")]
3273    pub fn with_cgroup_placement(
3274        mut self,
3275        cgroup_placement: Option<subc_cgroup::Placement>,
3276    ) -> Self {
3277        self.child_roster
3278            .set_cgroup_placement(cgroup_placement.clone());
3279        self.cgroup_placement = cgroup_placement;
3280        self
3281    }
3282
3283    /// Spawn `spec.program` and start monitoring it.
3284    ///
3285    /// The child is expected to parse `--subc <connection-file-path>`, read the
3286    /// TCP+key connection file, authenticate to the already-running listener, and
3287    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3288    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3289        validate_spec(&spec)?;
3290        self.establish_identity(&spec);
3291
3292        let runtime = self.runtime_config();
3293        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3294        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3295        let registration_watch = watch_registration(&self.registry, &spec);
3296        let spawned = spawn_child(
3297            &spec,
3298            runtime.connection_file_path.as_deref(),
3299            self.supervisor_handle.as_ref(),
3300            &runtime.stderr_ring,
3301            runtime.capture_logs_dir.as_deref(),
3302            &runtime.child_roster,
3303            #[cfg(target_os = "linux")]
3304            runtime.cgroup_placement.as_ref(),
3305        );
3306        #[cfg(test)]
3307        self.test_after_first_spawn.run(&spec.module_id);
3308        let mut child = match spawned {
3309            Ok(child) => child,
3310            Err(err) => {
3311                // Unlike the configured paths, a failed `spawn` leaves nothing
3312                // on the roster, so the module must not stay marked configured.
3313                self.abandon_unrostered(&spec.module_id);
3314                return Err(err);
3315            }
3316        };
3317        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3318        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3319        if let Some(spawn) = activity_spawn {
3320            spawn.complete(&snapshot);
3321        }
3322
3323        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3324    }
3325
3326    /// Make `spec`'s module count as configured, with its identity gates
3327    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3328    /// exists.
3329    ///
3330    /// Every path that takes on a new module calls this before `spawn_child`.
3331    /// The order is the point: the child can connect, register, sync its
3332    /// scopes and ask about them as soon as it is spawned, and the module is
3333    /// only put on the roster after `spawn_child` returns. Were the mark set
3334    /// with the roster entry, a fast child would see its own owner reported
3335    /// as not configured, and a scoped `route.open` in that window would be
3336    /// refused as terminal `scope_not_live` ("will never sync") instead of
3337    /// retryable `scope_not_synced`.
3338    fn establish_identity(&self, spec: &ModuleSpec) {
3339        if let Some(supervisor_handle) = &self.supervisor_handle {
3340            supervisor_handle.apply_identity_configuration(spec);
3341            supervisor_handle.mark_configured(&spec.module_id);
3342        }
3343    }
3344
3345    /// Take back [`Self::establish_identity`]'s configured mark when the
3346    /// module will not be put on the roster after all.
3347    fn abandon_unrostered(&self, module_id: &str) {
3348        if let Some(supervisor_handle) = &self.supervisor_handle {
3349            supervisor_handle.unmark_configured_unless_rostered(module_id);
3350        }
3351    }
3352
3353    /// Record a freshly spawned first process as running. On failure the
3354    /// module never reaches the roster, so its configured mark is taken back.
3355    fn mark_first_process_running(
3356        &self,
3357        spec: &ModuleSpec,
3358        runtime: &SupervisorRuntimeConfig,
3359        snapshot: &SharedSnapshot,
3360        child: &mut SupervisedChild,
3361    ) -> Result<(), SuperviseError> {
3362        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3363            containment::terminate_blocking(&runtime.child_roster.containment);
3364            self.abandon_unrostered(&spec.module_id);
3365            return Err(err);
3366        }
3367        self.process_liveness
3368            .track(spec.module_id.clone(), Arc::clone(snapshot));
3369        Ok(())
3370    }
3371
3372    /// Start supervising a module declared in daemon configuration.
3373    ///
3374    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3375    /// failures in the supervisor handle so operator-facing `supervisor.list`
3376    /// reflects every configured module while daemon startup continues.
3377    pub fn supervise_configured(
3378        &self,
3379        spec: ModuleSpec,
3380        enabled: bool,
3381    ) -> Result<SupervisedModule, SuperviseError> {
3382        self.supervise_configured_requested(
3383            spec,
3384            enabled,
3385            self.health.clone(),
3386            None,
3387            self.restart_policy,
3388            DEFAULT_REGISTER_TIMEOUT,
3389            None,
3390        )
3391    }
3392
3393    /// Supervise a configured module with its own health, drain, and crash
3394    /// budget. The restart policy is per-module because the config file is:
3395    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3396    /// module that is expensive to restart should not be forced onto the same
3397    /// budget as one that is cheap.
3398    pub fn supervise_configured_with_health(
3399        &self,
3400        spec: ModuleSpec,
3401        enabled: bool,
3402        health: HealthConfig,
3403        drain_timeout_ms: Option<u64>,
3404        restart_policy: RestartPolicy,
3405    ) -> Result<SupervisedModule, SuperviseError> {
3406        self.supervise_configured_requested(
3407            spec,
3408            enabled,
3409            health,
3410            drain_timeout_ms,
3411            restart_policy,
3412            DEFAULT_REGISTER_TIMEOUT,
3413            None,
3414        )
3415    }
3416
3417    #[allow(clippy::too_many_arguments)]
3418    pub(crate) fn supervise_configured_requested(
3419        &self,
3420        spec: ModuleSpec,
3421        enabled: bool,
3422        health: HealthConfig,
3423        drain_timeout_ms: Option<u64>,
3424        restart_policy: RestartPolicy,
3425        register_timeout: Duration,
3426        requester: Option<Principal>,
3427    ) -> Result<SupervisedModule, SuperviseError> {
3428        let registered = self.register_configured_with_health(
3429            spec,
3430            enabled,
3431            health,
3432            drain_timeout_ms,
3433            restart_policy,
3434            register_timeout,
3435        )?;
3436        let daemon_initiated = requester.is_none();
3437        self.start_registered_configured(registered, requester, daemon_initiated)
3438    }
3439
3440    /// Record a configured module as starting or disabled, without launching a
3441    /// process or running its monitor. Bootstrap registers the entire roster
3442    /// before publishing discovery, then activates it once children can read
3443    /// the connection file passed in their `--subc` argument.
3444    pub(crate) fn register_configured_with_health(
3445        &self,
3446        spec: ModuleSpec,
3447        enabled: bool,
3448        health: HealthConfig,
3449        drain_timeout_ms: Option<u64>,
3450        restart_policy: RestartPolicy,
3451        register_timeout: Duration,
3452    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3453        validate_spec(&spec)?;
3454        self.establish_identity(&spec);
3455
3456        let mut runtime = self.runtime_config();
3457        runtime.health = health;
3458        runtime.restart_policy = restart_policy;
3459        runtime.register_timeout = register_timeout;
3460        if let Some(ms) = drain_timeout_ms {
3461            runtime.drain_timeout = Duration::from_millis(ms);
3462            *runtime
3463                .effective_drain_timeout
3464                .lock()
3465                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3466        }
3467        let snapshot = if enabled {
3468            SupervisorSnapshot::starting()
3469        } else {
3470            SupervisorSnapshot::disabled()
3471        };
3472        let snapshot = {
3473            let mut snapshot = snapshot;
3474            if enabled {
3475                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3476                {
3477                    snapshot.waiting_for_previous_pid = Some(pid);
3478                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3479                }
3480            }
3481            snapshot
3482        };
3483        let (module, commands) =
3484            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3485        Ok(RegisteredConfiguredModule {
3486            spec,
3487            runtime,
3488            module,
3489            commands,
3490        })
3491    }
3492
3493    /// Activate a previously registered module. Queued control requests are
3494    /// served by the monitor after the initial configured spawn has settled.
3495    pub(crate) fn start_registered_configured(
3496        &self,
3497        registered: RegisteredConfiguredModule,
3498        requester: Option<Principal>,
3499        daemon_initiated: bool,
3500    ) -> Result<SupervisedModule, SuperviseError> {
3501        let RegisteredConfiguredModule {
3502            spec,
3503            mut runtime,
3504            module,
3505            commands,
3506        } = registered;
3507        runtime.start_requester = requester;
3508        runtime.recovery_is_daemon = daemon_initiated;
3509        let snapshot = &module.inner.snapshot;
3510        let should_spawn = {
3511            let state = lock_snapshot(snapshot)?;
3512            state.enabled && state.waiting_for_previous_pid.is_none()
3513        };
3514        let child = if should_spawn {
3515            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3516            let registration_watch = watch_registration(&self.registry, &spec);
3517            let spawned = spawn_child(
3518                &spec,
3519                runtime.connection_file_path.as_deref(),
3520                self.supervisor_handle.as_ref(),
3521                &runtime.stderr_ring,
3522                runtime.capture_logs_dir.as_deref(),
3523                &runtime.child_roster,
3524                #[cfg(target_os = "linux")]
3525                runtime.cgroup_placement.as_ref(),
3526            );
3527            #[cfg(test)]
3528            self.test_after_first_spawn.run(&spec.module_id);
3529            match spawned {
3530                Ok(mut child) => {
3531                    arm_registration_deadline(
3532                        &mut child,
3533                        runtime.register_timeout,
3534                        registration_watch,
3535                    );
3536                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3537                    if let Some(spawn) = activity_spawn {
3538                        spawn.complete(snapshot);
3539                    }
3540                    Some(child)
3541                }
3542                Err(err) => {
3543                    if runtime.health.critical {
3544                        error!(
3545                            module_id = %spec.module_id,
3546                            program = %spec.program.display(),
3547                            error = %err,
3548                            "critical configured module failed to spawn; marking failed and alerting"
3549                        );
3550                    } else {
3551                        error!(
3552                            module_id = %spec.module_id,
3553                            program = %spec.program.display(),
3554                            error = %err,
3555                            "configured module failed to spawn; marking failed and continuing"
3556                        );
3557                    }
3558                    fail_snapshot(
3559                        snapshot,
3560                        Some(&spec.module_id),
3561                        None,
3562                        &runtime.failure_context(daemon_initiated),
3563                    );
3564                    None
3565                }
3566            }
3567        } else {
3568            None
3569        };
3570        self.start_monitor(spec, runtime, &module, child, commands);
3571        Ok(module)
3572    }
3573
3574    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3575        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3576        SupervisorRuntimeConfig {
3577            registry: self.registry.clone(),
3578            recovery_is_daemon: true,
3579            start_requester: None,
3580            pending_activity_restart: false,
3581            provenance_probe: self.provenance_probe.clone(),
3582            scheduled_respawn: Arc::default(),
3583            deferred_reload_reply: Arc::default(),
3584            restart_policy: self.restart_policy,
3585            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3586            drain_timeout: self.drain_timeout,
3587            // Shared with this module's roster copy: daemon shutdown waits on
3588            // each child for the module's own drain budget, as resolved now.
3589            child_roster: self
3590                .child_roster
3591                .for_module(Arc::clone(&effective_drain_timeout)),
3592            effective_drain_timeout,
3593            default_drain_timeout: self.drain_timeout,
3594            health: self.health.clone(),
3595            host_health: self.host_health.clone(),
3596            connection_file_path: self.connection_file_path.clone(),
3597            capture_logs_dir: self.capture_logs_dir.clone(),
3598            forwarding: self.forwarding.clone(),
3599            supervisor_handle: self.supervisor_handle.clone(),
3600            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3601            terminal_ring: Arc::new(Mutex::new(
3602                TerminalRing::new(
3603                    TerminalRingConfig::default(),
3604                    self.daemon_start_clock.started_at_ms(),
3605                )
3606                .with_start_clock(self.daemon_start_clock)
3607                .with_journal(self.terminal_journal.clone())
3608                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3609            )),
3610            spawn_events: self.spawn_events.clone(),
3611            #[cfg(target_os = "linux")]
3612            cgroup_placement: self.cgroup_placement.clone(),
3613            #[cfg(test)]
3614            test_seed_stale_facts_before_enable_spawn: false,
3615            #[cfg(test)]
3616            test_reload_exit_record_gate: None,
3617        }
3618    }
3619
3620    fn supervised_module(
3621        &self,
3622        spec: ModuleSpec,
3623        runtime: SupervisorRuntimeConfig,
3624        snapshot: SharedSnapshot,
3625        child: Option<SupervisedChild>,
3626    ) -> SupervisedModule {
3627        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3628        self.start_monitor(spec, runtime, &module, child, commands);
3629        module
3630    }
3631
3632    fn register_module(
3633        &self,
3634        spec: &ModuleSpec,
3635        runtime: &SupervisorRuntimeConfig,
3636        snapshot: SharedSnapshot,
3637    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3638        runtime.host_health.track(spec, &snapshot);
3639        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3640            spec: spec.clone(),
3641            health: runtime.health.clone(),
3642        }));
3643        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3644        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3645        // The module's OWN policy, which may be its per-module config rather than
3646        // the supervisor-wide one; status must report the budget the supervise
3647        // loop actually enforces.
3648        let restart_policy = runtime.restart_policy;
3649        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3650        let (tx, rx) = mpsc::channel(4);
3651        let module_id = spec.module_id.clone();
3652        let module = SupervisedModule {
3653            inner: Arc::new(SupervisedModuleInner {
3654                module_id: module_id.clone(),
3655                registry: Arc::clone(&self.registry),
3656                snapshot,
3657                configuration,
3658                stderr_ring,
3659                terminal_ring,
3660                commands: tx,
3661                monitor: Mutex::new(None),
3662                restart_policy,
3663                effective_drain_timeout,
3664                provenance_probe: self.provenance_probe.clone(),
3665                retained: runtime.child_roster.retained_view(),
3666            }),
3667        };
3668        // Identity gates and the configured mark must precede the roster entry,
3669        // so a client never sees a module without its admission policy.
3670        if let Some(supervisor_handle) = &self.supervisor_handle {
3671            supervisor_handle.insert(module.clone());
3672        }
3673        (module, rx)
3674    }
3675
3676    fn start_monitor(
3677        &self,
3678        spec: ModuleSpec,
3679        runtime: SupervisorRuntimeConfig,
3680        module: &SupervisedModule,
3681        child: Option<SupervisedChild>,
3682        commands: mpsc::Receiver<SupervisorCommand>,
3683    ) {
3684        let monitor = tokio::spawn(supervise_loop(
3685            spec,
3686            runtime,
3687            Arc::clone(&self.registry),
3688            Arc::clone(&self.process_liveness),
3689            Arc::clone(&module.inner.snapshot),
3690            child,
3691            commands,
3692        ));
3693        *module
3694            .inner
3695            .monitor
3696            .lock()
3697            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3698    }
3699}
3700
3701impl Default for Supervisor {
3702    fn default() -> Self {
3703        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3704    }
3705}
3706
3707/// Handle to one supervised child process.
3708#[derive(Clone)]
3709pub struct SupervisedModule {
3710    inner: Arc<SupervisedModuleInner>,
3711}
3712
3713struct SupervisedModuleInner {
3714    retained: crate::child_roster::RetainedView,
3715    module_id: String,
3716    registry: Arc<Registry>,
3717    snapshot: SharedSnapshot,
3718    configuration: Arc<Mutex<SupervisedConfiguration>>,
3719    stderr_ring: Arc<Mutex<StderrRing>>,
3720    terminal_ring: Arc<Mutex<TerminalRing>>,
3721    commands: mpsc::Sender<SupervisorCommand>,
3722    monitor: Mutex<Option<JoinHandle<()>>>,
3723    /// Copied from the supervisor's runtime config at spawn so `status()` can
3724    /// report the restart budget without reaching back into the supervisor. The
3725    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3726    restart_policy: RestartPolicy,
3727    effective_drain_timeout: Arc<Mutex<Duration>>,
3728    provenance_probe: ExecutableIdentityProbe,
3729}
3730
3731impl fmt::Debug for SupervisedModule {
3732    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3733        f.debug_struct("SupervisedModule")
3734            .field("module_id", &self.inner.module_id)
3735            .field("status", &self.status())
3736            .finish_non_exhaustive()
3737    }
3738}
3739
3740impl SupervisedModule {
3741    pub fn module_id(&self) -> &str {
3742        &self.inner.module_id
3743    }
3744
3745    /// Test-only: put one probe miss on the streak, the way
3746    /// `handle_health_probe_failure` does, so tests can assert what a later
3747    /// event does to the streak without driving the whole probe loop.
3748    #[cfg(test)]
3749    pub(crate) fn record_health_probe_failure_for_test(
3750        &self,
3751        detail: &str,
3752    ) -> Result<(), SuperviseError> {
3753        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3754            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3755            state.health.detail = Some(detail.to_string());
3756        })
3757    }
3758
3759    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3760        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3761    }
3762
3763    pub(crate) fn registration_end_reason(
3764        &self,
3765    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3766        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3767        Ok(match snapshot.state {
3768            ModuleState::Draining if snapshot.draining_to_replace => {
3769                Some(RegistrationEndReason::SupervisorRestart)
3770            }
3771            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3772            ModuleState::Restarting if snapshot.draining_to_replace => {
3773                Some(RegistrationEndReason::SupervisorRestart)
3774            }
3775            _ => None,
3776        })
3777    }
3778
3779    /// The module's retained stderr, newest lines last.
3780    ///
3781    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3782    /// module, `supervisor.list` renders every module, and putting it in the
3783    /// shared snapshot would make each status read carry a payload almost nobody
3784    /// asked for. Callers that want the text ask for it.
3785    pub fn stderr_tail(
3786        &self,
3787        max_lines: Option<usize>,
3788        max_bytes: Option<usize>,
3789    ) -> StderrTailSnapshot {
3790        self.inner
3791            .stderr_ring
3792            .lock()
3793            .unwrap_or_else(|poisoned| poisoned.into_inner())
3794            .snapshot(max_lines, max_bytes)
3795    }
3796
3797    /// The module's bounded terminal history, oldest retained exit first.
3798    ///
3799    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3800    /// daemon whose in-memory history was necessarily reset.
3801    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3802        self.inner
3803            .terminal_ring
3804            .lock()
3805            .unwrap_or_else(|poisoned| poisoned.into_inner())
3806            .snapshot()
3807    }
3808
3809    /// Retained observations from the current ring and all journal generations.
3810    ///
3811    /// Blocking: this reads the journal files. Async callers use
3812    /// `Self::read_durable_terminal_history`.
3813    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3814        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3815    }
3816
3817    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3818    /// read (up to every retained generation) never occupies a runtime worker.
3819    /// Fails only if the blocking task could not finish (runtime shutdown or a
3820    /// panic in the read).
3821    pub(crate) async fn read_durable_terminal_history(
3822        &self,
3823    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3824        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3825        let module_id = self.inner.module_id.clone();
3826        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3827            .await
3828    }
3829
3830    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3831        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3832            .map(|(status, _)| status)
3833    }
3834
3835    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3836        let state = lock_snapshot(&self.inner.snapshot)?;
3837        Ok(state
3838            .failed_retry
3839            .pending
3840            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3841            .map(|retry| {
3842                unix_ms_now().saturating_add(
3843                    u64::try_from(
3844                        retry
3845                            .deadline
3846                            .saturating_duration_since(Instant::now())
3847                            .as_millis(),
3848                    )
3849                    .unwrap_or(u64::MAX),
3850                )
3851            }))
3852    }
3853
3854    pub(crate) fn record_deliberate_severance(
3855        &self,
3856        identity: ProcessIdentity,
3857    ) -> Result<bool, SuperviseError> {
3858        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3859        if snapshot.pid != Some(identity.pid)
3860            || snapshot.process_start_time != Some(identity.start_time)
3861        {
3862            return Ok(false);
3863        }
3864        snapshot.deliberate_severance = Some(identity);
3865        Ok(true)
3866    }
3867
3868    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3869    ///
3870    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3871    /// does not produce reader-observability logs.
3872    pub(crate) fn status_for_control(
3873        &self,
3874        caller: &'static str,
3875    ) -> Result<ModuleStatus, SuperviseError> {
3876        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3877            .map(|(status, _)| status)
3878    }
3879
3880    fn status_with_snapshot_lock(
3881        &self,
3882        snapshot: &SharedSnapshot,
3883        caller: Option<&'static str>,
3884    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3885        let mut guard = match caller {
3886            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3887            None => lock_snapshot(snapshot)?,
3888        };
3889        // Read the budget through the pruning path so a reader sees the same
3890        // in-window count the restart decision would use, not a stale total.
3891        let restart_count =
3892            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3893        let snapshot = guard.clone();
3894        drop(guard);
3895        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3896            SuperviseError::StatePoisoned {
3897                module_id: Some(self.inner.module_id.clone()),
3898            }
3899        })?;
3900        let registration_active = self
3901            .inner
3902            .registry
3903            .get_module(&self.inner.module_id)
3904            .map_err(SuperviseError::Registry)?
3905            .is_some();
3906        let protocol = snapshot
3907            .spawned_protocol
3908            .unwrap_or(self.declared_protocol()?);
3909        let running_process =
3910            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3911        // Registration is the difference between the two protocols and the only
3912        // one: a subc module that has not registered cannot serve a request even
3913        // though its process is up, and a `none` module never registers at all,
3914        // so requiring it there would pin `live` to false for the whole life of
3915        // a perfectly healthy process.
3916        let live = match protocol {
3917            ModuleProtocol::Subc => running_process && registration_active,
3918            ModuleProtocol::None => running_process,
3919        };
3920        // Read only the retained cache: a status read must not resend signals,
3921        // rewrite the record, or keep the containment resource alive.
3922        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3923        let carry_configured = self
3924            .inner
3925            .configuration
3926            .lock()
3927            .map_err(|_| SuperviseError::StatePoisoned {
3928                module_id: Some(self.inner.module_id.clone()),
3929            })?
3930            .spec
3931            .carry_containment;
3932        let recovery_block = carries_containment(
3933            carry_configured,
3934            self.inner
3935                .retained
3936                .has_carried_container(&self.inner.module_id),
3937        )
3938        .then(|| self.inner.retained.recovery_block())
3939        .flatten();
3940        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3941        let pid = snapshot.reported_pid();
3942        let mut health = snapshot.health;
3943        if let Some(detail) = recovery_block
3944            .as_ref()
3945            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3946        {
3947            health.detail = Some(match health.detail {
3948                Some(previous) => format!("{previous}; {detail}"),
3949                None => detail.clone(),
3950            });
3951        }
3952
3953        Ok((
3954            ModuleStatus {
3955                waiting_for_previous_pid: retained
3956                    .as_ref()
3957                    .map(|(pid, _)| *pid)
3958                    .filter(|pid| *pid > 0)
3959                    .or(snapshot.waiting_for_previous_pid),
3960                waiting_for_previous_detail: recovery_block
3961                    .or_else(|| retained.map(|(_, detail)| detail))
3962                    .or(snapshot.waiting_for_previous_detail),
3963                module_id: self.inner.module_id.clone(),
3964                launch_nonce_source,
3965                state: snapshot.state,
3966                enabled: snapshot.enabled,
3967                process_alive: snapshot.process_alive,
3968                registration_active,
3969                protocol,
3970                live,
3971                restart_count,
3972                lifetime_restarts: snapshot.lifetime_restarts,
3973                spawn_generation: snapshot.spawn_generation,
3974                max_restarts: self.inner.restart_policy.max_restarts,
3975                restart_window: self.inner.restart_policy.window,
3976                drain_timeout,
3977                restart_backoff: self.inner.restart_policy.backoff,
3978                restart_max_backoff: self.inner.restart_policy.max_backoff,
3979                pid,
3980                spawned_at_ms: snapshot.spawned_at_ms,
3981                spawned_from: snapshot.spawned_from,
3982                launch_spec_digest: snapshot.launch_spec_digest,
3983                process_start_time: snapshot.process_start_time,
3984                last_exit: snapshot.last_exit,
3985                health,
3986            },
3987            snapshot.spawned_file_identity.clone(),
3988        ))
3989    }
3990
3991    #[cfg(test)]
3992    pub(crate) fn hold_snapshot_for_test(
3993        &self,
3994        acquired: std::sync::mpsc::Sender<()>,
3995        hold: Duration,
3996    ) -> std::thread::JoinHandle<()> {
3997        let snapshot = Arc::clone(&self.inner.snapshot);
3998        std::thread::spawn(move || {
3999            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
4000            acquired
4001                .send(())
4002                .expect("test receiver waits for snapshot lock");
4003            std::thread::sleep(hold);
4004        })
4005    }
4006
4007    /// The status and the running-image check for `supervisor.provenance`,
4008    /// taken from one status read. The exec acknowledgement can land between
4009    /// two separate reads, and the reply would then pair "no pid yet" with an
4010    /// image observed after the module started, which describes no single
4011    /// moment.
4012    pub(crate) async fn status_and_running_image_agreement(
4013        &self,
4014    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
4015        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
4016        let image = self
4017            .inner
4018            .provenance_probe
4019            .observe(
4020                status.pid,
4021                status.spawned_from.as_deref(),
4022                identity,
4023                status.process_start_time,
4024            )
4025            .await;
4026        Ok((status, image))
4027    }
4028
4029    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4030        let snapshot = match lock_snapshot(&self.inner.snapshot) {
4031            Ok(snapshot) => snapshot.clone(),
4032            Err(_) => {
4033                return subc_control::RunningImageAgreement::Unavailable {
4034                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
4035                };
4036            }
4037        };
4038        self.inner
4039            .provenance_probe
4040            .observe(
4041                snapshot.reported_pid(),
4042                snapshot.spawned_from.as_deref(),
4043                snapshot.spawned_file_identity.clone(),
4044                snapshot.process_start_time,
4045            )
4046            .await
4047    }
4048
4049    /// Memory and CPU time of the module's current process, read now. Only the
4050    /// process the supervisor spawned is read, not processes it has started.
4051    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4052        #[cfg(windows)]
4053        {
4054            let image = lock_snapshot(&self.inner.snapshot)
4055                .ok()
4056                .and_then(|snapshot| {
4057                    snapshot
4058                        .spawned_file_identity
4059                        .as_ref()
4060                        .and_then(|identity| identity.windows_image.clone())
4061                });
4062            match image {
4063                Some(image) => {
4064                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
4065                }
4066                None => subc_control::ChildResourceUsage::Unavailable {
4067                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4068                },
4069            }
4070        }
4071        #[cfg(not(windows))]
4072        {
4073            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4074                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4075                Err(_) => {
4076                    return subc_control::ChildResourceUsage::Unavailable {
4077                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4078                    }
4079                }
4080            };
4081            crate::child_resources::read(pid, start_time)
4082        }
4083    }
4084
4085    #[cfg(all(test, any(target_os = "linux", windows)))]
4086    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4087        self.inner
4088            .snapshot
4089            .try_lock()
4090            .map(|snapshot| snapshot.subtree_target.clone())
4091            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4092    }
4093
4094    #[cfg(test)]
4095    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4096        self.inner.snapshot.lock().unwrap().subtree_target = target;
4097    }
4098
4099    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4100        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4101        Ok(match snapshot.state {
4102            ModuleState::Restarting => true,
4103            ModuleState::Failed | ModuleState::Disabled => false,
4104            _ if snapshot.enabled && snapshot.respawn_pending => true,
4105            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4106        })
4107    }
4108
4109    #[cfg(test)]
4110    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4111        self.is_warming_with_snapshot_lock(None)
4112    }
4113
4114    pub(crate) fn is_warming_for_control(
4115        &self,
4116        caller: &'static str,
4117    ) -> Result<bool, SuperviseError> {
4118        self.is_warming_with_snapshot_lock(Some(caller))
4119    }
4120
4121    fn is_warming_with_snapshot_lock(
4122        &self,
4123        caller: Option<&'static str>,
4124    ) -> Result<bool, SuperviseError> {
4125        let snapshot = match caller {
4126            Some(caller) => {
4127                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4128            }
4129            None => lock_snapshot(&self.inner.snapshot)?,
4130        }
4131        .clone();
4132        Ok(matches!(
4133            snapshot.state,
4134            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4135        ))
4136    }
4137
4138    /// Drain the module and stop monitoring it.
4139    pub async fn drain(&self) -> Result<(), SuperviseError> {
4140        self.stop().await
4141    }
4142
4143    pub(crate) async fn retire_requested(
4144        &self,
4145        requester: Option<Principal>,
4146    ) -> Result<(), SuperviseError> {
4147        match self.state()? {
4148            ModuleState::Stopped => return Ok(()),
4149            ModuleState::Failed
4150                if lock_snapshot(&self.inner.snapshot)?
4151                    .failed_retry
4152                    .pending
4153                    .is_none() =>
4154            {
4155                return Ok(())
4156            }
4157            ModuleState::Failed
4158            | ModuleState::Starting
4159            | ModuleState::Running
4160            | ModuleState::Unresponsive
4161            | ModuleState::Restarting
4162            | ModuleState::Draining
4163            | ModuleState::Disabled => {}
4164        }
4165
4166        let (reply_tx, reply_rx) = oneshot::channel();
4167        self.inner
4168            .commands
4169            .send(SupervisorCommand::Retire {
4170                requester,
4171                reply: reply_tx,
4172            })
4173            .await
4174            .map_err(|_| SuperviseError::CommandClosed {
4175                module_id: self.inner.module_id.clone(),
4176            })?;
4177        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4178            module_id: self.inner.module_id.clone(),
4179        })?
4180    }
4181
4182    pub async fn stop(&self) -> Result<(), SuperviseError> {
4183        self.stop_requested(None).await
4184    }
4185
4186    pub(crate) async fn stop_requested(
4187        &self,
4188        requester: Option<Principal>,
4189    ) -> Result<(), SuperviseError> {
4190        match self.state()? {
4191            ModuleState::Stopped => return Ok(()),
4192            ModuleState::Failed
4193                if lock_snapshot(&self.inner.snapshot)?
4194                    .failed_retry
4195                    .pending
4196                    .is_none() =>
4197            {
4198                return Ok(())
4199            }
4200            ModuleState::Failed
4201            | ModuleState::Starting
4202            | ModuleState::Running
4203            | ModuleState::Unresponsive
4204            | ModuleState::Restarting
4205            | ModuleState::Draining
4206            | ModuleState::Disabled => {}
4207        }
4208
4209        let (reply_tx, reply_rx) = oneshot::channel();
4210        self.inner
4211            .commands
4212            .send(SupervisorCommand::Drain {
4213                requester,
4214                reply: reply_tx,
4215            })
4216            .await
4217            .map_err(|_| SuperviseError::CommandClosed {
4218                module_id: self.inner.module_id.clone(),
4219            })?;
4220        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4221            module_id: self.inner.module_id.clone(),
4222        })?
4223    }
4224
4225    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4226        self.restart_requested(drain_timeout_ms, None).await
4227    }
4228
4229    pub(crate) async fn restart_requested(
4230        &self,
4231        drain_timeout_ms: Option<u64>,
4232        requester: Option<Principal>,
4233    ) -> Result<(), SuperviseError> {
4234        let queued_at = Instant::now();
4235        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4236        let (reply_tx, reply_rx) = oneshot::channel();
4237        self.inner
4238            .commands
4239            .send(SupervisorCommand::Restart {
4240                requester,
4241                drain_timeout_ms,
4242                received_at_generation,
4243                queued_at,
4244                reply: reply_tx,
4245            })
4246            .await
4247            .map_err(|_| SuperviseError::CommandClosed {
4248                module_id: self.inner.module_id.clone(),
4249            })?;
4250        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4251            module_id: self.inner.module_id.clone(),
4252        })?
4253    }
4254
4255    /// Blue/green restart: see `SupervisorCommand::Swap` and the
4256    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4257    /// process then drains in the background of the supervise loop) or has
4258    /// failed, leaving the old process serving.
4259    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4260        self.swap_requested(ready_timeout, None).await
4261    }
4262
4263    pub(crate) async fn swap_requested(
4264        &self,
4265        ready_timeout: Option<Duration>,
4266        requester: Option<Principal>,
4267    ) -> Result<(), SuperviseError> {
4268        let (reply_tx, reply_rx) = oneshot::channel();
4269        self.inner
4270            .commands
4271            .send(SupervisorCommand::Swap {
4272                requester,
4273                ready_timeout,
4274                reply: reply_tx,
4275            })
4276            .await
4277            .map_err(|_| SuperviseError::CommandClosed {
4278                module_id: self.inner.module_id.clone(),
4279            })?;
4280        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4281            module_id: self.inner.module_id.clone(),
4282        })?
4283    }
4284
4285    pub async fn reload(&self) -> Result<(), SuperviseError> {
4286        self.reload_requested(None).await
4287    }
4288
4289    pub(crate) async fn reload_requested(
4290        &self,
4291        requester: Option<Principal>,
4292    ) -> Result<(), SuperviseError> {
4293        let (reply_tx, reply_rx) = oneshot::channel();
4294        self.inner
4295            .commands
4296            .send(SupervisorCommand::Reload {
4297                requester,
4298                reply: reply_tx,
4299            })
4300            .await
4301            .map_err(|_| SuperviseError::CommandClosed {
4302                module_id: self.inner.module_id.clone(),
4303            })?;
4304        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4305            module_id: self.inner.module_id.clone(),
4306        })?
4307    }
4308
4309    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4310        self.set_enabled_requested(enabled, None).await
4311    }
4312
4313    pub(crate) async fn set_enabled_requested(
4314        &self,
4315        enabled: bool,
4316        requester: Option<Principal>,
4317    ) -> Result<bool, SuperviseError> {
4318        let (reply_tx, reply_rx) = oneshot::channel();
4319        self.inner
4320            .commands
4321            .send(SupervisorCommand::SetEnabled {
4322                requester,
4323                enabled,
4324                reply: reply_tx,
4325            })
4326            .await
4327            .map_err(|_| SuperviseError::CommandClosed {
4328                module_id: self.inner.module_id.clone(),
4329            })?;
4330        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4331            module_id: self.inner.module_id.clone(),
4332        })?
4333    }
4334
4335    /// The current process's protocol, or the configured protocol when down.
4336    /// A rescan stores the next launch spec without changing how an existing
4337    /// process registers, serves routes, is probed, or exits.
4338    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4339        let configured = self
4340            .inner
4341            .configuration
4342            .lock()
4343            .map_err(|_| SuperviseError::StatePoisoned {
4344                module_id: Some(self.inner.module_id.clone()),
4345            })?
4346            .spec
4347            .protocol;
4348        let state = lock_snapshot(&self.inner.snapshot)?;
4349        Ok(state.spawned_protocol.unwrap_or(configured))
4350    }
4351
4352    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4353        let configuration =
4354            self.inner
4355                .configuration
4356                .lock()
4357                .map_err(|_| SuperviseError::StatePoisoned {
4358                    module_id: Some(self.inner.module_id.clone()),
4359                })?;
4360        Ok((configuration.spec.clone(), configuration.health.clone()))
4361    }
4362
4363    /// Replace this module's launch spec, keeping its health and drain policy,
4364    /// the way a rescan does for a changed config entry. The running process is
4365    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4366    #[cfg(any(test, feature = "test-support"))]
4367    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4368        let (_, health) = self.configuration()?;
4369        let drain_timeout_ms = u64::try_from(
4370            self.inner
4371                .effective_drain_timeout
4372                .lock()
4373                .unwrap_or_else(|poisoned| poisoned.into_inner())
4374                .as_millis(),
4375        )
4376        .ok();
4377        self.update_configuration(spec, health, drain_timeout_ms)
4378            .await
4379    }
4380
4381    pub(crate) async fn update_configuration(
4382        &self,
4383        spec: ModuleSpec,
4384        health: HealthConfig,
4385        drain_timeout_ms: Option<u64>,
4386    ) -> Result<(), SuperviseError> {
4387        if spec.module_id != self.inner.module_id {
4388            return Err(SuperviseError::InvalidSpec {
4389                reason: "a supervised module's module_id cannot be changed".to_string(),
4390            });
4391        }
4392        validate_spec(&spec)?;
4393        let (reply_tx, reply_rx) = oneshot::channel();
4394        self.inner
4395            .commands
4396            .send(SupervisorCommand::UpdateConfiguration {
4397                spec: spec.clone(),
4398                health: health.clone(),
4399                drain_timeout_ms,
4400                reply: reply_tx,
4401            })
4402            .await
4403            .map_err(|_| SuperviseError::CommandClosed {
4404                module_id: self.inner.module_id.clone(),
4405            })?;
4406        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4407            module_id: self.inner.module_id.clone(),
4408        })?;
4409        let mut configuration =
4410            self.inner
4411                .configuration
4412                .lock()
4413                .map_err(|_| SuperviseError::StatePoisoned {
4414                    module_id: Some(self.inner.module_id.clone()),
4415                })?;
4416        configuration.spec = spec;
4417        configuration.health = health;
4418        Ok(())
4419    }
4420}
4421
4422impl Drop for SupervisedModuleInner {
4423    fn drop(&mut self) {
4424        let Ok(mut monitor) = self.monitor.lock() else {
4425            return;
4426        };
4427        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4428            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4429                state.state = ModuleState::Stopped;
4430                clear_current_process_facts(state);
4431            });
4432            monitor.abort();
4433        }
4434        let _ = monitor.take();
4435    }
4436}
4437
4438#[derive(Debug)]
4439enum SupervisorCommand {
4440    Drain {
4441        requester: Option<Principal>,
4442        reply: oneshot::Sender<Result<(), SuperviseError>>,
4443    },
4444    Retire {
4445        requester: Option<Principal>,
4446        reply: oneshot::Sender<Result<(), SuperviseError>>,
4447    },
4448    Restart {
4449        requester: Option<Principal>,
4450        /// Operator override for this one restart's drain budget, in ms. `None`
4451        /// uses the module's configured/default budget; `Some(0)` cuts
4452        /// immediately (wedge bounce: a stuck request never settles, so
4453        /// waiting only delays recovery).
4454        drain_timeout_ms: Option<u64>,
4455        /// The module's `spawn_generation` when the request was received, before
4456        /// it waited in the command queue. A queued restart whose module has
4457        /// since spawned a newer process is already satisfied (see the handler).
4458        received_at_generation: u64,
4459        /// When the request entered the command queue, so the handler can log
4460        /// how long it waited behind the loop's other work.
4461        queued_at: Instant,
4462        reply: oneshot::Sender<Result<(), SuperviseError>>,
4463    },
4464    Reload {
4465        requester: Option<Principal>,
4466        reply: oneshot::Sender<Result<(), SuperviseError>>,
4467    },
4468    SetEnabled {
4469        requester: Option<Principal>,
4470        enabled: bool,
4471        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4472    },
4473    UpdateConfiguration {
4474        spec: ModuleSpec,
4475        health: HealthConfig,
4476        /// Per-module drain override from the new config; `None` re-resolves to
4477        /// the supervisor-wide default.
4478        drain_timeout_ms: Option<u64>,
4479        reply: oneshot::Sender<()>,
4480    },
4481    Swap {
4482        requester: Option<Principal>,
4483        /// How long the candidate may take to register and declare itself
4484        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4485        ready_timeout: Option<Duration>,
4486        /// Answered at cutover or failure; the incumbent's drain follows.
4487        reply: oneshot::Sender<Result<(), SuperviseError>>,
4488    },
4489}
4490
4491#[derive(Debug)]
4492pub enum SuperviseError {
4493    InvalidSpec {
4494        reason: String,
4495    },
4496    Spawn {
4497        program: PathBuf,
4498        source: io::Error,
4499        cgroup_path: Option<PathBuf>,
4500    },
4501    Cgroup {
4502        module_id: String,
4503        source: io::Error,
4504    },
4505    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4506    /// than spawn a reserved module without its identity binding.
4507    LaunchNonce {
4508        reason: String,
4509    },
4510    Wait {
4511        module_id: String,
4512        source: io::Error,
4513    },
4514    Kill {
4515        module_id: String,
4516        source: io::Error,
4517    },
4518    Forwarding(ForwardingError),
4519    Registry(RegistryError),
4520    ReloadUnavailable {
4521        module_id: String,
4522        reason: String,
4523    },
4524    /// An operator restart/reload was requested for a module that is currently
4525    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4526    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4527    /// by a restart, so these commands are rejected instead of re-enabling it.
4528    Disabled {
4529        module_id: String,
4530    },
4531    ReloadFailed {
4532        module_id: String,
4533        reason: String,
4534    },
4535    RegistrationStillActive {
4536        module_id: String,
4537        waited: Duration,
4538    },
4539    StatePoisoned {
4540        module_id: Option<String>,
4541    },
4542    CommandClosed {
4543        module_id: String,
4544    },
4545    /// A restart or reload arrived while a swap's candidate was warming. The
4546    /// swap owns the module until it cuts over or fails; a stop or disable
4547    /// would have aborted it instead.
4548    SwapInProgress {
4549        module_id: String,
4550    },
4551    /// A swap was refused before anything was spawned.
4552    SwapRefused {
4553        module_id: String,
4554        reason: SwapRefusal,
4555    },
4556    /// A swap spawned a candidate and gave up on it. The candidate has been
4557    /// killed and its slot freed; the incumbent was left serving and was never
4558    /// drained, except in the one `CutoverLost` case described on that arm.
4559    SwapFailed {
4560        module_id: String,
4561        arm: SwapFailureArm,
4562        detail: String,
4563        /// How the candidate exited, when it exited on its own before the
4564        /// supervisor gave up on it.
4565        candidate_exit: Option<ExitReport>,
4566    },
4567}
4568
4569/// Why a swap was refused before a candidate was spawned.
4570#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4571pub enum SwapRefusal {
4572    /// The module's config does not declare `overlap: "safe"`.
4573    OverlapExclusive,
4574    /// The module is not registered, so there is no incumbent to keep serving
4575    /// and nothing a swap would improve on; a plain restart is the tool.
4576    NotRegistered,
4577    /// The module does not speak the subc wire, so a candidate could never
4578    /// register or declare itself ready.
4579    ProtocolNone,
4580    /// The supervisor lacks the forwarding table (to cut routes over) or the
4581    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4582    NotConfigured,
4583    /// A swap is already open for this module.
4584    AlreadySwapping,
4585}
4586
4587impl SwapRefusal {
4588    pub fn as_str(self) -> &'static str {
4589        match self {
4590            Self::OverlapExclusive => "overlap_exclusive",
4591            Self::NotRegistered => "not_registered",
4592            Self::ProtocolNone => "protocol_none",
4593            Self::NotConfigured => "not_configured",
4594            Self::AlreadySwapping => "already_swapping",
4595        }
4596    }
4597}
4598
4599/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4600/// serving and undrained; see `CutoverLost`.
4601#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4602pub enum SwapFailureArm {
4603    /// The candidate process could not be started.
4604    SpawnFailed,
4605    /// The candidate did not register within the readiness budget.
4606    NeverRegistered,
4607    /// The candidate registered but did not declare itself ready in time.
4608    NeverReady,
4609    /// The candidate exited before cutover.
4610    CandidateExited,
4611    /// The candidate declared itself ready but failed its health probe.
4612    CandidateUnhealthy,
4613    /// An operator stop, disable or retire arrived while the candidate warmed.
4614    /// The candidate was killed and the operator's command then carried out on
4615    /// the incumbent.
4616    Interrupted,
4617    /// The candidate's connection closed at the moment of cutover. If it
4618    /// closed before forwarding moved, the incumbent is untouched. If it closed
4619    /// between the forwarding and registry halves of cutover, forwarding can no
4620    /// longer route to the incumbent, so the module is restarted plainly.
4621    CutoverLost,
4622}
4623
4624impl SwapFailureArm {
4625    pub fn as_str(self) -> &'static str {
4626        match self {
4627            Self::SpawnFailed => "spawn_failed",
4628            Self::NeverRegistered => "never_registered",
4629            Self::NeverReady => "never_ready",
4630            Self::CandidateExited => "candidate_exited",
4631            Self::CandidateUnhealthy => "candidate_unhealthy",
4632            Self::Interrupted => "interrupted",
4633            Self::CutoverLost => "cutover_lost",
4634        }
4635    }
4636}
4637
4638impl fmt::Display for SuperviseError {
4639    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4640        match self {
4641            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4642            Self::Spawn {
4643                program,
4644                source,
4645                cgroup_path: Some(cgroup_path),
4646            } => write!(
4647                f,
4648                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4649                cgroup_path.display(),
4650                program.display()
4651            ),
4652            Self::Spawn {
4653                program,
4654                source,
4655                cgroup_path: None,
4656            } => write!(
4657                f,
4658                "failed to spawn module '{}': {source}",
4659                program.display()
4660            ),
4661            Self::Cgroup { module_id, source } => {
4662                write!(
4663                    f,
4664                    "failed to prepare cgroup for module '{module_id}': {source}"
4665                )
4666            }
4667            Self::LaunchNonce { reason } => {
4668                write!(
4669                    f,
4670                    "failed to generate reserved-module launch nonce: {reason}"
4671                )
4672            }
4673            Self::Wait { module_id, source } => {
4674                write!(f, "failed to wait for module '{module_id}': {source}")
4675            }
4676            Self::Kill { module_id, source } => {
4677                write!(f, "failed to kill module '{module_id}': {source}")
4678            }
4679            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4680            Self::Registry(err) => write!(f, "registry error: {err}"),
4681            Self::ReloadUnavailable { module_id, reason } => {
4682                write!(f, "reload unavailable for module '{module_id}': {reason}")
4683            }
4684            Self::Disabled { module_id } => {
4685                write!(
4686                    f,
4687                    "module '{module_id}' is disabled; enable it before restart or reload"
4688                )
4689            }
4690            Self::ReloadFailed { module_id, reason } => {
4691                write!(f, "reload failed for module '{module_id}': {reason}")
4692            }
4693            Self::RegistrationStillActive { module_id, waited } => write!(
4694                f,
4695                "module '{module_id}' registration remained active after waiting {waited:?}"
4696            ),
4697            Self::StatePoisoned { module_id } => match module_id {
4698                Some(module_id) => {
4699                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4700                }
4701                None => write!(f, "supervisor state was poisoned"),
4702            },
4703            Self::CommandClosed { module_id } => {
4704                write!(
4705                    f,
4706                    "supervisor command channel for module '{module_id}' is closed"
4707                )
4708            }
4709            Self::SwapInProgress { module_id } => write!(
4710                f,
4711                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4712            ),
4713            Self::SwapRefused { module_id, reason } => match reason {
4714                SwapRefusal::OverlapExclusive => write!(
4715                    f,
4716                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4717                ),
4718                SwapRefusal::NotRegistered => write!(
4719                    f,
4720                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4721                ),
4722                SwapRefusal::ProtocolNone => write!(
4723                    f,
4724                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4725                ),
4726                SwapRefusal::NotConfigured => write!(
4727                    f,
4728                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4729                ),
4730                SwapRefusal::AlreadySwapping => {
4731                    write!(f, "module '{module_id}' is already being swapped")
4732                }
4733            },
4734            Self::SwapFailed {
4735                module_id,
4736                arm,
4737                detail,
4738                ..
4739            } => write!(
4740                f,
4741                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4742                arm.as_str()
4743            ),
4744        }
4745    }
4746}
4747
4748impl Error for SuperviseError {
4749    fn source(&self) -> Option<&(dyn Error + 'static)> {
4750        match self {
4751            Self::Spawn { source, .. }
4752            | Self::Cgroup { source, .. }
4753            | Self::Wait { source, .. }
4754            | Self::Kill { source, .. } => Some(source),
4755            Self::Forwarding(err) => Some(err),
4756            Self::Registry(err) => Some(err),
4757            Self::LaunchNonce { .. }
4758            | Self::InvalidSpec { .. }
4759            | Self::ReloadUnavailable { .. }
4760            | Self::Disabled { .. }
4761            | Self::ReloadFailed { .. }
4762            | Self::RegistrationStillActive { .. }
4763            | Self::StatePoisoned { .. }
4764            | Self::CommandClosed { .. }
4765            | Self::SwapInProgress { .. }
4766            | Self::SwapRefused { .. }
4767            | Self::SwapFailed { .. } => None,
4768        }
4769    }
4770}
4771
4772pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4773    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4774        return Err(SuperviseError::InvalidSpec {
4775            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4776        });
4777    }
4778    if spec.module_id.trim().is_empty() {
4779        return Err(SuperviseError::InvalidSpec {
4780            reason: "module_id must not be empty".to_string(),
4781        });
4782    }
4783
4784    Ok(())
4785}
4786
4787#[derive(Debug, Default)]
4788struct HealthProbeRuntime {
4789    spawn_generation: Option<u64>,
4790    configured_health: Option<HealthConfig>,
4791    registered_connection: Option<crate::ConnectionId>,
4792    advertised: bool,
4793    next_probe_at: Option<Instant>,
4794    probe_index: u64,
4795}
4796
4797fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4798    lock_snapshot(snapshot)
4799        .ok()
4800        .and_then(|state| state.spawned_protocol)
4801        .unwrap_or(spec.protocol)
4802}
4803
4804impl HealthProbeRuntime {
4805    fn refresh_registration(
4806        &mut self,
4807        spec: &ModuleSpec,
4808        runtime: &SupervisorRuntimeConfig,
4809        registry: &Registry,
4810        snapshot: &SharedSnapshot,
4811    ) {
4812        let generation = lock_snapshot(snapshot)
4813            .ok()
4814            .map(|state| state.spawn_generation);
4815        if self.spawn_generation != generation {
4816            // A replacement cannot reuse a deadline that is already due for its
4817            // predecessor. HTTP children get a full cadence to start; wire
4818            // children still wait for their own health-advertising HELLO.
4819            *self = Self {
4820                spawn_generation: generation,
4821                ..Self::default()
4822            };
4823        }
4824        if self.configured_health.as_ref() != Some(&runtime.health) {
4825            self.configured_health = Some(runtime.health.clone());
4826            self.next_probe_at = None;
4827            self.registered_connection = None;
4828            self.probe_index = 0;
4829        }
4830        // A non-wire process never registers. Only an explicitly configured
4831        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4832        // health failure for that kind of process.
4833        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4834            self.registered_connection = None;
4835            self.advertised = runtime.health.http.is_some();
4836            if !self.advertised {
4837                self.next_probe_at = None;
4838                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4839                    let unknown = ModuleHealthStatus::default();
4840                    if state.health != unknown {
4841                        state.health = unknown;
4842                        state.last_probe_evidence = None;
4843                    }
4844                });
4845            } else if self.next_probe_at.is_none() {
4846                self.next_probe_at = Some(
4847                    Instant::now()
4848                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4849                );
4850            }
4851            return;
4852        }
4853
4854        let registration = match registry.get_module(&spec.module_id) {
4855            Ok(registration) => registration,
4856            Err(err) => {
4857                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4858                self.advertised = false;
4859                self.next_probe_at = None;
4860                return;
4861            }
4862        };
4863
4864        let Some(registration) = registration else {
4865            self.registered_connection = None;
4866            self.advertised = false;
4867            self.next_probe_at = None;
4868            return;
4869        };
4870
4871        let advertised = registration
4872            .control_ops
4873            .iter()
4874            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4875        if !advertised {
4876            self.registered_connection = Some(registration.connection_id);
4877            self.advertised = false;
4878            self.next_probe_at = None;
4879            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4880                if state.health.status != SupervisorHealthStatus::Unknown
4881                    || state.health.consecutive_failures != 0
4882                    || state.health.last_probe_ms.is_some()
4883                    || state.health.detail.is_some()
4884                    || state.health.metrics.is_some()
4885                {
4886                    state.health.status = SupervisorHealthStatus::Unknown;
4887                    state.last_probe_evidence = None;
4888                    state.health.consecutive_failures = 0;
4889                    state.health.last_probe_ms = None;
4890                    state.health.detail = None;
4891                    state.health.metrics = None;
4892                }
4893            });
4894            return;
4895        }
4896
4897        let reregistered = self.registered_connection != Some(registration.connection_id);
4898        self.registered_connection = Some(registration.connection_id);
4899        self.advertised = true;
4900        if reregistered || self.next_probe_at.is_none() {
4901            self.probe_index = 0;
4902            self.next_probe_at = Some(
4903                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4904            );
4905            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4906                state.health.status = SupervisorHealthStatus::Unknown;
4907                state.last_probe_evidence = None;
4908                state.health.consecutive_failures = 0;
4909                state.health.detail = None;
4910                state.health.metrics = None;
4911            });
4912        }
4913    }
4914
4915    fn wake_at(&self) -> Option<Instant> {
4916        if !self.advertised {
4917            return None;
4918        }
4919        self.next_probe_at
4920    }
4921
4922    fn due(&self) -> bool {
4923        self.advertised
4924            && self
4925                .next_probe_at
4926                .is_some_and(|next| Instant::now() >= next)
4927    }
4928
4929    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4930        self.probe_index = self.probe_index.wrapping_add(1);
4931        self.next_probe_at = Some(
4932            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4933        );
4934    }
4935}
4936
4937/// Evidence is separate from diagnostic prose so host starvation can hold only
4938/// silence-based restarts. A missing lane is still actionable during overload;
4939/// a timeout can instead mean the host has not scheduled a healthy module.
4940#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4941enum HealthProbeEvidence {
4942    /// The module's control lane is gone. Proof of death.
4943    LaneDead,
4944    /// No reply within the deadline. Proves nothing about the module's state.
4945    NoAnswer,
4946    /// The module replied, but not with a usable health report. Proves it is alive.
4947    BadAnswer,
4948    /// The daemon could not ask. Says nothing about the module.
4949    Misconfigured,
4950}
4951
4952#[derive(Debug)]
4953struct HealthProbeError {
4954    evidence: HealthProbeEvidence,
4955    message: String,
4956}
4957
4958impl HealthProbeError {
4959    fn lane_dead(message: impl Into<String>) -> Self {
4960        Self::with(HealthProbeEvidence::LaneDead, message)
4961    }
4962
4963    fn no_answer(message: impl Into<String>) -> Self {
4964        Self::with(HealthProbeEvidence::NoAnswer, message)
4965    }
4966
4967    fn bad_answer(message: impl Into<String>) -> Self {
4968        Self::with(HealthProbeEvidence::BadAnswer, message)
4969    }
4970
4971    fn misconfigured(message: impl Into<String>) -> Self {
4972        Self::with(HealthProbeEvidence::Misconfigured, message)
4973    }
4974
4975    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4976        Self {
4977            evidence,
4978            message: message.into(),
4979        }
4980    }
4981
4982    /// Whether this observation is proof the module cannot serve.
4983    ///
4984    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4985    /// variant that fires under CPU starvation, and treating it as proof is the
4986    /// defect this enum exists to make impossible to reintroduce silently.
4987    ///
4988    /// Outside a host-overload hold, silence still reaches the normal failure
4989    /// threshold so an isolated, genuinely wedged module can recover.
4990    #[allow(dead_code)]
4991    fn is_proof_of_death(&self) -> bool {
4992        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4993    }
4994
4995    /// Short stable label for logs and the health snapshot.
4996    ///
4997    /// An operator reading `ck health` currently cannot tell "the module is gone"
4998    /// from "the module did not answer in five seconds", because both render as
4999    /// prose in the same field. These labels are what make the two
5000    /// distinguishable at a glance, and they are what a later restart-policy
5001    /// change will be argued from.
5002    fn label(&self) -> &'static str {
5003        match self.evidence {
5004            HealthProbeEvidence::LaneDead => "lane-dead",
5005            HealthProbeEvidence::NoAnswer => "no-answer",
5006            HealthProbeEvidence::BadAnswer => "bad-answer",
5007            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5008        }
5009    }
5010}
5011
5012impl fmt::Display for HealthProbeError {
5013    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5014        f.write_str(&self.message)
5015    }
5016}
5017
5018async fn run_health_probe_cycle(
5019    spec: &ModuleSpec,
5020    runtime: &SupervisorRuntimeConfig,
5021    registry: &Registry,
5022    process_liveness: &SupervisorProcessLiveness,
5023    snapshot: &SharedSnapshot,
5024    child: &mut Option<SupervisedChild>,
5025) {
5026    let now_ms = unix_ms_now();
5027    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5028        .then_some(runtime.health.http.as_deref())
5029        .flatten();
5030    let result = match http {
5031        Some(url) => probe_http_health(url, runtime.health.deadline).await,
5032        None => probe_module_health(&spec.module_id, runtime, None).await,
5033    };
5034    match result {
5035        Ok(report) => {
5036            handle_health_report(
5037                spec,
5038                runtime,
5039                registry,
5040                process_liveness,
5041                snapshot,
5042                child,
5043                report,
5044                now_ms,
5045            )
5046            .await;
5047        }
5048        Err(err) => {
5049            if http.is_some() {
5050                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5051                    state.health.status = SupervisorHealthStatus::Failing;
5052                });
5053            }
5054            handle_health_probe_failure(
5055                spec,
5056                runtime,
5057                registry,
5058                process_liveness,
5059                snapshot,
5060                child,
5061                err,
5062                now_ms,
5063            )
5064            .await;
5065        }
5066    }
5067}
5068
5069pub(crate) struct HttpProbeTarget<'a> {
5070    address: std::net::SocketAddr,
5071    localhost: bool,
5072    authority: &'a str,
5073    path: String,
5074}
5075
5076/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
5077/// or TLS. A URL cannot turn a local health check into an outbound connection.
5078pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5079    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5080        return Err("must not contain whitespace, controls, or a fragment".into());
5081    }
5082    let rest = url
5083        .strip_prefix("http://")
5084        .ok_or("must use plain http://")?;
5085    let split = rest.find(['/', '?']).unwrap_or(rest.len());
5086    let (authority, suffix) = rest.split_at(split);
5087    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5088        ("::1", rest)
5089    } else {
5090        let split = authority.find(':').unwrap_or(authority.len());
5091        authority.split_at(split)
5092    };
5093    let ip: std::net::IpAddr = match host {
5094        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5095        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5096        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5097    };
5098    let port = if port.is_empty() {
5099        80
5100    } else {
5101        port.strip_prefix(':')
5102            .and_then(|p| p.parse::<u16>().ok())
5103            .filter(|p| *p > 0)
5104            .ok_or("must have a valid nonzero TCP port")?
5105    };
5106    let path = if suffix.is_empty() {
5107        "/".into()
5108    } else if suffix.starts_with('?') {
5109        format!("/{suffix}")
5110    } else {
5111        suffix.into()
5112    };
5113    Ok(HttpProbeTarget {
5114        address: std::net::SocketAddr::new(ip, port),
5115        localhost: host == "localhost",
5116        authority,
5117        path,
5118    })
5119}
5120
5121async fn probe_http_health(
5122    url: &str,
5123    deadline: Duration,
5124) -> Result<HealthReport, HealthProbeError> {
5125    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5126    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5127    // Keep partial diagnostics outside the timed future so cancellation does
5128    // not discard a status line or body bytes already received.
5129    let mut response_status = String::new();
5130    let mut body = Vec::new();
5131    let probe = async {
5132        // Resolve localhost ourselves so a hosts-file override cannot turn
5133        // this into an outbound request, while IPv6-only local servers work.
5134        let connection = match tokio::net::TcpStream::connect(target.address).await {
5135            Err(_) if target.localhost => {
5136                tokio::net::TcpStream::connect((
5137                    std::net::Ipv6Addr::LOCALHOST,
5138                    target.address.port(),
5139                ))
5140                .await
5141            }
5142            result => result,
5143        };
5144        let mut stream = connection.map_err(|error| {
5145            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5146        })?;
5147        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5148            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5149        let mut reader = BufReader::new(stream);
5150        let mut budget = 16 * 1024;
5151        let status = http_line(&mut reader, &mut budget).await?;
5152        let mut words = status.split_ascii_whitespace();
5153        let version = words.next();
5154        let code = words
5155            .next()
5156            .filter(|word| word.len() == 3)
5157            .and_then(|word| word.parse::<u16>().ok());
5158        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5159            || !code.is_some_and(|code| (100..600).contains(&code))
5160        {
5161            return Err(HealthProbeError::bad_answer(format!(
5162                "invalid HTTP status: {status}"
5163            )));
5164        }
5165        let code = code.expect("validated status code");
5166        response_status = status.clone();
5167        let mut length = None;
5168        let mut chunked = false;
5169        loop {
5170            let line = http_line(&mut reader, &mut budget).await?;
5171            if line.is_empty() {
5172                break;
5173            }
5174            if let Some((name, value)) = line.split_once(':') {
5175                if name.eq_ignore_ascii_case("content-length") {
5176                    length = Some(value.trim().parse::<u64>().map_err(|_| {
5177                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
5178                    })?);
5179                } else if name.eq_ignore_ascii_case("transfer-encoding") {
5180                    chunked = value.trim().eq_ignore_ascii_case("chunked");
5181                }
5182            }
5183        }
5184        if chunked {
5185            while body.len() < 200 {
5186                let line = http_line(&mut reader, &mut budget).await?;
5187                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5188                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5189                if size == 0 {
5190                    break;
5191                }
5192                let count = size.min((200 - body.len()) as u64) as usize;
5193                let start = body.len();
5194                (&mut reader)
5195                    .take(count as u64)
5196                    .read_to_end(&mut body)
5197                    .await
5198                    .map_err(|error| {
5199                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5200                    })?;
5201                if body.len() - start != count {
5202                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5203                }
5204                if size > count as u64 || body.len() == 200 {
5205                    break;
5206                }
5207                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5208                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5209                }
5210            }
5211        } else {
5212            reader
5213                .take(length.unwrap_or(200).min(200))
5214                .read_to_end(&mut body)
5215                .await
5216                .map_err(|error| {
5217                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5218                })?;
5219        }
5220        if (200..300).contains(&code) {
5221            Ok(HealthReport::ok())
5222        } else {
5223            Err(HealthProbeError::bad_answer(
5224                "HTTP health endpoint returned non-2xx",
5225            ))
5226        }
5227    };
5228    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5229        Err(HealthProbeError::no_answer(format!(
5230            "HTTP probe timed out after {deadline:?}"
5231        )))
5232    });
5233    if let Err(error) = &mut result {
5234        if !response_status.is_empty() {
5235            error.message = format!(
5236                "{}; {response_status}: {}",
5237                error.message,
5238                String::from_utf8_lossy(&body)
5239            );
5240        }
5241    }
5242    result
5243}
5244
5245async fn http_line(
5246    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5247    remaining: &mut usize,
5248) -> Result<String, HealthProbeError> {
5249    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5250    let mut line = Vec::new();
5251    (&mut *reader)
5252        .take(*remaining as u64)
5253        .read_until(b'\n', &mut line)
5254        .await
5255        .map_err(|error| {
5256            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5257        })?;
5258    *remaining -= line.len();
5259    if !line.ends_with(b"\r\n") {
5260        return Err(HealthProbeError::bad_answer(
5261            "HTTP headers are incomplete or exceed 16 KiB",
5262        ));
5263    }
5264    line.truncate(line.len() - 2);
5265    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5266}
5267
5268async fn probe_module_health(
5269    module_id: &str,
5270    runtime: &SupervisorRuntimeConfig,
5271    drain_deadline: Option<Instant>,
5272) -> Result<HealthReport, HealthProbeError> {
5273    let Some(forwarding) = runtime.forwarding.as_ref() else {
5274        return Err(HealthProbeError::misconfigured(
5275            "supervisor was not configured with a forwarding table",
5276        ));
5277    };
5278    let probe_started_at = Instant::now();
5279    let mut deadline = probe_started_at + runtime.health.deadline;
5280    if let Some(drain_deadline) = drain_deadline {
5281        deadline = deadline.min(drain_deadline);
5282    }
5283    let pending = if drain_deadline.is_some() {
5284        forwarding.begin_drain_health_probe_rpc_for(
5285            module_id,
5286            MODULE_CONTROL_OP_HEALTH_CHECK,
5287            probe_started_at,
5288            deadline,
5289        )
5290    } else {
5291        forwarding.begin_health_probe_rpc_for(
5292            module_id,
5293            MODULE_CONTROL_OP_HEALTH_CHECK,
5294            probe_started_at,
5295            deadline,
5296        )
5297    }
5298    .map_err(|err| {
5299        // The endpoint is not registered, so there is no live control lane to
5300        // ask. That is the module being absent, not slow.
5301        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5302    })?;
5303    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5304}
5305
5306/// [`probe_module_health`] for one endpoint rather than the id's active one.
5307///
5308/// A swap probes two processes that no by-id lookup reaches: its candidate
5309/// before cutover, and its superseded incumbent (for busy gauges) while the
5310/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5311/// bounds the by-id drain probe.
5312async fn probe_endpoint_health(
5313    endpoint: crate::ModuleEndpointId,
5314    runtime: &SupervisorRuntimeConfig,
5315    deadline_cap: Option<Instant>,
5316) -> Result<HealthReport, HealthProbeError> {
5317    let Some(forwarding) = runtime.forwarding.as_ref() else {
5318        return Err(HealthProbeError::misconfigured(
5319            "supervisor was not configured with a forwarding table",
5320        ));
5321    };
5322    let probe_started_at = Instant::now();
5323    let mut deadline = probe_started_at + runtime.health.deadline;
5324    if let Some(cap) = deadline_cap {
5325        deadline = deadline.min(cap);
5326    }
5327    let pending = forwarding
5328        .begin_endpoint_health_probe_rpc_for(
5329            endpoint,
5330            MODULE_CONTROL_OP_HEALTH_CHECK,
5331            probe_started_at,
5332            deadline,
5333        )
5334        .map_err(|err| {
5335            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5336        })?;
5337    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5338}
5339
5340/// Send a begun health probe and classify its answer.
5341async fn await_health_probe(
5342    forwarding: &ForwardingTable,
5343    pending: PendingModuleControlRpc,
5344    deadline: Instant,
5345    probe_budget: Duration,
5346) -> Result<HealthReport, HealthProbeError> {
5347    let PendingModuleControlRpc {
5348        endpoint,
5349        module_sink,
5350        negotiated_ver,
5351        corr,
5352        receiver,
5353    } = pending;
5354    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5355        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5356    })?;
5357    let frame = Frame::build_with_version(
5358        negotiated_ver,
5359        FrameType::Request,
5360        control_flags(),
5361        0,
5362        0,
5363        corr,
5364        body,
5365    )
5366    .map_err(|err| {
5367        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5368    })?;
5369
5370    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5371    // blocks waiting for capacity when the module's egress queue is full, and an
5372    // unbounded await here freezes the whole supervision actor (it stops polling
5373    // Child::wait and supervisor commands), making the module unrecoverable
5374    // in-band. On timeout the probe fails like any transport failure.
5375    match timeout_at(deadline, module_sink.send(frame)).await {
5376        Ok(Ok(())) => {}
5377        Ok(Err(err)) => {
5378            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5379            // A closed sink means the module's egress channel is gone -- the
5380            // receiving half is dropped when its connection tears down. Proof.
5381            return Err(HealthProbeError::lane_dead(format!(
5382                "failed to send health.check: {err}"
5383            )));
5384        }
5385        Err(_elapsed) => {
5386            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5387            // A full egress queue means the module is not draining its socket, which
5388            // is consistent with a wedged module AND with one whose reader is merely
5389            // starved. Silence, not proof.
5390            return Err(HealthProbeError::no_answer(
5391                "health.check send timed out before enqueue (module egress full)",
5392            ));
5393        }
5394    }
5395
5396    match timeout_at(deadline, receiver).await {
5397        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5398        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5399        // and those prove it is alive even though the probe failed.
5400        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5401            response.health_report().ok_or_else(|| {
5402                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5403            })
5404        }
5405        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5406            format!("health.check rejected: {}", body.message),
5407        )),
5408        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5409            Err(HealthProbeError::lane_dead(message))
5410        }
5411        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5412            Err(HealthProbeError::bad_answer(message))
5413        }
5414        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5415            Err(HealthProbeError::bad_answer(format!(
5416                "expected module-control op '{expected}', got '{actual}'"
5417            )))
5418        }
5419        // A reply that crosses the deadline before this waiter observes it is
5420        // still proof of life. The forwarding path records its end-to-end latency
5421        // before delivering this classification.
5422        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5423            "module answered health.check after its daemon deadline",
5424        )),
5425        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5426            "health.check waiter was canceled before the module responded",
5427        )),
5428        Err(_) => {
5429            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5430            Err(HealthProbeError::no_answer(format!(
5431                "module did not answer health.check within {probe_budget:?}"
5432            )))
5433        }
5434    }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn handle_health_report(
5439    spec: &ModuleSpec,
5440    runtime: &SupervisorRuntimeConfig,
5441    registry: &Registry,
5442    process_liveness: &SupervisorProcessLiveness,
5443    snapshot: &SharedSnapshot,
5444    child: &mut Option<SupervisedChild>,
5445    report: HealthReport,
5446    now_ms: u64,
5447) {
5448    let status = supervisor_health_status(report.status);
5449    let detail = report.detail.clone();
5450    let metrics = truncate_health_metrics(report.metrics);
5451    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5452        state.health.status = status;
5453        state.last_probe_evidence = None;
5454        state.health.last_probe_ms = Some(now_ms);
5455        state.health.detail = detail.clone();
5456        state.health.metrics = metrics.clone();
5457        state.health.consecutive_failures = 0;
5458        state.failed_retry.observe_health(
5459            report.status == HealthStatus::Ok,
5460            runtime.restart_policy.window,
5461        );
5462    });
5463
5464    let action = match report.status {
5465        HealthStatus::Ok => return,
5466        HealthStatus::Degraded => runtime.health.on_degraded,
5467        HealthStatus::Failing => runtime.health.on_failing,
5468    };
5469    apply_l3_health_action(
5470        spec,
5471        runtime,
5472        registry,
5473        process_liveness,
5474        snapshot,
5475        child,
5476        status,
5477        detail.as_deref(),
5478        action,
5479        now_ms,
5480    )
5481    .await;
5482}
5483
5484#[allow(clippy::too_many_arguments)]
5485async fn handle_health_probe_failure(
5486    spec: &ModuleSpec,
5487    runtime: &SupervisorRuntimeConfig,
5488    registry: &Registry,
5489    process_liveness: &SupervisorProcessLiveness,
5490    snapshot: &SharedSnapshot,
5491    child: &mut Option<SupervisedChild>,
5492    err: HealthProbeError,
5493    now_ms: u64,
5494) {
5495    let threshold = runtime.health.failure_threshold.max(1);
5496    let held = if err.evidence == HealthProbeEvidence::NoAnswer {
5497        runtime.host_health.record_late_module(&spec.module_id)
5498    } else {
5499        false
5500    };
5501    let mut failures = 0;
5502    // Carry the evidence class into the operator-visible detail. Without it,
5503    // "module did not answer within 5s" and "the control lane is gone" are two
5504    // prose strings in the same field, and the reader has to know the codebase to
5505    // tell which one is proof of anything.
5506    let detail = if held {
5507        format!("[{}] {err}; host overloaded", err.label())
5508    } else {
5509        format!("[{}] {err}", err.label())
5510    };
5511    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5512        state.last_probe_evidence = Some(err.evidence);
5513        state
5514            .failed_retry
5515            .observe_health(false, runtime.restart_policy.window);
5516        // A failed wire probe invalidates the last report, even before the
5517        // restart threshold. HTTP probes already mark failures as Failing.
5518        if held {
5519            state.health.status = SupervisorHealthStatus::Degraded;
5520        } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5521            state.health.status = SupervisorHealthStatus::Unknown;
5522        }
5523        state.health.last_probe_ms = Some(now_ms);
5524        // Held silence breaks the streak: after the host recovers, only new
5525        // failures count. Replaying pre-overload misses would restart immediately.
5526        state.health.consecutive_failures = if held {
5527            0
5528        } else {
5529            state.health.consecutive_failures.saturating_add(1)
5530        };
5531        state.health.detail = Some(detail.clone());
5532        state.health.metrics = None;
5533        failures = state.health.consecutive_failures;
5534    });
5535
5536    if held || failures < threshold {
5537        warn!(
5538            module_id = %spec.module_id,
5539            consecutive_failures = failures,
5540            threshold,
5541            evidence = err.label(),
5542            detail = %detail,
5543            "health.check probe failed"
5544        );
5545        return;
5546    }
5547
5548    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5549        state.state = ModuleState::Unresponsive;
5550        state.health.status = SupervisorHealthStatus::Unresponsive;
5551    });
5552    // The evidence class is logged at the kill site because this is the line an
5553    // operator reads after an unexplained restart. A streak of `no-answer` under
5554    // machine load is the known false-positive shape; a `lane-dead` is not.
5555    if runtime.health.critical {
5556        error!(
5557            module_id = %spec.module_id,
5558            status = "unresponsive",
5559            evidence = err.label(),
5560            detail = %detail,
5561            "critical module health alert"
5562        );
5563    } else {
5564        warn!(
5565            module_id = %spec.module_id,
5566            status = "unresponsive",
5567            evidence = err.label(),
5568            detail = %detail,
5569            "module health threshold breached"
5570        );
5571    }
5572    if let Err(err) = health_restart_child(
5573        spec,
5574        runtime,
5575        registry,
5576        process_liveness,
5577        snapshot,
5578        child,
5579        SupervisorHealthStatus::Unresponsive,
5580        Some(&detail),
5581        now_ms,
5582    )
5583    .await
5584    {
5585        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5586    }
5587}
5588
5589#[allow(clippy::too_many_arguments)]
5590async fn apply_l3_health_action(
5591    spec: &ModuleSpec,
5592    runtime: &SupervisorRuntimeConfig,
5593    registry: &Registry,
5594    process_liveness: &SupervisorProcessLiveness,
5595    snapshot: &SharedSnapshot,
5596    child: &mut Option<SupervisedChild>,
5597    status: SupervisorHealthStatus,
5598    detail: Option<&str>,
5599    action: HealthAction,
5600    now_ms: u64,
5601) {
5602    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5603    match action {
5604        HealthAction::Report => {
5605            info!(
5606                module_id = %spec.module_id,
5607                status = ?status,
5608                detail,
5609                "module reported non-ok health"
5610            );
5611        }
5612        HealthAction::Alert => {
5613            error!(
5614                module_id = %spec.module_id,
5615                status = ?status,
5616                detail,
5617                "module health alert"
5618            );
5619        }
5620        HealthAction::Restart => {
5621            if let Err(err) = health_restart_child(
5622                spec,
5623                runtime,
5624                registry,
5625                process_liveness,
5626                snapshot,
5627                child,
5628                status,
5629                detail,
5630                now_ms,
5631            )
5632            .await
5633            {
5634                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5635            }
5636        }
5637    }
5638}
5639
5640#[allow(clippy::too_many_arguments)]
5641async fn health_restart_child(
5642    spec: &ModuleSpec,
5643    runtime: &SupervisorRuntimeConfig,
5644    registry: &Registry,
5645    process_liveness: &SupervisorProcessLiveness,
5646    snapshot: &SharedSnapshot,
5647    child: &mut Option<SupervisedChild>,
5648    status: SupervisorHealthStatus,
5649    detail: Option<&str>,
5650    now_ms: u64,
5651) -> Result<(), SuperviseError> {
5652    let (enabled, schedule) = {
5653        let mut state = lock_snapshot(snapshot)?;
5654        let enabled = state.enabled;
5655        let schedule = if enabled {
5656            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5657        } else {
5658            None
5659        };
5660        (enabled, schedule)
5661    };
5662
5663    if !enabled {
5664        return Err(SuperviseError::Disabled {
5665            module_id: spec.module_id.clone(),
5666        });
5667    }
5668
5669    if schedule.is_none() {
5670        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5671        error!(
5672            module_id = %spec.module_id,
5673            status = ?status,
5674            detail,
5675            max_restarts = runtime.restart_policy.max_restarts,
5676            window_secs = runtime.restart_policy.window.as_secs(),
5677            reason = %runtime.restart_policy.budget_exhausted_detail(),
5678            "health restart budget exhausted; marking module failed"
5679        );
5680        let stop_notice = begin_forwarding_drain_if_configured(
5681            spec,
5682            runtime,
5683            registry,
5684            snapshot,
5685            Some(true),
5686            RouteCloseReason::Disable,
5687        )
5688        .await?;
5689        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5690            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5691        })?;
5692        let result = drain_optional_child(
5693            &spec.module_id,
5694            spec.protocol,
5695            stop_notice,
5696            registry,
5697            runtime.forwarding.as_deref(),
5698            snapshot,
5699            &runtime.terminal_ring,
5700            &runtime.spawn_events,
5701            child,
5702            runtime.drain_timeout,
5703            ModuleState::Failed,
5704            Some(true),
5705            &runtime.failure_context(true),
5706        )
5707        .await;
5708        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5709            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5710                state.failed_retry.schedule(&spec.module_id);
5711            }
5712        })?;
5713        result?;
5714        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5715        return Ok(());
5716    }
5717
5718    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5719    let mut restart_count = 0;
5720    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5721        restart_count = state.crash_restarts.len();
5722        state.state = ModuleState::Unresponsive;
5723        state.health.status = status;
5724        state.health.last_action = Some(HealthAction::Restart.to_string());
5725        state.health.last_action_ms = Some(now_ms);
5726    })?;
5727    warn!(
5728        module_id = %spec.module_id,
5729        status = ?status,
5730        detail,
5731        restart_count,
5732        restart_in_window = schedule.restart_in_window,
5733        delay_ms = schedule.delay.as_millis() as u64,
5734        "health-triggered module restart"
5735    );
5736
5737    let stop_notice = begin_forwarding_drain_if_configured(
5738        spec,
5739        runtime,
5740        registry,
5741        snapshot,
5742        Some(true),
5743        RouteCloseReason::Restart,
5744    )
5745    .await?;
5746    drain_optional_child(
5747        &spec.module_id,
5748        spec.protocol,
5749        stop_notice,
5750        registry,
5751        runtime.forwarding.as_deref(),
5752        snapshot,
5753        &runtime.terminal_ring,
5754        &runtime.spawn_events,
5755        child,
5756        runtime.drain_timeout,
5757        ModuleState::Restarting,
5758        Some(true),
5759        &runtime.failure_context(true),
5760    )
5761    .await?;
5762    schedule_respawn(
5763        runtime,
5764        snapshot,
5765        &spec.module_id,
5766        schedule.delay,
5767        RespawnKind::Spawn,
5768        true,
5769    )
5770}
5771
5772fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5773    if let Some(reply) = runtime
5774        .deferred_reload_reply
5775        .lock()
5776        .unwrap_or_else(|p| p.into_inner())
5777        .take()
5778    {
5779        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5780            module_id: module_id.to_string(),
5781            reason: reason.to_string(),
5782        }));
5783    }
5784}
5785
5786fn schedule_respawn(
5787    runtime: &SupervisorRuntimeConfig,
5788    snapshot: &SharedSnapshot,
5789    module_id: &str,
5790    delay: Duration,
5791    kind: RespawnKind,
5792    daemon_initiated: bool,
5793) -> Result<(), SuperviseError> {
5794    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5795    update_snapshot(snapshot, Some(module_id), |state| {
5796        state.respawn_pending = true
5797    })?;
5798    *runtime
5799        .scheduled_respawn
5800        .lock()
5801        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5802        deadline: Instant::now() + delay,
5803        kind,
5804        daemon_initiated,
5805    });
5806    Ok(())
5807}
5808
5809fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5810    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5811        state.health.last_action = Some(action);
5812        state.health.last_action_ms = Some(now_ms);
5813    });
5814}
5815
5816fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5817    match status {
5818        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5819        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5820        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5821    }
5822}
5823
5824/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5825/// returned to every `supervisor.list` and `supervisor.health` caller.
5826///
5827/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5828/// path: that request exists to return a module's complete metrics object, and
5829/// `ck health <module-id>` documents it as the way to see what the cached view
5830/// truncates. The asymmetry is the feature.
5831///
5832/// So a new caller must decide which side it is on rather than assume the cap is
5833/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5834/// the truncation that path exists to avoid.
5835fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5836    let metrics = metrics?;
5837    match serde_json::to_vec(&metrics) {
5838        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5839            "truncated": true,
5840            "original_bytes": encoded.len(),
5841        })),
5842        Ok(_) | Err(_) => Some(metrics),
5843    }
5844}
5845
5846/// Spread health probes so a fleet-wide restart does not converge them.
5847///
5848/// The delay is derived from the module id and probe index rather than a random
5849/// source, so it is deterministic per module: a module keeps its own offset
5850/// across daemon restarts instead of re-rolling into a collision.
5851fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5852    if cadence.is_zero() {
5853        return Duration::ZERO;
5854    }
5855    let cadence_ms = cadence.as_millis() as u64;
5856    // This early return is REDUNDANT, deliberately, and a mutation run will show
5857    // it surviving removal. Recording why here so the next person to notice does
5858    // not have to re-derive it:
5859    //
5860    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5861    //   a zero cadence and builds the Duration from whole milliseconds, so a
5862    //   sub-millisecond cadence cannot come from config.
5863    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5864    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5865    //   -- exactly what this returns.
5866    //
5867    // Kept as a guard against a future widening of the config parser (accepting
5868    // microseconds, say), which would make the sub-millisecond case reachable.
5869    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5870    // divides by zero. Remove this and nothing changes.
5871    if cadence_ms == 0 {
5872        return cadence;
5873    }
5874    // Note that this never returns less than one cadence, including for the FIRST
5875    // probe. So a freshly registered module reports health `unknown` for a full
5876    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5877    // ready to answer.
5878    //
5879    // That is a property of the supervisor's schedule, not of any module: an
5880    // operator watching a restart sees `unknown` and cannot tell it from a module
5881    // that is slow to warm. Measured on two unrelated modules, both flipping to
5882    // `ok` between 22s and 32s after restart.
5883    //
5884    // Left as-is because spreading the first probe is what keeps a fleet-wide
5885    // restart from firing fourteen simultaneous probes into a cold machine. The
5886    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5887    // that thundering herd for a faster first reading.
5888    let jitter_span = (cadence_ms / 10).max(1);
5889    let hash = module_id.as_bytes().iter().fold(
5890        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5891        |acc, byte| {
5892            acc.wrapping_mul(1099511628211)
5893                .wrapping_add(u64::from(*byte))
5894        },
5895    );
5896    cadence + Duration::from_millis(hash % jitter_span)
5897}
5898
5899#[cfg(test)]
5900mod tests {
5901    use super::*;
5902
5903    #[test]
5904    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5905        let handle = SupervisorHandle::new();
5906        let module_id = "readded-tombstone";
5907        handle.record_rescan_removal(module_id);
5908        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5909
5910        handle.apply_identity_configuration(&ModuleSpec {
5911            module_id: module_id.to_string(),
5912            program: PathBuf::from("/test/module"),
5913            args: Vec::new(),
5914            env: Vec::new(),
5915            reserved: false,
5916            reserved_prefixes: Vec::new(),
5917            protocol: ModuleProtocol::Subc,
5918            overlap: Default::default(),
5919            carry_containment: false,
5920        });
5921
5922        assert!(
5923            handle.removal_tombstone_age_ms(module_id).is_none(),
5924            "a re-added module must not retain a stale removal tombstone"
5925        );
5926    }
5927
5928    /// What one module's owner looked like from the control plane at the
5929    /// instant after its first process was spawned.
5930    #[derive(Debug, PartialEq, Eq)]
5931    struct OwnerInSpawnWindow {
5932        module_id: String,
5933        configured: bool,
5934        on_roster: bool,
5935        admission_refusal: Option<&'static str>,
5936    }
5937
5938    /// A supervised module's process can connect, register, sync its scopes
5939    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5940    /// the module on the roster until afterward; configured supervision does
5941    /// so before spawning. In both cases the owner must
5942    /// already read as configured, so a scoped `route.open` against it is
5943    /// refused as retryable `scope_not_synced` and not as terminal
5944    /// `scope_not_live` ("will never sync").
5945    ///
5946    /// The hook runs just after each initial spawn attempt, before returning to
5947    /// the caller, so no race with a real child's registration is needed.
5948    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5949    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5950        use crate::scopes::ScopeTable;
5951        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5952
5953        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5954        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5955            module_id: module_id.to_string(),
5956            program,
5957            args: Vec::new(),
5958            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5959                .into_iter()
5960                .map(|key| (key.to_string(), dir.path().display().to_string()))
5961                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5962                .collect(),
5963            reserved: false,
5964            reserved_prefixes: Vec::new(),
5965            protocol: ModuleProtocol::Subc,
5966            overlap: Default::default(),
5967            carry_containment: false,
5968        };
5969        let live = super::terminal_history_tests::fake_aft_stub_path();
5970        let missing = dir.path().join("definitely-missing-module");
5971
5972        let handle = SupervisorHandle::new();
5973        let mut supervisor =
5974            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5975                .with_handle(handle.clone());
5976        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5977        let hook_handle = handle.clone();
5978        let hook_observed = Arc::clone(&observed);
5979        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5980            // Exactly what the control plane computes for a scoped route.open
5981            // naming this module as the owner of a scope it has not synced.
5982            let configured = hook_handle.is_configured(module_id);
5983            let selector = ScopeSelector::new(
5984                Principal::Reserved {
5985                    module_id: module_id.to_string(),
5986                },
5987                "s".to_string(),
5988                Some(1),
5989            );
5990            let carrier = Principal::Reserved {
5991                module_id: "carrier".to_string(),
5992            };
5993            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5994                .admit(&carrier, module_id, &selector, configured)
5995            {
5996                Ok(_) => None,
5997                Err(refusal) => Some(refusal.code),
5998            };
5999            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
6000                module_id: module_id.to_string(),
6001                configured,
6002                on_roster: hook_handle.get(module_id).is_some(),
6003                admission_refusal,
6004            });
6005        })));
6006
6007        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
6008        let configured = supervisor
6009            .supervise_configured(stub("configured", live.clone()), true)
6010            .unwrap();
6011        let with_health = supervisor
6012            .supervise_configured_with_health(
6013                stub("with-health", live.clone()),
6014                true,
6015                HealthConfig::default(),
6016                None,
6017                RestartPolicy::default(),
6018            )
6019            .unwrap();
6020        // The failed-spawn path still puts the module on the roster (as
6021        // failed), so it is configured throughout.
6022        let failed = supervisor
6023            .supervise_configured_with_health(
6024                stub("failed-spawn", missing.clone()),
6025                true,
6026                HealthConfig::default(),
6027                None,
6028                RestartPolicy::default(),
6029            )
6030            .unwrap();
6031        // A failed plain `spawn` puts nothing on the roster, so its mark is
6032        // taken back once the spawn has failed.
6033        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6034
6035        let expected = [
6036            "plain",
6037            "configured",
6038            "with-health",
6039            "failed-spawn",
6040            "spawn-error",
6041        ]
6042        .into_iter()
6043        .map(|module_id| OwnerInSpawnWindow {
6044            module_id: module_id.to_string(),
6045            configured: true,
6046            on_roster: !matches!(module_id, "plain" | "spawn-error"),
6047            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6048        })
6049        .collect::<Vec<_>>();
6050        assert_eq!(*observed.lock().unwrap(), expected);
6051
6052        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6053            assert!(
6054                handle.get(module_id).is_some(),
6055                "{module_id} is on the roster"
6056            );
6057            assert!(
6058                handle.is_configured(module_id),
6059                "{module_id} stays configured"
6060            );
6061        }
6062        assert!(handle.get("spawn-error").is_none());
6063        assert!(
6064            !handle.is_configured("spawn-error"),
6065            "a plain spawn that failed must not leave its module marked configured"
6066        );
6067
6068        // Leaving the roster clears the mark with it.
6069        handle.retire("failed-spawn");
6070        assert!(!handle.is_configured("failed-spawn"));
6071
6072        for module in [plain, configured, with_health] {
6073            module.stop().await.unwrap();
6074        }
6075        drop(failed);
6076    }
6077
6078    /// A configured module reads as starting until the supervisor has admitted
6079    /// its first process (exec confirmed, identity durably recorded), even
6080    /// though that process is already running by then and can register over
6081    /// HELLO, which is what lists it in catalog.list. So a client that needs
6082    /// the supervisor state must wait for that state; seeing the module in
6083    /// the catalog does not mean supervisor.list already says running.
6084    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6085    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6086        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6087        let spec = ModuleSpec {
6088            module_id: "spawned-not-admitted".to_string(),
6089            program: super::terminal_history_tests::fake_aft_stub_path(),
6090            args: Vec::new(),
6091            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6092                .into_iter()
6093                .map(|key| (key.to_string(), dir.path().display().to_string()))
6094                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6095                .collect(),
6096            reserved: false,
6097            reserved_prefixes: Vec::new(),
6098            protocol: ModuleProtocol::Subc,
6099            overlap: Default::default(),
6100            carry_containment: false,
6101        };
6102        let handle = SupervisorHandle::new();
6103        let mut supervisor =
6104            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6105                .with_handle(handle.clone());
6106        let observed = Arc::new(Mutex::new(None));
6107        let hook_handle = handle.clone();
6108        let hook_observed = Arc::clone(&observed);
6109        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6110            let state = hook_handle
6111                .get(module_id)
6112                .expect("a configured module is on the roster before it is spawned")
6113                .state()
6114                .unwrap();
6115            *hook_observed.lock().unwrap() = Some(state);
6116        })));
6117
6118        let module = supervisor.supervise_configured(spec, true).unwrap();
6119        assert_eq!(
6120            *observed.lock().unwrap(),
6121            Some(ModuleState::Starting),
6122            "between the spawn and its admission the supervisor still reports starting"
6123        );
6124        assert_eq!(module.state().unwrap(), ModuleState::Running);
6125        module.stop().await.unwrap();
6126    }
6127
6128    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6129        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6130        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6131            snapshot.process_alive = true;
6132            snapshot.pid = Some(41);
6133            snapshot.spawned_at_ms = Some(42);
6134            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6135            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6136            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6137                device: 43,
6138                inode: 44,
6139                #[cfg(windows)]
6140                windows_image: None,
6141            });
6142        })
6143        .unwrap();
6144        snapshot
6145    }
6146
6147    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6148        let snapshot = lock_snapshot(snapshot).unwrap();
6149        assert!(!snapshot.process_alive);
6150        assert_eq!(snapshot.pid, None);
6151        assert_eq!(snapshot.spawned_at_ms, None);
6152        assert_eq!(snapshot.spawned_from, None);
6153        assert_eq!(snapshot.launch_spec_digest, None);
6154        assert_eq!(snapshot.spawned_file_identity, None);
6155    }
6156
6157    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6158    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6159        let supervisor =
6160            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6161        let mut runtime = supervisor.runtime_config();
6162        runtime.test_seed_stale_facts_before_enable_spawn = true;
6163        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6164        let mut child = None;
6165        let spec = ModuleSpec {
6166            module_id: "failed-enable-clears-facts".to_string(),
6167            program: PathBuf::from("/definitely/missing/failed-enable-module"),
6168            args: Vec::new(),
6169            env: Vec::new(),
6170            reserved: false,
6171            reserved_prefixes: Vec::new(),
6172            protocol: ModuleProtocol::Subc,
6173            overlap: Default::default(),
6174            carry_containment: false,
6175        };
6176
6177        let result = set_child_enabled(
6178            &spec,
6179            &runtime,
6180            &supervisor.registry,
6181            &supervisor.process_liveness,
6182            &snapshot,
6183            &mut child,
6184            true,
6185            &runtime.failure_context(false),
6186        )
6187        .await;
6188
6189        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6190        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6191        assert_snapshot_process_facts_cleared(&snapshot);
6192    }
6193
6194    #[tokio::test]
6195    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6196        let supervisor =
6197            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6198        let runtime = supervisor.runtime_config();
6199        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6200            ModuleState::Restarting,
6201            true,
6202        )));
6203        let spec = ModuleSpec {
6204            module_id: "start-stranded-restarting".to_string(),
6205            program: super::terminal_history_tests::fake_aft_stub_path(),
6206            args: Vec::new(),
6207            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6208            reserved: false,
6209            reserved_prefixes: Vec::new(),
6210            protocol: ModuleProtocol::None,
6211            overlap: Default::default(),
6212            carry_containment: false,
6213        };
6214        let mut child = None;
6215        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6216        assert!(!super::set_child_enabled(
6217            &spec,
6218            &runtime,
6219            &Registry::default(),
6220            &supervisor.process_liveness,
6221            &snapshot,
6222            &mut child,
6223            true,
6224            &runtime.failure_context(false),
6225        )
6226        .await
6227        .unwrap());
6228        assert!(child.is_none());
6229        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6230        assert!(super::set_child_enabled(
6231            &spec,
6232            &runtime,
6233            &Registry::default(),
6234            &supervisor.process_liveness,
6235            &snapshot,
6236            &mut child,
6237            true,
6238            &runtime.failure_context(false),
6239        )
6240        .await
6241        .unwrap());
6242        assert_eq!(
6243            lock_snapshot(&snapshot).unwrap().state,
6244            ModuleState::Running
6245        );
6246        let mut child = child.unwrap();
6247        child.start_kill().unwrap();
6248        child.wait().await.unwrap();
6249    }
6250
6251    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6252    async fn failed_reload_spawn_clears_current_process_facts() {
6253        let supervisor =
6254            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6255        let mut runtime = supervisor.runtime_config();
6256        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6257        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6258        let mut child = None;
6259        let spec = ModuleSpec {
6260            module_id: "failed-reload-clears-facts".to_string(),
6261            program: PathBuf::from("/unused/failed-reload-module"),
6262            args: Vec::new(),
6263            env: Vec::new(),
6264            reserved: false,
6265            reserved_prefixes: Vec::new(),
6266            protocol: ModuleProtocol::Subc,
6267            overlap: Default::default(),
6268            carry_containment: false,
6269        };
6270
6271        let result = handle_reload_spawn_failure(
6272            &spec,
6273            &runtime,
6274            &supervisor.process_liveness,
6275            &snapshot,
6276            &mut child,
6277            "forced reload spawn failure".to_string(),
6278        )
6279        .await;
6280
6281        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6282        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6283        assert_snapshot_process_facts_cleared(&snapshot);
6284    }
6285
6286    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6287    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6288        let supervisor =
6289            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6290        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6291        let module = supervisor.supervised_module(
6292            ModuleSpec {
6293                module_id: "drop-clears-facts".to_string(),
6294                program: PathBuf::from("/unused/drop-module"),
6295                args: Vec::new(),
6296                env: Vec::new(),
6297                reserved: false,
6298                reserved_prefixes: Vec::new(),
6299                protocol: ModuleProtocol::Subc,
6300                overlap: Default::default(),
6301                carry_containment: false,
6302            },
6303            supervisor.runtime_config(),
6304            Arc::clone(&snapshot),
6305            None,
6306        );
6307        assert!(!module
6308            .inner
6309            .monitor
6310            .lock()
6311            .unwrap()
6312            .as_ref()
6313            .unwrap()
6314            .is_finished());
6315
6316        drop(module);
6317
6318        assert_eq!(
6319            lock_snapshot(&snapshot).unwrap().state,
6320            ModuleState::Stopped
6321        );
6322        assert_snapshot_process_facts_cleared(&snapshot);
6323    }
6324
6325    #[cfg(unix)]
6326    #[tokio::test]
6327    async fn rescan_preserves_running_protocol_until_respawn() {
6328        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6329        let initial = ModuleSpec {
6330            module_id: "rescan-protocol".into(),
6331            program: PathBuf::from("/bin/sleep"),
6332            args: vec!["60".into()],
6333            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6334                .into_iter()
6335                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6336                .collect(),
6337            reserved: false,
6338            reserved_prefixes: vec![],
6339            protocol: ModuleProtocol::None,
6340            overlap: Default::default(),
6341            carry_containment: false,
6342        };
6343        let supervisor =
6344            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6345        let module = supervisor.spawn(initial.clone()).unwrap();
6346        assert!(module.status().unwrap().live);
6347        let mut next = initial;
6348        next.protocol = ModuleProtocol::Subc;
6349        module
6350            .update_configuration(next.clone(), HealthConfig::default(), None)
6351            .await
6352            .unwrap();
6353        assert!(
6354            module.status().unwrap().live,
6355            "rescan must not require HELLO from the old non-wire process"
6356        );
6357        let runtime = supervisor.runtime_config();
6358        let action = on_child_exit(
6359            &next,
6360            RestartPolicy::default(),
6361            &supervisor.registry,
6362            &module.inner.snapshot,
6363            &runtime.terminal_ring,
6364            &runtime.spawn_events,
6365            &runtime.child_roster,
6366            ExitReport {
6367                kind: ExitKind::Clean,
6368                code: Some(0),
6369                signal: None,
6370                at_ms: unix_ms_now(),
6371            },
6372            true,
6373        )
6374        .await;
6375        assert!(
6376            matches!(action, NextAction::Restart { .. }),
6377            "the old non-wire process's clean exit must restart"
6378        );
6379        module.drain().await.unwrap();
6380    }
6381
6382    #[cfg(unix)]
6383    #[tokio::test]
6384    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6385        use std::os::unix::fs::PermissionsExt;
6386        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6387        let script = dir.join("module.sh");
6388        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6389        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6390        let record_path = dir.join("live-children.json");
6391        let supervisor =
6392            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6393                .with_live_children_record(&record_path);
6394        for (program, args) in [
6395            (PathBuf::from("sleep"), vec!["60".into()]),
6396            (script, vec![]),
6397        ] {
6398            let spec = ModuleSpec {
6399                module_id: "image-identity".into(),
6400                program: program.clone(),
6401                args,
6402                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6403                    .into_iter()
6404                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6405                    .collect(),
6406                reserved: false,
6407                reserved_prefixes: vec![],
6408                protocol: ModuleProtocol::None,
6409                overlap: Default::default(),
6410                carry_containment: false,
6411            };
6412            let module = supervisor.spawn(spec).unwrap();
6413            #[cfg(target_os = "macos")]
6414            {
6415                // SETEXEC confirmation is asynchronous; the orphan record must
6416                // identify the final image, never the intermediate trampoline.
6417                let deadline = Instant::now() + Duration::from_secs(5);
6418                while crate::live_children::read_record(&record_path)
6419                    .unwrap()
6420                    .iter()
6421                    .all(|entry| entry.executable.is_none())
6422                {
6423                    assert!(Instant::now() < deadline, "module image was not confirmed");
6424                    tokio::time::sleep(Duration::from_millis(5)).await;
6425                }
6426            }
6427            let entry = crate::live_children::read_record(&record_path)
6428                .unwrap()
6429                .pop()
6430                .unwrap();
6431            let observed = subc_os::Process::open(entry.pid)
6432                .unwrap()
6433                .unwrap()
6434                .observe()
6435                .unwrap();
6436            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6437            module.drain().await.unwrap();
6438            assert_eq!(
6439                verdict,
6440                crate::live_children::IdentityVerdict::Matches,
6441                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6442            );
6443        }
6444    }
6445
6446    #[cfg(unix)]
6447    fn http_fixture(
6448        dir: &std::path::Path,
6449        url: &str,
6450        threshold: u32,
6451    ) -> crate::daemon_config::ConfiguredModule {
6452        let path = dir.join("subc.jsonc");
6453        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6454            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6455            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6456            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6457            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6458        }}}).to_string()).unwrap();
6459        crate::daemon_config::load(&path)
6460            .unwrap()
6461            .unwrap()
6462            .modules
6463            .pop()
6464            .unwrap()
6465    }
6466
6467    #[cfg(unix)]
6468    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6469        timeout(Duration::from_secs(5), async {
6470            loop {
6471                if module.status().unwrap().health.status == status {
6472                    break;
6473                }
6474                sleep(Duration::from_millis(5)).await;
6475            }
6476        })
6477        .await
6478        .unwrap_or_else(|_| {
6479            panic!(
6480                "expected {status:?}, got {:?}",
6481                module.status().unwrap().health
6482            )
6483        });
6484    }
6485
6486    #[cfg(unix)]
6487    #[tokio::test]
6488    async fn http_health_status_flips_ok_failing_ok() {
6489        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6490        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6491        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6492        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6493        let serving_status = status.clone();
6494        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6495        let server = tokio::spawn(async move {
6496            loop {
6497                let (mut stream, _) = listener.accept().await.unwrap();
6498                let mut request = [0u8; 2048];
6499                let count = stream.read(&mut request).await.unwrap();
6500                assert!(count > 0, "a probe must send an HTTP request");
6501                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6502                let body = if code == 200 {
6503                    "ready"
6504                } else {
6505                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6506                };
6507                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6508                let _ = stream.write_all(response.as_bytes()).await;
6509            }
6510        });
6511        let configured = http_fixture(&dir, &url, 1000);
6512        let module =
6513            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6514                .supervise_configured_with_health(
6515                    configured.module_spec(),
6516                    true,
6517                    configured.health,
6518                    configured.drain_timeout_ms,
6519                    configured.restart,
6520                )
6521                .unwrap();
6522        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6523        status.store(503, std::sync::atomic::Ordering::SeqCst);
6524        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6525        assert!(module
6526            .status()
6527            .unwrap()
6528            .health
6529            .detail
6530            .unwrap()
6531            .contains("scratch failure"));
6532        status.store(200, std::sync::atomic::Ordering::SeqCst);
6533        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6534        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6535        let before = module.status().unwrap();
6536        let (spec, mut health) = module.configuration().unwrap();
6537        health.http = None;
6538        module
6539            .update_configuration(spec.clone(), health.clone(), Some(10))
6540            .await
6541            .unwrap();
6542        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6543        health.http = Some(url);
6544        module
6545            .update_configuration(spec, health, Some(10))
6546            .await
6547            .unwrap();
6548        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6549        assert_eq!(
6550            module.status().unwrap().pid,
6551            before.pid,
6552            "changing a probe must apply live, not restart its process"
6553        );
6554        let (spec, mut health) = module.configuration().unwrap();
6555        health.failure_threshold = 2;
6556        module
6557            .update_configuration(spec, health, Some(10))
6558            .await
6559            .unwrap();
6560        status.store(503, std::sync::atomic::Ordering::SeqCst);
6561        timeout(Duration::from_secs(5), async {
6562            while module.status().unwrap().spawn_generation == before.spawn_generation {
6563                sleep(Duration::from_millis(5)).await;
6564            }
6565        })
6566        .await
6567        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6568        module.drain().await.unwrap();
6569        server.abort();
6570    }
6571
6572    #[cfg(unix)]
6573    #[tokio::test]
6574    async fn http_health_replacement_recovers_after_one_restart() {
6575        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6576        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6577        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6578        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6579        let mut configured = http_fixture(&dir, &url, 2);
6580        configured.restart.max_restarts = 2;
6581        let module =
6582            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6583                .supervise_configured_with_health(
6584                    configured.module_spec(),
6585                    true,
6586                    configured.health,
6587                    configured.drain_timeout_ms,
6588                    configured.restart,
6589                )
6590                .unwrap();
6591        let first_generation = module.status().unwrap().spawn_generation;
6592        let observed = module.clone();
6593        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6594        let server = tokio::spawn(async move {
6595            let mut generation = first_generation;
6596            let mut probes = 0;
6597            loop {
6598                let (mut stream, _) = listener.accept().await.unwrap();
6599                let mut request = [0; 2048];
6600                assert!(stream.read(&mut request).await.unwrap() > 0);
6601                let current = observed.status().unwrap().spawn_generation;
6602                if current != generation {
6603                    generation = current;
6604                    probes = 0;
6605                }
6606                probes += 1;
6607                // The original process never recovers. Each replacement has one
6608                // transient miss, then answers successfully for its remaining life.
6609                let healthy = generation != first_generation && probes > 1;
6610                let code = if healthy { 200 } else { 503 };
6611                let response = format!(
6612                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6613                );
6614                let _ = stream.write_all(response.as_bytes()).await;
6615                if healthy {
6616                    healthy_tx.send(()).await.unwrap();
6617                }
6618            }
6619        });
6620        timeout(Duration::from_secs(5), async {
6621            let mut healthy_probes = 0;
6622            loop {
6623                if module.state().unwrap() == ModuleState::Failed {
6624                    break;
6625                }
6626                while healthy_rx.try_recv().is_ok() {
6627                    healthy_probes += 1;
6628                }
6629                if healthy_probes >= 3
6630                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6631                {
6632                    break;
6633                }
6634                sleep(Duration::from_millis(5)).await;
6635            }
6636        })
6637        .await
6638        .expect("the replacement must settle healthy without exhausting its restart budget");
6639        let status = module.status().unwrap();
6640        module.drain().await.unwrap();
6641        server.abort();
6642        assert_eq!(status.state, ModuleState::Running);
6643        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6644        assert_eq!(status.health.consecutive_failures, 0);
6645        assert_eq!(status.lifetime_restarts, 1);
6646        assert_eq!(status.restart_count, 1);
6647    }
6648
6649    #[cfg(unix)]
6650    #[tokio::test]
6651    async fn http_health_sustained_failure_exhausts_restart_budget() {
6652        use std::collections::BTreeMap;
6653        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6654        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6655        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6656        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6657        let mut configured = http_fixture(&dir, &url, 2);
6658        configured.restart.max_restarts = 2;
6659        let module =
6660            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6661                .supervise_configured_with_health(
6662                    configured.module_spec(),
6663                    true,
6664                    configured.health,
6665                    configured.drain_timeout_ms,
6666                    configured.restart,
6667                )
6668                .unwrap();
6669        let observed = module.clone();
6670        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6671        let served = probes.clone();
6672        let server = tokio::spawn(async move {
6673            loop {
6674                let (mut stream, _) = listener.accept().await.unwrap();
6675                let mut request = [0; 2048];
6676                assert!(stream.read(&mut request).await.unwrap() > 0);
6677                *served
6678                    .lock()
6679                    .unwrap()
6680                    .entry(observed.status().unwrap().spawn_generation)
6681                    .or_default() += 1;
6682                let _ = stream
6683                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6684                    .await;
6685            }
6686        });
6687        timeout(Duration::from_secs(5), async {
6688            while module.state().unwrap() != ModuleState::Failed {
6689                sleep(Duration::from_millis(5)).await;
6690            }
6691        })
6692        .await
6693        .expect("sustained failures must still exhaust the health restart budget");
6694        let status = module.status().unwrap();
6695        server.abort();
6696        assert_eq!(status.restart_count, 2);
6697        assert_eq!(status.lifetime_restarts, 2);
6698        assert!(!status.process_alive);
6699        assert_eq!(
6700            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6701            vec![2, 2, 2]
6702        );
6703    }
6704
6705    #[cfg(unix)]
6706    #[tokio::test]
6707    async fn spawn_clears_the_previous_process_health() {
6708        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6709        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6710        configured.health.cadence = Duration::from_secs(3600);
6711        let module =
6712            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6713                .supervise_configured_with_health(
6714                    configured.module_spec(),
6715                    false,
6716                    configured.health,
6717                    configured.drain_timeout_ms,
6718                    configured.restart,
6719                )
6720                .unwrap();
6721        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6722            state.health.status = SupervisorHealthStatus::Unresponsive;
6723            state.health.consecutive_failures = 3;
6724            state.health.detail = Some("old process refused connections".into());
6725            state.health.metrics = Some(serde_json::json!({"old": true}));
6726            state.health.last_probe_ms = Some(123);
6727            state.health.last_action = Some("restart".into());
6728            state.health.last_action_ms = Some(124);
6729        })
6730        .unwrap();
6731        module.set_enabled(true).await.unwrap();
6732        let health = module.status().unwrap().health;
6733        module.drain().await.unwrap();
6734        assert_eq!(health.consecutive_failures, 0);
6735        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6736        assert_eq!(health.detail, None);
6737        assert_eq!(health.metrics, None);
6738        assert_eq!(health.last_probe_ms, None);
6739        assert_eq!(health.last_action.as_deref(), Some("restart"));
6740        assert_eq!(health.last_action_ms, Some(124));
6741    }
6742
6743    #[cfg(unix)]
6744    #[tokio::test]
6745    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6746        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6747        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6748        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6749        drop(unused);
6750        let configured = http_fixture(&dir, &url, 2);
6751        let module =
6752            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6753                .supervise_configured_with_health(
6754                    configured.module_spec(),
6755                    true,
6756                    configured.health,
6757                    configured.drain_timeout_ms,
6758                    configured.restart,
6759                )
6760                .unwrap();
6761        let before = module.status().unwrap().spawn_generation;
6762        timeout(Duration::from_secs(5), async {
6763            loop {
6764                let status = module.status().unwrap();
6765                if status.spawn_generation > before {
6766                    assert!(status.lifetime_restarts > 0);
6767                    break;
6768                }
6769                sleep(Duration::from_millis(5)).await;
6770            }
6771        })
6772        .await
6773        .expect("sustained HTTP refusal must trigger the health restart policy");
6774        module.drain().await.unwrap();
6775    }
6776
6777    #[cfg(unix)]
6778    #[tokio::test]
6779    async fn http_health_timeout_honours_deadline() {
6780        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6781        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6782        let server = tokio::spawn(async move {
6783            let _held = listener.accept().await.unwrap();
6784            std::future::pending::<()>().await;
6785        });
6786        let error = timeout(
6787            Duration::from_secs(1),
6788            probe_http_health(&url, Duration::from_millis(10)),
6789        )
6790        .await
6791        .expect("the probe must enforce its own deadline")
6792        .unwrap_err();
6793        server.abort();
6794        assert!(error.to_string().contains("timed out"));
6795    }
6796
6797    #[cfg(unix)]
6798    #[tokio::test]
6799    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6800        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6801        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6802        let url = format!(
6803            "http://localhost:{}/healthz",
6804            listener.local_addr().unwrap().port()
6805        );
6806        let server = tokio::spawn(async move {
6807            let (mut stream, _) = listener.accept().await.unwrap();
6808            let mut request = [0u8; 2048];
6809            assert!(stream.read(&mut request).await.unwrap() > 0);
6810            stream
6811                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6812                .await
6813                .unwrap();
6814        });
6815        // The deadline only bounds a hang. A probe that never tried the IPv6
6816        // address would be refused on 127.0.0.1 and fail at once, so a longer
6817        // deadline does not weaken the assertion; one second timed out under a
6818        // loaded parallel test run.
6819        assert_eq!(
6820            probe_http_health(&url, Duration::from_secs(10))
6821                .await
6822                .unwrap()
6823                .status,
6824            HealthStatus::Ok
6825        );
6826        server.await.unwrap();
6827    }
6828
6829    #[cfg(unix)]
6830    #[tokio::test]
6831    async fn http_health_timeout_keeps_partial_status_and_body() {
6832        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6833        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6834        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6835        let server = tokio::spawn(async move {
6836            let (mut stream, _) = listener.accept().await.unwrap();
6837            let mut request = [0u8; 2048];
6838            assert!(stream.read(&mut request).await.unwrap() > 0);
6839            stream
6840                .write_all(
6841                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6842                )
6843                .await
6844                .unwrap();
6845            std::future::pending::<()>().await;
6846        });
6847        let error = probe_http_health(&url, Duration::from_secs(1))
6848            .await
6849            .unwrap_err()
6850            .to_string();
6851        server.abort();
6852        assert!(
6853            error.contains("timed out")
6854                && error.contains("503 Unavailable")
6855                && error.contains("partial diagnostic"),
6856            "{error}"
6857        );
6858    }
6859
6860    #[cfg(unix)]
6861    #[tokio::test]
6862    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6863        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6864        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6865        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6866        let server = tokio::spawn(async move {
6867            let (mut stream, _) = listener.accept().await.unwrap();
6868            let mut request = [0u8; 2048];
6869            assert!(stream.read(&mut request).await.unwrap() > 0);
6870            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6871            let response = format!(
6872                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6873                body.len()
6874            );
6875            stream.write_all(response.as_bytes()).await.unwrap();
6876        });
6877        let error = probe_http_health(&url, Duration::from_secs(1))
6878            .await
6879            .unwrap_err()
6880            .to_string();
6881        server.await.unwrap();
6882        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6883        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6884        assert!(!error.contains("not-in-diagnostic"));
6885    }
6886
6887    #[cfg(unix)]
6888    #[tokio::test]
6889    async fn http_health_real_nats_server_monitoring() {
6890        if std::process::Command::new("nats-server")
6891            .arg("--version")
6892            .env("XDG_DATA_HOME", std::env::temp_dir())
6893            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6894            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6895            .output()
6896            .is_err()
6897        {
6898            eprintln!(
6899                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6900            );
6901            return;
6902        }
6903        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6904        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6905        let port = monitor.local_addr().unwrap().port();
6906        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6907        let client_port = client.local_addr().unwrap().port();
6908        let config = dir.join("server.conf");
6909        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6910        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6911        configured.program = PathBuf::from("nats-server");
6912        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6913        drop(monitor);
6914        drop(client);
6915        let module =
6916            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6917                .supervise_configured_with_health(
6918                    configured.module_spec(),
6919                    true,
6920                    configured.health,
6921                    configured.drain_timeout_ms,
6922                    configured.restart,
6923                )
6924                .unwrap();
6925        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6926        module.drain().await.unwrap();
6927    }
6928
6929    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6930    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6931        let supervisor =
6932            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6933        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6934        let initial = ModuleSpec {
6935            module_id: "rescan-preserves-spawn-facts".to_string(),
6936            program: PathBuf::from("/spawned/module"),
6937            args: Vec::new(),
6938            env: Vec::new(),
6939            reserved: false,
6940            reserved_prefixes: Vec::new(),
6941            protocol: ModuleProtocol::Subc,
6942            overlap: Default::default(),
6943            carry_containment: false,
6944        };
6945        let module = supervisor.supervised_module(
6946            initial.clone(),
6947            supervisor.runtime_config(),
6948            snapshot,
6949            None,
6950        );
6951        let before = module.status().unwrap();
6952        let mut replacement = initial;
6953        replacement.program = PathBuf::from("/rescanned/replacement-module");
6954
6955        module
6956            .update_configuration(replacement, HealthConfig::default(), None)
6957            .await
6958            .unwrap();
6959
6960        let after = module.status().unwrap();
6961        assert_eq!(after.pid, before.pid);
6962        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6963        assert_eq!(after.spawned_from, before.spawned_from);
6964        drop(module);
6965    }
6966}
6967
6968fn unix_ms_now() -> u64 {
6969    SystemTime::now()
6970        .duration_since(UNIX_EPOCH)
6971        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6972        .unwrap_or(0)
6973}
6974
6975async fn supervise_loop(
6976    mut spec: ModuleSpec,
6977    mut runtime: SupervisorRuntimeConfig,
6978    registry: Arc<Registry>,
6979    process_liveness: Arc<SupervisorProcessLiveness>,
6980    snapshot: SharedSnapshot,
6981    mut child: Option<SupervisedChild>,
6982    mut commands: mpsc::Receiver<SupervisorCommand>,
6983) {
6984    let mut health_probe = HealthProbeRuntime::default();
6985    // Registry writes (including embedded callers) notify this module only.
6986    // Subscribe before the first refresh; watch retains changes that arrive
6987    // while commands or probes are running. No polling fallback is needed.
6988    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6989        Ok(changes) => Some(changes),
6990        Err(err) => {
6991            // A poisoned registry cannot accept further writes, so it cannot
6992            // recover via a timer. Keep exit and command handling alive.
6993            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6994            None
6995        }
6996    };
6997    // All restart backoffs run here, including health and operator requests.
6998    // While one is pending the loop serves commands, so disable or drain can
6999    // cancel the replacement without spawning a process just to stop it.
7000    let mut pending_respawn: Option<PendingRespawn> = None;
7001    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
7002    // before anything else so a stop that interrupted a swap runs at once.
7003    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
7004    let mut previous_instance_recheck = tokio::time::interval_at(
7005        Instant::now() + Duration::from_secs(5),
7006        Duration::from_secs(5),
7007    );
7008    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
7009    loop {
7010        #[cfg(test)]
7011        {
7012            lock_snapshot(&snapshot).unwrap().actor_select = None;
7013        }
7014        #[cfg(target_os = "macos")]
7015        if let Some(active) = child.as_mut() {
7016            active.confirm_privacy_exec().await;
7017        }
7018        if let Some(scheduled) = runtime
7019            .scheduled_respawn
7020            .lock()
7021            .unwrap_or_else(|p| p.into_inner())
7022            .take()
7023        {
7024            runtime.recovery_is_daemon = scheduled.daemon_initiated;
7025            pending_respawn = Some(scheduled);
7026        }
7027        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7028            pending_respawn = None;
7029            cancel_deferred_reload(
7030                &runtime,
7031                &spec.module_id,
7032                "respawn cancelled by a supervisor command",
7033            );
7034        }
7035        if child.is_none()
7036            && pending_respawn.is_none()
7037            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7038        {
7039            cancel_deferred_reload(
7040                &runtime,
7041                &spec.module_id,
7042                "respawn cancelled before a replacement was spawned",
7043            );
7044            let mut stranded = false;
7045            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7046                state.respawn_pending = false;
7047                state.coalesced_restart_pending = false;
7048                if matches!(
7049                    state.state,
7050                    ModuleState::Restarting
7051                        | ModuleState::Starting
7052                        | ModuleState::Draining
7053                        | ModuleState::Unresponsive
7054                ) {
7055                    error!(module_id = %spec.module_id, state = ?state.state,
7056                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7057                    stranded = true;
7058                }
7059            });
7060            if stranded {
7061                fail_snapshot(
7062                    &snapshot,
7063                    Some(&spec.module_id),
7064                    None,
7065                    &runtime.failure_context(runtime.recovery_is_daemon),
7066                );
7067            }
7068        }
7069        if child.is_none()
7070            && (runtime.child_roster.is_closed()
7071                || pending_respawn.is_none()
7072                || !lock_snapshot(&snapshot)
7073                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7074        {
7075            containment::terminate(&runtime.child_roster.containment).await;
7076        }
7077        if let Some(command) = requeued.pop_front() {
7078            if !handle_supervisor_command(
7079                command,
7080                &mut spec,
7081                &mut runtime,
7082                &registry,
7083                &process_liveness,
7084                &snapshot,
7085                &mut child,
7086                &mut commands,
7087                &mut requeued,
7088            )
7089            .await
7090            {
7091                containment::terminate(&runtime.child_roster.containment).await;
7092                return;
7093            }
7094            if child.is_some() || !respawn_still_pending(&snapshot) {
7095                pending_respawn = None;
7096                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7097                    state.respawn_pending = false
7098                });
7099            }
7100            continue;
7101        }
7102        if child.is_some() {
7103            runtime.recovery_is_daemon = true;
7104            #[cfg(test)]
7105            {
7106                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7107            }
7108            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
7109            let active = child.as_mut().expect("child checked above");
7110            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7111                active.registration_deadline = None;
7112            }
7113            let registration_deadline = active.registration_deadline;
7114            let no_checks = runtime.health.http.is_none()
7115                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7116                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7117            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7118                state
7119                    .failed_retry
7120                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
7121            });
7122            let wake_at = match (health_probe.wake_at(), uptime_reset) {
7123                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7124                (probe, None) => probe,
7125            };
7126            let wake_at = match (wake_at, registration_deadline) {
7127                (Some(wake), Some(register)) => Some(wake.min(register)),
7128                (wake, register) => wake.or(register),
7129            };
7130            #[cfg(test)]
7131            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7132            let probe_sleep = supervisor_sleep(wake_at);
7133            tokio::pin!(probe_sleep);
7134            // A `protocol: "none"` child (a plain process that never registers
7135            // over the subc wire, such as nats-server) with no HTTP health
7136            // check has nothing to probe, so it waits only for its exit or a
7137            // supervisor command. A rescan that adds an HTTP check is a command.
7138            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7139            #[cfg(test)]
7140            {
7141                let registration_pending = wire_child
7142                    && registration_changes
7143                        .as_ref()
7144                        .is_some_and(|changes| changes.has_changed().unwrap());
7145                let mut state = lock_snapshot(&snapshot).unwrap();
7146                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7147                    generation: state.spawn_generation,
7148                    turn: state.actor_turns,
7149                    registered_connection: health_probe.registered_connection,
7150                    next_probe_at: health_probe.next_probe_at,
7151                    wake_after,
7152                });
7153            }
7154            let active_child = child.as_mut().expect("child checked above");
7155            tokio::select! {
7156                wait_result = active_child.wait() => {
7157                    if let Ok(mut state) = lock_snapshot(&snapshot) {
7158                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7159                    }
7160                    #[cfg(test)]
7161                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7162                    // Every arm below that gives up on the CHILD must keep the
7163                    // supervision task itself alive (child = None, loop
7164                    // continues into command-serving mode). Returning here
7165                    // closes the command channel, which makes the module
7166                    // permanently unrestartable in-band: a clean child exit
7167                    // of an enabled module once wedged the fleet this way
7168                    // ('supervisor command channel is closed') and required a
7169                    // full daemon restart to recover.
7170                    let exit_report = match wait_result {
7171                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7172                        Err(err) => {
7173                            active_child.drain_stderr(&spec.module_id).await;
7174                            fail_after_wait_error(
7175                                &spec.module_id,
7176                                &snapshot,
7177                                &runtime.terminal_ring,
7178                                &runtime.spawn_events,
7179                                format!("failed to wait for supervised module: {err}"),
7180                            &runtime.failure_context(true),
7181                            );
7182                            untrack_if_registration_released(
7183                                &process_liveness,
7184                                &registry,
7185                                &spec.module_id,
7186                                &snapshot,
7187                            );
7188                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7189                            child = None;
7190                            continue;
7191                        }
7192                    };
7193                    active_child.drain_stderr(&spec.module_id).await;
7194
7195                    let next = on_child_exit(
7196                        &spec,
7197                        runtime.restart_policy,
7198                        &registry,
7199                        &snapshot,
7200                        &runtime.terminal_ring,
7201                        &runtime.spawn_events,
7202                        &runtime.child_roster,
7203                        exit_report,
7204                        true,
7205                    ).await;
7206                    runtime.start_requester = None;
7207                    runtime.pending_activity_restart = false;
7208                    // The exit is recorded, so a daemon shutdown may stop
7209                    // waiting for this child (see `SupervisedChild::wait`).
7210                    let carry = matches!(next, NextAction::Restart { .. })
7211                        && active_child.spawn_failure.is_none()
7212                        && !runtime.child_roster.is_closed();
7213                    if let Err(error) = active_child.finish_exit(carry).await {
7214                        containment::terminate(&runtime.child_roster.containment).await;
7215                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7216                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7217                            &runtime.failure_context(true));
7218                        child = None;
7219                        continue;
7220                    }
7221                    match next {
7222                        NextAction::Stop { registration_released } => {
7223                            if registration_released {
7224                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7225                            }
7226                            child = None;
7227                        }
7228                        NextAction::Restart { schedule } => {
7229                            let delay = schedule.map_or(
7230                                runtime.restart_policy.delay_for_restart(0),
7231                                |schedule| schedule.delay,
7232                            );
7233                            if let Some(schedule) = schedule {
7234                                log_crash_respawn(&spec.module_id, schedule);
7235                            }
7236                            // The exited child is fully recorded at this point,
7237                            // so release it and count the backoff down in the
7238                            // command-serving branch below rather than sleeping
7239                            // here: commands cannot be received from inside this
7240                            // select arm, and an operator disable or drain that
7241                            // arrives during the backoff must cancel the pending
7242                            // respawn instead of waiting for it to spawn first.
7243                            child = None;
7244                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7245                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7246                        }
7247                    }
7248                }
7249                command = commands.recv() => {
7250                    #[cfg(test)]
7251                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7252                    let Some(command) = command else {
7253                        containment::terminate(&runtime.child_roster.containment).await;
7254                        return;
7255                    };
7256                    if !handle_supervisor_command(
7257                        command,
7258                        &mut spec,
7259                        &mut runtime,
7260                        &registry,
7261                        &process_liveness,
7262                        &snapshot,
7263                        &mut child,
7264                        &mut commands,
7265                        &mut requeued,
7266                    ).await {
7267                        containment::terminate(&runtime.child_roster.containment).await;
7268                        return;
7269                    }
7270                }
7271                _ = async {
7272                    match registration_changes.as_mut() {
7273                        Some(changes) => { let _ = changes.changed().await; }
7274                        None => std::future::pending().await,
7275                    }
7276                }, if wire_child => {
7277                    #[cfg(test)]
7278                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7279                }
7280                _ = &mut probe_sleep => {
7281                    #[cfg(test)]
7282                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7283                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7284                        // Recheck after the wake: HELLO, exit or shutdown may
7285                        // have become ready in the same executor tick.
7286                        active_child.registration_deadline = None;
7287                        if !runtime.child_roster.is_closed()
7288                            && !active_child.registered_since_spawn()
7289                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7290                        {
7291                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7292                                wait_secs = runtime.register_timeout.as_secs(),
7293                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7294                            active_child.spawn_failure = Some("registration_timeout".into());
7295                            if let Err(error) = stop_unregistered_child(active_child).await {
7296                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7297                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7298                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7299                                    &runtime.failure_context(true));
7300                                child = None;
7301                            }
7302                            // The ordinary exit handler records the child's exit,
7303                            // spends crash budget, and schedules recovery.
7304                            continue;
7305                        }
7306                    }
7307                    if health_probe.due() {
7308                        run_health_probe_cycle(
7309                            &spec,
7310                            &runtime,
7311                            &registry,
7312                            &process_liveness,
7313                            &snapshot,
7314                            &mut child,
7315                        ).await;
7316                        if child.is_none() {
7317                            runtime.start_requester = None;
7318                            runtime.pending_activity_restart = false;
7319                        }
7320                        if child.is_some() {
7321                            health_probe.schedule_next(&spec, runtime.health.cadence);
7322                        }
7323                    }
7324                }
7325            }
7326        } else if let Some(pending) = pending_respawn {
7327            tokio::select! {
7328                _ = runtime.child_roster.closed() => {
7329                    pending_respawn = None;
7330                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7331                        state.state = ModuleState::Stopped;
7332                        state.respawn_pending = false;
7333                    });
7334                    containment::terminate(&runtime.child_roster.containment).await;
7335                }
7336                _ = sleep_until(pending.deadline) => {
7337                    pending_respawn = None;
7338                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7339                    // A command handled below while the backoff elapsed may
7340                    // have stopped the module; never respawn past an operator's
7341                    // disable or drain.
7342                    if !respawn_still_pending(&snapshot) {
7343                        continue;
7344                    }
7345                    // The daemon began shutting down during the backoff: the
7346                    // spawn would be refused anyway, and refusing it here
7347                    // leaves the module stopped instead of reporting a
7348                    // failed restart.
7349                    if runtime.child_roster.is_closed() {
7350                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7351                            state.state = ModuleState::Stopped;
7352                        });
7353                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7354                        continue;
7355                    }
7356                    if let Err(err) = release_dead_registration(
7357                        &registry,
7358                        runtime.forwarding.as_deref(),
7359                        &snapshot,
7360                        &spec.module_id,
7361                        &runtime.failure_context(runtime.recovery_is_daemon),
7362                    ).await {
7363                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7364                        continue;
7365                    }
7366
7367                    if matches!(pending.kind, RespawnKind::Reload) {
7368                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7369                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7370                        if result.is_ok() && runtime.pending_activity_restart {
7371                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7372                        }
7373                        runtime.pending_activity_restart = false;
7374                        if let Some(reply) = reply { let _ = reply.send(result); }
7375                        continue;
7376                    }
7377                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7378                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7379                        Ok(next_child) => {
7380                            child = Some(next_child);
7381                            if runtime.pending_activity_restart {
7382                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7383                                runtime.pending_activity_restart = false;
7384                            }
7385                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7386                        }
7387                        Err(err) => {
7388                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7389                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7390                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7391                        }
7392                    }
7393                }
7394                command = commands.recv() => {
7395                    let Some(command) = command else {
7396                        containment::terminate(&runtime.child_roster.containment).await;
7397                        return;
7398                    };
7399                    if !handle_supervisor_command(
7400                        command,
7401                        &mut spec,
7402                        &mut runtime,
7403                        &registry,
7404                        &process_liveness,
7405                        &snapshot,
7406                        &mut child,
7407                        &mut commands,
7408                        &mut requeued,
7409                    ).await {
7410                        containment::terminate(&runtime.child_roster.containment).await;
7411                        return;
7412                    }
7413                    // Reconcile the pending respawn with what the command did:
7414                    // a start may already have spawned a fresh child,
7415                    // while a disable or drain moved the snapshot out of the
7416                    // state the respawn was counting down from.
7417                    if child.is_some() || !respawn_still_pending(&snapshot) {
7418                        pending_respawn = None;
7419                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7420                    }
7421                }
7422            }
7423        } else {
7424            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7425                (state.enabled && state.state == ModuleState::Failed)
7426                    .then_some(state.failed_retry.pending)
7427                    .flatten()
7428            });
7429            let command = if lock_snapshot(&snapshot)
7430                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7431            {
7432                tokio::select! {
7433                    command = commands.recv() => command,
7434                    _ = previous_instance_recheck.tick() => {
7435                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7436                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7437                            if enabled && !runtime.child_roster.is_closed() {
7438                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7439                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7440                                    Ok(next_child) => child = Some(next_child),
7441                                    Err(error) => {
7442                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7443                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7444                                    }
7445                                }
7446                            } else {
7447                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7448                                    state.waiting_for_previous_pid = None;
7449                                    state.waiting_for_previous_detail = None;
7450                                });
7451                            }
7452                        }
7453                        continue;
7454                    }
7455                }
7456            } else {
7457                tokio::select! {
7458                    biased;
7459                    command = commands.recv() => command,
7460                    _ = async {
7461                        match slow_retry {
7462                            Some(retry) => sleep_until(retry.deadline).await,
7463                            None => std::future::pending().await,
7464                        }
7465                    } => {
7466                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7467                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7468                            state.failed_retry.pending = None;
7469                        });
7470                        if runtime.child_roster.is_closed() {
7471                            continue;
7472                        }
7473                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7474                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7475                        if let Err(error) = set_child_enabled(
7476                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7477                            &runtime.failure_context(true),
7478                        ).await {
7479                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7480                        }
7481                        continue;
7482                    }
7483                }
7484            };
7485            let Some(command) = command else {
7486                containment::terminate(&runtime.child_roster.containment).await;
7487                return;
7488            };
7489            if !handle_supervisor_command(
7490                command,
7491                &mut spec,
7492                &mut runtime,
7493                &registry,
7494                &process_liveness,
7495                &snapshot,
7496                &mut child,
7497                &mut commands,
7498                &mut requeued,
7499            )
7500            .await
7501            {
7502                containment::terminate(&runtime.child_roster.containment).await;
7503                return;
7504            }
7505        }
7506    }
7507}
7508
7509// Preserve the deadline until the future is polled. Starting a relative sleep
7510// after the executor yields would restart part of the countdown.
7511async fn supervisor_sleep(wake_at: Option<Instant>) {
7512    match wake_at {
7513        Some(deadline) => sleep_until(deadline).await,
7514        None => std::future::pending().await,
7515    }
7516}
7517
7518fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7519    info!(
7520        module_id,
7521        restart_in_window = schedule.restart_in_window,
7522        delay_ms = schedule.delay.as_millis() as u64,
7523        "respawning after crash"
7524    );
7525}
7526
7527/// Whether the respawn a backoff was counting down to is still wanted. A
7528/// disable or drain handled while the backoff elapsed moves the snapshot out
7529/// of `Restarting`, and the operator's stop must win over the pending respawn,
7530/// so every sleep-then-spawn path re-validates against the live snapshot
7531/// instead of assuming the state it left behind still holds.
7532fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7533    matches!(
7534        lock_snapshot(snapshot),
7535        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7536    )
7537}
7538
7539enum NextAction {
7540    Stop {
7541        registration_released: bool,
7542    },
7543    Restart {
7544        schedule: Option<CrashRestartSchedule>,
7545    },
7546}
7547
7548#[allow(clippy::too_many_arguments)]
7549async fn handle_supervisor_command(
7550    command: SupervisorCommand,
7551    spec: &mut ModuleSpec,
7552    runtime: &mut SupervisorRuntimeConfig,
7553    registry: &Arc<Registry>,
7554    process_liveness: &SupervisorProcessLiveness,
7555    snapshot: &SharedSnapshot,
7556    child: &mut Option<SupervisedChild>,
7557    commands: &mut mpsc::Receiver<SupervisorCommand>,
7558    requeued: &mut VecDeque<SupervisorCommand>,
7559) -> bool {
7560    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7561        runtime.recovery_is_daemon = false;
7562    }
7563    if matches!(
7564        command,
7565        SupervisorCommand::Drain { .. }
7566            | SupervisorCommand::Retire { .. }
7567            | SupervisorCommand::SetEnabled { .. }
7568            | SupervisorCommand::Restart { .. }
7569    ) {
7570        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7571            state.failed_retry.cancel()
7572        });
7573    }
7574    let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7575        runtime
7576            .child_roster
7577            .previous_instance(&spec.module_id)
7578            .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7579    });
7580    let command = if let Some(error) = blocked {
7581        match command {
7582            SupervisorCommand::Restart { reply, .. }
7583            | SupervisorCommand::Reload { reply, .. }
7584            | SupervisorCommand::Swap { reply, .. } => {
7585                let _ = reply.send(Err(error));
7586                return true;
7587            }
7588            SupervisorCommand::SetEnabled {
7589                enabled: true,
7590                reply,
7591                ..
7592            } => {
7593                let _ = reply.send(Err(error));
7594                return true;
7595            }
7596            other => other,
7597        }
7598    } else {
7599        command
7600    };
7601    match command {
7602        SupervisorCommand::Drain { requester, reply } => {
7603            // A plain stop runs no forwarding drain, so nothing reaches the
7604            // module over its connection before the wait: ask by signal.
7605            let result = drain_optional_child(
7606                &spec.module_id,
7607                spec.protocol,
7608                StopNotice::NotSent,
7609                registry,
7610                runtime.forwarding.as_deref(),
7611                snapshot,
7612                &runtime.terminal_ring,
7613                &runtime.spawn_events,
7614                child,
7615                runtime.drain_timeout,
7616                ModuleState::Stopped,
7617                None,
7618                &FailureContext::operator(),
7619            )
7620            .await;
7621            containment::terminate(&runtime.child_roster.containment).await;
7622            let registration_released = result.is_ok();
7623            if registration_released {
7624                runtime.record_activity_transition(
7625                    "subc.module_stopped",
7626                    &spec.module_id,
7627                    requester,
7628                );
7629            }
7630            let _ = reply.send(result);
7631            if registration_released {
7632                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7633            }
7634            false
7635        }
7636        SupervisorCommand::Retire { requester, reply } => {
7637            let result = async {
7638                let stop_notice = begin_forwarding_drain_if_configured(
7639                    spec,
7640                    runtime,
7641                    registry,
7642                    snapshot,
7643                    None,
7644                    RouteCloseReason::Disable,
7645                )
7646                .await?;
7647                drain_optional_child(
7648                    &spec.module_id,
7649                    spec.protocol,
7650                    stop_notice,
7651                    registry,
7652                    runtime.forwarding.as_deref(),
7653                    snapshot,
7654                    &runtime.terminal_ring,
7655                    &runtime.spawn_events,
7656                    child,
7657                    runtime.drain_timeout,
7658                    ModuleState::Stopped,
7659                    None,
7660                    &FailureContext::operator(),
7661                )
7662                .await
7663            }
7664            .await;
7665            containment::terminate(&runtime.child_roster.containment).await;
7666            let registration_released = result.is_ok();
7667            if registration_released {
7668                runtime.record_activity_transition(
7669                    "subc.module_stopped",
7670                    &spec.module_id,
7671                    requester,
7672                );
7673            }
7674            let _ = reply.send(result);
7675            if registration_released {
7676                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7677            }
7678            false
7679        }
7680        SupervisorCommand::Restart {
7681            requester,
7682            drain_timeout_ms,
7683            received_at_generation,
7684            queued_at,
7685            reply,
7686        } => {
7687            // Without this line a restart that waited in the queue (behind a
7688            // health probe cycle or another command) was invisible: the log
7689            // showed only the drain timing out, minutes after the operator's call.
7690            info!(
7691                module_id = %spec.module_id,
7692                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7693                "restart command dequeued"
7694            );
7695            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7696            // caller whose own request lane rides the module being restarted: the
7697            // caller's in-flight request keeps the drain from quiescing, the drain
7698            // keeps the restart from completing, and the completion keeps the reply
7699            // from releasing the caller — so the drain always timed out and cut the
7700            // initiator with a GOODBYE, even on a healthy module. Replying once the
7701            // restart is validated lets a self-lane caller settle, which is exactly
7702            // what makes the drain succeed. Completion is observable via
7703            // supervisor.list / module status; a post-ack failure lands the module
7704            // in a visible terminal state below rather than in a reply nobody can
7705            // receive.
7706            let validation = match lock_snapshot(snapshot) {
7707                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7708                    module_id: spec.module_id.clone(),
7709                }),
7710                Ok(_) => Ok(()),
7711                Err(err) => Err(err),
7712            };
7713            let initiated = validation.is_ok();
7714            let _ = reply.send(validation);
7715            // A restart asks for a fresh process. Commands run one at a time,
7716            // so a restart queued behind another restart (two operator calls
7717            // in quick succession) is dequeued the moment the first one has
7718            // spawned its replacement -- before that process has sent HELLO.
7719            // Running it would drain and kill the process the first restart
7720            // just produced, which is the opposite of what both callers asked
7721            // for. If a process spawned after this request was received is
7722            // still supervised, the request is already satisfied. Not when the
7723            // configuration changed since that spawn: then the newer process
7724            // predates the spec this restart may exist to apply.
7725            // Generation publication can lag a child that is already serving
7726            // while its identity is recorded. Only process creation after request
7727            // receipt satisfies a restart, not publication after receipt.
7728            let satisfied_by_generation = if initiated
7729                && child
7730                    .as_ref()
7731                    .is_some_and(|child| child.spawned_at > queued_at)
7732            {
7733                lock_snapshot(snapshot).ok().and_then(|state| {
7734                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7735                })
7736            } else {
7737                None
7738            };
7739            let satisfied_by_pending = initiated
7740                && child.is_none()
7741                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7742                    // No child exists yet, so this replacement will necessarily
7743                    // be created after the queued request, unlike a published
7744                    // generation whose process may already have been serving.
7745                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7746                    if pending {
7747                        state.coalesced_restart_pending = true;
7748                    }
7749                    pending
7750                });
7751            if satisfied_by_pending {
7752                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7753            } else if let Some(generation) = satisfied_by_generation {
7754                info!(
7755                    module_id = %spec.module_id,
7756                    received_at_generation,
7757                    "restart already satisfied by generation {generation}; not restarting again"
7758                );
7759            } else if initiated {
7760                // Precedence: this restart's operator override, else the module's
7761                // configured budget (already resolved into the runtime).
7762                let drain_timeout = drain_timeout_ms
7763                    .map(Duration::from_millis)
7764                    .unwrap_or(runtime.drain_timeout);
7765                if let Err(err) = restart_child(
7766                    spec,
7767                    runtime,
7768                    registry,
7769                    process_liveness,
7770                    snapshot,
7771                    child,
7772                    drain_timeout,
7773                )
7774                .await
7775                {
7776                    warn!(
7777                        module_id = %spec.module_id,
7778                        error = %err,
7779                        "operator restart failed after initiation ack; module state carries the outcome"
7780                    );
7781                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7782                        state.state = ModuleState::Failed;
7783                        clear_current_process_facts(state);
7784                    });
7785                } else {
7786                    runtime.start_requester = requester;
7787                    runtime.pending_activity_restart = true;
7788                }
7789            }
7790            true
7791        }
7792        SupervisorCommand::Reload { requester, reply } => {
7793            let result =
7794                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7795            if result.is_ok() {
7796                runtime.start_requester = requester;
7797                runtime.pending_activity_restart = true;
7798            }
7799            if result.is_ok()
7800                && runtime
7801                    .scheduled_respawn
7802                    .lock()
7803                    .unwrap_or_else(|p| p.into_inner())
7804                    .as_ref()
7805                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7806            {
7807                *runtime
7808                    .deferred_reload_reply
7809                    .lock()
7810                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7811            } else {
7812                let _ = reply.send(result);
7813            }
7814            true
7815        }
7816        SupervisorCommand::SetEnabled {
7817            enabled,
7818            requester,
7819            reply,
7820        } => {
7821            let previous_requester = runtime.start_requester.clone();
7822            if enabled {
7823                runtime.start_requester = requester.clone();
7824            }
7825            let result = set_child_enabled(
7826                spec,
7827                runtime,
7828                registry,
7829                process_liveness,
7830                snapshot,
7831                child,
7832                enabled,
7833                &runtime.failure_context(false),
7834            )
7835            .await;
7836            if !enabled {
7837                containment::terminate(&runtime.child_roster.containment).await;
7838            }
7839            if matches!(result, Ok(true)) {
7840                runtime.pending_activity_restart = false;
7841                if enabled {
7842                    runtime.start_requester = requester;
7843                } else {
7844                    runtime.start_requester = None;
7845                    runtime.record_activity_transition(
7846                        "subc.module_disabled",
7847                        &spec.module_id,
7848                        requester,
7849                    );
7850                }
7851            } else {
7852                runtime.start_requester = previous_requester;
7853            }
7854            let _ = reply.send(result);
7855            true
7856        }
7857        SupervisorCommand::UpdateConfiguration {
7858            spec: next_spec,
7859            health,
7860            drain_timeout_ms,
7861            reply,
7862        } => {
7863            if let Some(handle) = &runtime.supervisor_handle {
7864                handle.apply_identity_configuration(&next_spec);
7865            }
7866            *spec = next_spec;
7867            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7868                state.configuration_updated_since_spawn = true;
7869            });
7870            let health_changed = runtime.health != health;
7871            runtime.health = health;
7872            // Reset the cadence and old endpoint's failure streak on a live
7873            // health-policy change rather than waiting for its old deadline.
7874            if health_changed {
7875                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7876                    state.health = ModuleHealthStatus::default();
7877                    state.last_probe_evidence = None;
7878                });
7879            }
7880            runtime.drain_timeout = drain_timeout_ms
7881                .map(Duration::from_millis)
7882                .unwrap_or(runtime.default_drain_timeout);
7883            *runtime
7884                .effective_drain_timeout
7885                .lock()
7886                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7887            let _ = reply.send(());
7888            true
7889        }
7890        SupervisorCommand::Swap {
7891            requester,
7892            ready_timeout,
7893            reply,
7894        } => {
7895            if spec.carry_containment
7896                || containment::current(&runtime.child_roster.containment).is_some()
7897            {
7898                let _ = reply.send(Err(SuperviseError::InvalidSpec {
7899                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
7900                }));
7901                return true;
7902            }
7903            let mut swap_runtime = runtime.clone();
7904            swap_runtime.start_requester = requester;
7905            let before = lock_snapshot(snapshot)
7906                .ok()
7907                .map(|state| state.spawn_generation);
7908            let end = swap::run_swap(
7909                spec,
7910                &swap_runtime,
7911                registry,
7912                process_liveness,
7913                snapshot,
7914                child,
7915                commands,
7916                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7917                reply,
7918            )
7919            .await;
7920            if before
7921                != lock_snapshot(snapshot)
7922                    .ok()
7923                    .map(|state| state.spawn_generation)
7924            {
7925                runtime.start_requester = swap_runtime.start_requester;
7926            }
7927            requeued.extend(end.requeue);
7928            true
7929        }
7930    }
7931}
7932
7933async fn restart_child(
7934    spec: &ModuleSpec,
7935    runtime: &SupervisorRuntimeConfig,
7936    registry: &Registry,
7937    process_liveness: &SupervisorProcessLiveness,
7938    snapshot: &SharedSnapshot,
7939    child: &mut Option<SupervisedChild>,
7940    drain_timeout: Duration,
7941) -> Result<(), SuperviseError> {
7942    // Restart cycles a running module; it must not silently start a disabled one.
7943    if !lock_snapshot(snapshot)?.enabled {
7944        return Err(SuperviseError::Disabled {
7945            module_id: spec.module_id.clone(),
7946        });
7947    }
7948    let stop_notice = begin_forwarding_drain_with_timeout(
7949        spec,
7950        runtime,
7951        registry,
7952        snapshot,
7953        None,
7954        RouteCloseReason::Restart,
7955        drain_timeout,
7956    )
7957    .await?;
7958
7959    if child.is_some() {
7960        drain_optional_child(
7961            &spec.module_id,
7962            spec.protocol,
7963            stop_notice,
7964            registry,
7965            runtime.forwarding.as_deref(),
7966            snapshot,
7967            &runtime.terminal_ring,
7968            &runtime.spawn_events,
7969            child,
7970            drain_timeout,
7971            ModuleState::Restarting,
7972            Some(true),
7973            &FailureContext::operator(),
7974        )
7975        .await?;
7976    } else {
7977        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7978            state.enabled = true;
7979            state.state = ModuleState::Restarting;
7980            clear_current_process_facts(state);
7981        })?;
7982        release_dead_registration(
7983            registry,
7984            runtime.forwarding.as_deref(),
7985            snapshot,
7986            &spec.module_id,
7987            &runtime.failure_context(false),
7988        )
7989        .await?;
7990    }
7991
7992    reset_restart_count(snapshot, &spec.module_id)?;
7993    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7994    schedule_respawn(
7995        runtime,
7996        snapshot,
7997        &spec.module_id,
7998        runtime.restart_policy.backoff,
7999        RespawnKind::Spawn,
8000        false,
8001    )
8002}
8003
8004async fn reload_child(
8005    spec: &ModuleSpec,
8006    runtime: &SupervisorRuntimeConfig,
8007    registry: &Registry,
8008    process_liveness: &SupervisorProcessLiveness,
8009    snapshot: &SharedSnapshot,
8010    child: &mut Option<SupervisedChild>,
8011) -> Result<(), SuperviseError> {
8012    // Reload cycles a running module; it must not silently start a disabled one.
8013    if !lock_snapshot(snapshot)?.enabled {
8014        return Err(SuperviseError::Disabled {
8015            module_id: spec.module_id.clone(),
8016        });
8017    }
8018    let stop_notice = begin_forwarding_drain(
8019        spec,
8020        runtime,
8021        registry,
8022        snapshot,
8023        Some(true),
8024        RouteCloseReason::Reload,
8025    )
8026    .await?;
8027
8028    if child.is_some() {
8029        drain_optional_child(
8030            &spec.module_id,
8031            spec.protocol,
8032            stop_notice,
8033            registry,
8034            runtime.forwarding.as_deref(),
8035            snapshot,
8036            &runtime.terminal_ring,
8037            &runtime.spawn_events,
8038            child,
8039            runtime.drain_timeout,
8040            ModuleState::Restarting,
8041            Some(true),
8042            &FailureContext::operator(),
8043        )
8044        .await?;
8045    } else {
8046        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8047            state.enabled = true;
8048            state.state = ModuleState::Restarting;
8049            clear_current_process_facts(state);
8050        })?;
8051        release_dead_registration(
8052            registry,
8053            runtime.forwarding.as_deref(),
8054            snapshot,
8055            &spec.module_id,
8056            &runtime.failure_context(false),
8057        )
8058        .await?;
8059    }
8060
8061    reset_restart_count(snapshot, &spec.module_id)?;
8062    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8063    schedule_respawn(
8064        runtime,
8065        snapshot,
8066        &spec.module_id,
8067        runtime.restart_policy.backoff,
8068        RespawnKind::Reload,
8069        false,
8070    )
8071}
8072
8073async fn finish_reload_child(
8074    spec: &ModuleSpec,
8075    runtime: &SupervisorRuntimeConfig,
8076    registry: &Registry,
8077    process_liveness: &SupervisorProcessLiveness,
8078    snapshot: &SharedSnapshot,
8079    child: &mut Option<SupervisedChild>,
8080) -> Result<(), SuperviseError> {
8081    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8082    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8083        Ok(next_child) => next_child,
8084        Err(err) => {
8085            return handle_reload_spawn_failure(
8086                spec,
8087                runtime,
8088                process_liveness,
8089                snapshot,
8090                child,
8091                format!("new child failed to spawn: {err}"),
8092            )
8093            .await;
8094        }
8095    };
8096    *child = Some(next_child);
8097
8098    let wait_outcome = {
8099        let active_child = child.as_mut().expect("new reload child was just stored");
8100        wait_for_registration_after_reload(
8101            registry,
8102            &spec.module_id,
8103            snapshot,
8104            active_child,
8105            REGISTRY_RELEASE_TIMEOUT,
8106        )
8107        .await?
8108    };
8109
8110    match wait_outcome {
8111        RegistrationWaitOutcome::Registered => {
8112            child
8113                .as_mut()
8114                .expect("registered reload child")
8115                .registration_deadline = None;
8116            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8117            Ok(())
8118        }
8119        RegistrationWaitOutcome::Exited(exit_report) => {
8120            if let Some(active_child) = child.as_mut() {
8121                active_child.drain_stderr(&spec.module_id).await;
8122            }
8123            // Keep the exited child's roster guard until its terminal is written.
8124            // Shutdown waits on that guard, not on the child Option used for respawn.
8125            let mut exited_child = child.take().expect("exited reload child is still stored");
8126            #[cfg(test)]
8127            if let Some(gate) = &runtime.test_reload_exit_record_gate {
8128                gate.reached.notify_one();
8129                gate.resume.notified().await;
8130            }
8131            let result = handle_reload_child_registration_failure(
8132                spec,
8133                runtime,
8134                registry,
8135                process_liveness,
8136                snapshot,
8137                child,
8138                ReloadRegistrationFailure {
8139                    exit_report: registration_failure_exit_report(exit_report),
8140                    reason: exited_child
8141                        .spawn_failure
8142                        .clone()
8143                        .unwrap_or_else(|| "new child exited before registering".to_string()),
8144                },
8145            )
8146            .await;
8147            exited_child
8148                .finish_exit(false)
8149                .await
8150                .map_err(|source| SuperviseError::Wait {
8151                    module_id: spec.module_id.clone(),
8152                    source,
8153                })?;
8154            result
8155        }
8156        RegistrationWaitOutcome::TimedOut => {
8157            let mut timed_out_child = child
8158                .take()
8159                .expect("timed-out reload child is still running");
8160            timed_out_child
8161                .start_kill()
8162                .map_err(|source| SuperviseError::Kill {
8163                    module_id: spec.module_id.clone(),
8164                    source,
8165                })?;
8166            let status = timed_out_child
8167                .wait()
8168                .await
8169                .map_err(|source| SuperviseError::Wait {
8170                    module_id: spec.module_id.clone(),
8171                    source,
8172                })?;
8173            timed_out_child.drain_stderr(&spec.module_id).await;
8174            let result = handle_reload_child_registration_failure(
8175                spec,
8176                runtime,
8177                registry,
8178                process_liveness,
8179                snapshot,
8180                child,
8181                ReloadRegistrationFailure {
8182                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8183                        snapshot,
8184                        &timed_out_child,
8185                        &status,
8186                    )),
8187                    reason: format!(
8188                        "new child did not register within {:?}",
8189                        REGISTRY_RELEASE_TIMEOUT
8190                    ),
8191                },
8192            )
8193            .await;
8194            timed_out_child
8195                .finish_exit(false)
8196                .await
8197                .map_err(|source| SuperviseError::Wait {
8198                    module_id: spec.module_id.clone(),
8199                    source,
8200                })?;
8201            result
8202        }
8203    }
8204}
8205
8206#[allow(clippy::too_many_arguments)]
8207async fn set_child_enabled(
8208    spec: &ModuleSpec,
8209    runtime: &SupervisorRuntimeConfig,
8210    registry: &Registry,
8211    process_liveness: &SupervisorProcessLiveness,
8212    snapshot: &SharedSnapshot,
8213    child: &mut Option<SupervisedChild>,
8214    enabled: bool,
8215    failure: &FailureContext,
8216) -> Result<bool, SuperviseError> {
8217    let (current_enabled, current_state, respawn_pending) = {
8218        let state = lock_snapshot(snapshot)?;
8219        (state.enabled, state.state, state.respawn_pending)
8220    };
8221    // `start` (enable on an already-enabled module) heals TERMINAL states instead
8222    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
8223    // clean (Stopped) has no live process and no other in-band recovery — the
8224    // operator's start is the explicit recovery act and resets the budget. Without
8225    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
8226    // which the 2026-07-14 aft outage proved is a trap when the failed module is
8227    // the one providing every agent's shell.
8228    let revive_terminal = enabled
8229        && current_enabled
8230        && child.is_none()
8231        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8232            || (current_state == ModuleState::Restarting && !respawn_pending));
8233    if current_enabled == enabled && !revive_terminal {
8234        return Ok(false);
8235    }
8236
8237    if enabled {
8238        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8239            state.enabled = true;
8240            state.state = ModuleState::Starting;
8241            clear_current_process_facts(state);
8242        })?;
8243        #[cfg(test)]
8244        if runtime.test_seed_stale_facts_before_enable_spawn {
8245            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8246                state.process_alive = true;
8247                state.pid = Some(41);
8248                state.spawned_at_ms = Some(42);
8249                state.spawned_from = Some(PathBuf::from("/spawned/module"));
8250                state.spawned_file_identity = Some(SpawnedFileIdentity {
8251                    device: 43,
8252                    inode: 44,
8253                    #[cfg(windows)]
8254                    windows_image: None,
8255                });
8256            })?;
8257        }
8258        release_dead_registration(
8259            registry,
8260            runtime.forwarding.as_deref(),
8261            snapshot,
8262            &spec.module_id,
8263            failure,
8264        )
8265        .await?;
8266        reset_restart_count(snapshot, &spec.module_id)?;
8267        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8268        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8269            Ok(next_child) => next_child,
8270            Err(err) => {
8271                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8272                process_liveness.untrack_if_current(&spec.module_id, snapshot);
8273                return Err(err);
8274            }
8275        };
8276        *child = Some(next_child);
8277        debug!(module_id = %spec.module_id, "supervised module enabled");
8278        Ok(true)
8279    } else {
8280        let stop_notice = begin_forwarding_drain_if_configured(
8281            spec,
8282            runtime,
8283            registry,
8284            snapshot,
8285            Some(false),
8286            RouteCloseReason::Disable,
8287        )
8288        .await?;
8289        drain_optional_child(
8290            &spec.module_id,
8291            spec.protocol,
8292            stop_notice,
8293            registry,
8294            runtime.forwarding.as_deref(),
8295            snapshot,
8296            &runtime.terminal_ring,
8297            &runtime.spawn_events,
8298            child,
8299            runtime.drain_timeout,
8300            ModuleState::Disabled,
8301            Some(false),
8302            failure,
8303        )
8304        .await?;
8305        debug!(module_id = %spec.module_id, "supervised module disabled");
8306        Ok(true)
8307    }
8308}
8309
8310#[allow(clippy::too_many_arguments)]
8311async fn on_child_exit(
8312    spec: &ModuleSpec,
8313    policy: RestartPolicy,
8314    registry: &Registry,
8315    snapshot: &SharedSnapshot,
8316    terminal_ring: &Arc<Mutex<TerminalRing>>,
8317    spawn_events: &SpawnEventFeed,
8318    roster: &ChildRoster,
8319    exit_report: ExitReport,
8320    daemon_initiated: bool,
8321) -> NextAction {
8322    // Once the daemon has begun shutting down, no exit is a crash to recover
8323    // from: the module is exiting because the daemon is going away (EOF on its
8324    // connection, or a service manager signalling the whole cgroup). Record it
8325    // as such and never schedule a respawn, which would only start a process
8326    // for the shutdown to end again.
8327    if roster.is_closed() {
8328        return on_child_exit_during_daemon_shutdown(
8329            spec,
8330            registry,
8331            snapshot,
8332            terminal_ring,
8333            spawn_events,
8334            exit_report,
8335        )
8336        .await;
8337    }
8338    // Deliberate stops the supervisor asks for (operator stop, disable,
8339    // restart, reload, swap, a health restart, a drain that runs out of budget)
8340    // takes the child out of the supervise loop and reaps it in
8341    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8342    // that reaches this point is either unrequested or a first-HELLO timeout,
8343    // which is deliberately classified as a crash even after a clean TERM exit.
8344    //
8345    // For a subc-wire module a clean exit is still a stop: those modules are
8346    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8347    // crash, and exiting 0 is a deliberate choice the module made. A
8348    // `protocol: "none"` module is a stock program we cannot change, and many
8349    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8350    // stop would leave the module down for good after any stray signal, so it
8351    // goes through the crash path instead: it spends restart budget, respawns
8352    // with the crash backoff, and ends `failed` when the budget runs out.
8353    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8354        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8355    match exit_report.kind {
8356        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8357            info!(
8358                module_id = %spec.module_id,
8359                exit_code = ?exit_report.code,
8360                exit_signal = ?exit_report.signal,
8361                "supervised module exited cleanly"
8362            );
8363            if let Err(err) = publish_after_terminal_record(
8364                &spec.module_id,
8365                || {
8366                    record_terminal(
8367                        &spec.module_id,
8368                        terminal_ring,
8369                        spawn_events,
8370                        &exit_report,
8371                        TerminalDisposition::Stopped,
8372                    )
8373                },
8374                || {
8375                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8376                        state.state = ModuleState::Stopped;
8377                        clear_current_process_facts(state);
8378                        state.last_exit = Some(exit_report.clone());
8379                    })
8380                },
8381            ) {
8382                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8383            }
8384            let registration_released = match wait_for_registration_release(
8385                registry,
8386                &spec.module_id,
8387                REGISTRY_RELEASE_TIMEOUT,
8388            )
8389            .await
8390            {
8391                Ok(()) => true,
8392                Err(err) => {
8393                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8394                    false
8395                }
8396            };
8397            NextAction::Stop {
8398                registration_released,
8399            }
8400        }
8401        ExitKind::Clean | ExitKind::Crash => {
8402            if unrequested_clean_exit_of_protocol_none {
8403                warn!(
8404                    module_id = %spec.module_id,
8405                    exit_code = ?exit_report.code,
8406                    exit_signal = ?exit_report.signal,
8407                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8408                );
8409            } else {
8410                warn!(
8411                    module_id = %spec.module_id,
8412                    exit_code = ?exit_report.code,
8413                    exit_signal = ?exit_report.signal,
8414                    "supervised module exited abnormally (crash)"
8415                );
8416            }
8417            let mut restart_schedule = None;
8418            let mut disposition = TerminalDisposition::Disabled;
8419            let mut final_state = ModuleState::Disabled;
8420            // Set only when the budget is what stopped the module, so the
8421            // terminal record says which limit was hit rather than leaving
8422            // `failed` to be read as "crashed once, badly".
8423            let mut disposition_detail = lock_snapshot(snapshot)
8424                .ok()
8425                .and_then(|mut state| state.spawn_failure.take());
8426            let now = Instant::now();
8427            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8428                if state.enabled {
8429                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8430                        // Reserving the last slot exhausts the budget for the
8431                        // replacement, not for this exit. Publish its scheduled
8432                        // recovery atomically with the reservation so connection
8433                        // cleanup sees it while terminal recording still holds
8434                        // the visible state at Running.
8435                        state.respawn_pending = true;
8436                        final_state = ModuleState::Restarting;
8437                        restart_schedule = Some(schedule);
8438                        disposition = TerminalDisposition::Restarting;
8439                    } else {
8440                        final_state = ModuleState::Failed;
8441                        disposition = TerminalDisposition::Failed;
8442                        let budget = policy.budget_exhausted_detail();
8443                        disposition_detail =
8444                            Some(disposition_detail.take().map_or_else(
8445                                || budget.clone(),
8446                                |cause| format!("{cause}; {budget}"),
8447                            ));
8448                    }
8449                } else {
8450                    disposition = TerminalDisposition::Disabled;
8451                }
8452            }) {
8453                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8454                return NextAction::Stop {
8455                    registration_released: false,
8456                };
8457            }
8458            if disposition == TerminalDisposition::Failed {
8459                // The window is in the message, not only in the fields: this line
8460                // is read in a scrollback where a bare `max_restarts=3` reads as a
8461                // lifetime cap and sends the operator looking for three crashes
8462                // that never happened together.
8463                error!(
8464                    module_id = %spec.module_id,
8465                    max_restarts = policy.max_restarts,
8466                    window_secs = policy.window.as_secs(),
8467                    "module stopped: {}",
8468                    policy.budget_exhausted_detail()
8469                );
8470            }
8471            if let Err(err) = publish_after_terminal_record(
8472                &spec.module_id,
8473                || {
8474                    record_terminal_with_detail(
8475                        &spec.module_id,
8476                        terminal_ring,
8477                        spawn_events,
8478                        &exit_report,
8479                        disposition,
8480                        disposition_detail,
8481                    )
8482                },
8483                || {
8484                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8485                        state.state = final_state;
8486                        clear_current_process_facts(state);
8487                        if final_state == ModuleState::Failed
8488                            && daemon_initiated
8489                            && !roster.is_closed()
8490                        {
8491                            state.failed_retry.schedule(&spec.module_id);
8492                        }
8493                        state.last_exit = Some(exit_report.clone());
8494                    })
8495                },
8496            ) {
8497                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8498            }
8499
8500            if let Some(schedule) = restart_schedule {
8501                NextAction::Restart {
8502                    schedule: Some(schedule),
8503                }
8504            } else {
8505                let registration_released = match wait_for_registration_release(
8506                    registry,
8507                    &spec.module_id,
8508                    REGISTRY_RELEASE_TIMEOUT,
8509                )
8510                .await
8511                {
8512                    Ok(()) => true,
8513                    Err(err) => {
8514                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8515                        false
8516                    }
8517                };
8518                NextAction::Stop {
8519                    registration_released,
8520                }
8521            }
8522        }
8523        ExitKind::DeliberateSeverance => {
8524            warn!(
8525                module_id = %spec.module_id,
8526                exit_code = ?exit_report.code,
8527                exit_signal = ?exit_report.signal,
8528                "supervised module exited after deliberate connection severance"
8529            );
8530            let should_restart = match lock_snapshot(snapshot) {
8531                Ok(state) => state.enabled,
8532                Err(err) => {
8533                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8534                    return NextAction::Stop {
8535                        registration_released: false,
8536                    };
8537                }
8538            };
8539            let final_state = if should_restart {
8540                ModuleState::Restarting
8541            } else {
8542                ModuleState::Disabled
8543            };
8544            if let Err(err) = publish_after_terminal_record(
8545                &spec.module_id,
8546                || {
8547                    record_terminal(
8548                        &spec.module_id,
8549                        terminal_ring,
8550                        spawn_events,
8551                        &exit_report,
8552                        terminal_disposition(final_state),
8553                    )
8554                },
8555                || {
8556                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8557                        state.state = final_state;
8558                        clear_current_process_facts(state);
8559                        state.last_exit = Some(exit_report.clone());
8560                        state.lifetime_restarts += 1;
8561                    })
8562                },
8563            ) {
8564                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8565                return NextAction::Stop {
8566                    registration_released: false,
8567                };
8568            }
8569
8570            if should_restart {
8571                NextAction::Restart { schedule: None }
8572            } else {
8573                let registration_released = match wait_for_registration_release(
8574                    registry,
8575                    &spec.module_id,
8576                    REGISTRY_RELEASE_TIMEOUT,
8577                )
8578                .await
8579                {
8580                    Ok(()) => true,
8581                    Err(err) => {
8582                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8583                        false
8584                    }
8585                };
8586                NextAction::Stop {
8587                    registration_released,
8588                }
8589            }
8590        }
8591    }
8592}
8593
8594async fn on_child_exit_during_daemon_shutdown(
8595    spec: &ModuleSpec,
8596    registry: &Registry,
8597    snapshot: &SharedSnapshot,
8598    terminal_ring: &Arc<Mutex<TerminalRing>>,
8599    spawn_events: &SpawnEventFeed,
8600    exit_report: ExitReport,
8601) -> NextAction {
8602    info!(
8603        module_id = %spec.module_id,
8604        exit_code = ?exit_report.code,
8605        exit_signal = ?exit_report.signal,
8606        exit_kind = ?exit_report.kind,
8607        "supervised module exited during daemon shutdown; not restarting it"
8608    );
8609    if let Err(err) = publish_after_terminal_record(
8610        &spec.module_id,
8611        || {
8612            record_terminal(
8613                &spec.module_id,
8614                terminal_ring,
8615                spawn_events,
8616                &exit_report,
8617                TerminalDisposition::DaemonShutdown,
8618            )
8619        },
8620        || {
8621            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8622                state.state = ModuleState::Stopped;
8623                clear_current_process_facts(state);
8624                state.last_exit = Some(exit_report.clone());
8625            })
8626        },
8627    ) {
8628        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8629    }
8630    let registration_released =
8631        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8632            .await
8633            .is_ok();
8634    NextAction::Stop {
8635        registration_released,
8636    }
8637}
8638
8639// Keep the status lock out of terminal I/O. Readers may observe the previous
8640// state while recording is in progress, but never a completed exit without its
8641// history. Journal errors are logged by the journal and retained in the ring;
8642// a recording panic still publishes the final state before resuming the unwind.
8643fn publish_after_terminal_record<T>(
8644    module_id: &str,
8645    record: impl FnOnce(),
8646    publish: impl FnOnce() -> T,
8647) -> T {
8648    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8649    let published = publish();
8650    if let Err(panic) = recorded {
8651        error!(
8652            module_id,
8653            "terminal recording panicked; published final state before resuming unwind"
8654        );
8655        std::panic::resume_unwind(panic);
8656    }
8657    published
8658}
8659
8660fn fail_after_wait_error(
8661    module_id: &str,
8662    snapshot: &SharedSnapshot,
8663    terminal_ring: &Arc<Mutex<TerminalRing>>,
8664    spawn_events: &SpawnEventFeed,
8665    detail: String,
8666    failure: &FailureContext,
8667) {
8668    let report = wait_error_exit_report();
8669    publish_after_terminal_record(
8670        module_id,
8671        || {
8672            record_terminal_with_detail(
8673                module_id,
8674                terminal_ring,
8675                spawn_events,
8676                &report,
8677                TerminalDisposition::Failed,
8678                Some(detail),
8679            )
8680        },
8681        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8682    );
8683}
8684
8685#[cfg(test)]
8686fn record_wait_error_terminal(
8687    module_id: &str,
8688    terminal_ring: &Arc<Mutex<TerminalRing>>,
8689    spawn_events: &SpawnEventFeed,
8690) {
8691    record_terminal(
8692        module_id,
8693        terminal_ring,
8694        spawn_events,
8695        &wait_error_exit_report(),
8696        TerminalDisposition::Failed,
8697    );
8698}
8699
8700fn record_terminal(
8701    module_id: &str,
8702    terminal_ring: &Arc<Mutex<TerminalRing>>,
8703    spawn_events: &SpawnEventFeed,
8704    exit_report: &ExitReport,
8705    disposition: TerminalDisposition,
8706) {
8707    record_terminal_with_detail(
8708        module_id,
8709        terminal_ring,
8710        spawn_events,
8711        exit_report,
8712        disposition,
8713        None,
8714    );
8715}
8716
8717/// The ring lock is held only to capture the read (see
8718/// `TerminalJournal::capture_read`), so this module's exits keep recording
8719/// while the journal files are read. Blocking: it reads files.
8720fn durable_terminal_history_of(
8721    terminal_ring: &Mutex<TerminalRing>,
8722    module_id: &str,
8723) -> subc_control::TerminalHistory {
8724    let read = terminal_ring
8725        .lock()
8726        .unwrap_or_else(|p| p.into_inner())
8727        .capture_durable_history();
8728    read.read(module_id)
8729}
8730
8731fn record_terminal_with_detail(
8732    module_id: &str,
8733    terminal_ring: &Arc<Mutex<TerminalRing>>,
8734    spawn_events: &SpawnEventFeed,
8735    exit_report: &ExitReport,
8736    disposition: TerminalDisposition,
8737    disposition_detail: Option<String>,
8738) {
8739    #[cfg(test)]
8740    terminal_history_tests::before_terminal_record(spawn_events);
8741    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8742    let record = TerminalRecord {
8743        exit_code: exit_report.code,
8744        exit_signal: exit_report.signal,
8745        at_ms: exit_report.at_ms,
8746        disposition,
8747        exit_kind: exit_report.kind.into(),
8748        disposition_detail,
8749    };
8750    terminal_ring
8751        .lock()
8752        .unwrap_or_else(|poisoned| poisoned.into_inner())
8753        .record_exit(module_id, record);
8754}
8755
8756fn untrack_if_registration_released(
8757    process_liveness: &SupervisorProcessLiveness,
8758    registry: &Registry,
8759    module_id: &str,
8760    snapshot: &SharedSnapshot,
8761) {
8762    match registry.get_module(module_id) {
8763        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8764        Ok(Some(_)) => {}
8765        Err(err) => {
8766            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8767        }
8768    }
8769}
8770
8771/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8772/// then apply the module's configured entries minus daemon-private capture keys.
8773///
8774/// Separated from `spawn_child` only so it can be asserted without spawning a
8775/// process — a duplicate of this logic in a test would pass while the real one
8776/// drifted, which is the defect class this function exists to avoid.
8777/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8778/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8779/// either and the argument would stop a stock binary from starting at all.
8780/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8781///
8782/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8783/// through [`apply_wire_spawn_args_for_role`].
8784#[cfg(test)]
8785fn apply_wire_spawn_args(
8786    command: &mut Command,
8787    spec: &ModuleSpec,
8788    connection_file_path: Option<&std::path::Path>,
8789    handle: Option<&SupervisorHandle>,
8790) -> Result<Option<NonceHandoff>, SuperviseError> {
8791    apply_wire_spawn_args_for_role(
8792        command,
8793        spec,
8794        connection_file_path,
8795        handle,
8796        SpawnRole::Plain,
8797    )
8798}
8799
8800/// The read end of a spawn's launch-nonce pipe, prepared by
8801/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8802/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8803/// server before spawn and authenticates its client after the child exists.
8804#[cfg(unix)]
8805type NonceHandoff = subc_os::LaunchNonceHandoff;
8806#[cfg(windows)]
8807type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8808#[cfg(not(any(unix, windows)))]
8809type NonceHandoff = std::convert::Infallible;
8810
8811/// Create the launch nonce that a new module process must present when it
8812/// registers or opens a consumer connection, including a replacement process
8813/// started alongside a still-serving module.
8814///
8815/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8816/// a separate candidate token so the still-serving incumbent and its consumers
8817/// keep their nonce. Both records are installed before the process exists, so
8818/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8819///
8820/// On Unix the nonce is delivered only through a pipe. It is written into
8821/// a pipe whose read end the child gets as descriptor 3, named by
8822/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8823/// process of the same user cannot read it with `ps eww`. That handoff is
8824/// returned rather than installed here, because installing it replaces
8825/// whatever the child has at descriptor 3 and so must be the last pre-exec
8826/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8827/// the Linux callback that writes the process into its cgroup. Only after that
8828/// does `handoff.install_last` register the descriptor-3 replacement, so it
8829/// cannot close a descriptor the cgroup callback still needs.
8830/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8831fn apply_wire_spawn_args_for_role(
8832    command: &mut Command,
8833    spec: &ModuleSpec,
8834    connection_file_path: Option<&std::path::Path>,
8835    handle: Option<&SupervisorHandle>,
8836    role: SpawnRole,
8837) -> Result<Option<NonceHandoff>, SuperviseError> {
8838    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8839    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8840    // included: a daemon started from a module's process tree inherits it,
8841    // and passing it on would point the child at a descriptor it does not
8842    // have.
8843    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8844    #[cfg(windows)]
8845    {
8846        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8847        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8848    }
8849    // Remove inherited or configured copies too: withholding must mean absent.
8850    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8851    if spec.protocol == ModuleProtocol::None {
8852        return Ok(None);
8853    }
8854    if let Some(connection_file_path) = connection_file_path {
8855        command.arg(SUBC_ARG).arg(connection_file_path);
8856    }
8857
8858    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8859    // route.open attestation. Reserved modules additionally use the same nonce
8860    // for HELLO id-squatting protection. A respawn rotates both records.
8861    let nonce = generate_launch_nonce()?;
8862    if let Some(handle) = handle {
8863        match role {
8864            SpawnRole::Plain => {
8865                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8866                if spec.reserved {
8867                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8868                }
8869            }
8870            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8871        }
8872    }
8873    #[cfg(unix)]
8874    let handoff = {
8875        let handoff =
8876            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8877                program: spec.program.clone(),
8878                source,
8879                cgroup_path: None,
8880            })?;
8881        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8882        Some(handoff)
8883    };
8884    #[cfg(windows)]
8885    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8886    #[cfg(not(any(unix, windows)))]
8887    let handoff = None;
8888    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8889    // until the daemon's per-spawn delivery records show that every running
8890    // Windows module has read its secret from the named pipe.
8891    #[cfg(not(unix))]
8892    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8893    Ok(handoff)
8894}
8895
8896fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8897    command.env_remove(CK_LOG_ENV);
8898    // The spawn role is the supervisor's to set, and only on a swap candidate
8899    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8900    // it, is what makes it absent on a plain spawn: the daemon's own
8901    // environment could carry it, and so could a spec built outside daemon
8902    // config (config refuses it as an `env` key). A module reading it on a
8903    // plain restart would pick the long swap budget and leave callers waiting.
8904    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8905    for (key, value) in &spec.env {
8906        // cortexkit-log currently exposes retention only as a Rust struct, not
8907        // environment names. These values are daemon-private sink metadata and
8908        // must never become a public child-process contract by being inherited.
8909        if matches!(
8910            key.as_str(),
8911            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8912        ) || key == SUBC_SPAWN_ROLE_ENV
8913        {
8914            continue;
8915        }
8916        command.env(key, value);
8917    }
8918}
8919
8920#[cfg(windows)]
8921fn prepare_windows_nonce_pipe(
8922    command: &mut Command,
8923    spec: &ModuleSpec,
8924    nonce: &str,
8925) -> Option<NonceHandoff> {
8926    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8927        Ok(handoff) => {
8928            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8929            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8930            // still register until every module reads the pipe. A reader without
8931            // this variable set to env returns the pipe error instead of using
8932            // an inherited or configured environment copy.
8933            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8934            Some(handoff)
8935        }
8936        Err(error) => {
8937            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8938            None
8939        }
8940    }
8941}
8942
8943#[cfg(windows)]
8944fn start_windows_nonce_pipe(
8945    handoff: Option<NonceHandoff>,
8946    pid: u32,
8947    spec: &ModuleSpec,
8948) -> Option<subc_os::LaunchNoncePipeDelivery> {
8949    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8950    // same maximum wait used for registration during a reload. Older modules
8951    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8952    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8953        Ok(delivery) => Some(delivery),
8954        Err(error) => {
8955            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8956            None
8957        }
8958    })
8959}
8960
8961fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8962    #[cfg(windows)]
8963    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8964        return Some(
8965            if snapshot
8966                .pipe_consumed
8967                .as_ref()
8968                .is_some_and(|consumed| consumed.get().is_some())
8969            {
8970                "pipe"
8971            } else {
8972                "env"
8973            }
8974            .to_owned(),
8975        );
8976    }
8977    let _ = snapshot;
8978    None
8979}
8980
8981/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8982/// of a blue/green swap.
8983#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8984enum SpawnRole {
8985    Plain,
8986    SwapCandidate,
8987}
8988
8989/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8990/// `apply_child_env` has already removed the variable for every spawn.
8991fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8992    if role == SpawnRole::SwapCandidate {
8993        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8994    }
8995}
8996
8997fn previous_instance_error(
8998    spec: &ModuleSpec,
8999    pid: u32,
9000    reason: crate::live_children::RetainedReason,
9001) -> SuperviseError {
9002    SuperviseError::Spawn {
9003        program: spec.program.clone(),
9004        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
9005        cgroup_path: None,
9006    }
9007}
9008
9009fn carries_containment(configured: bool, has_container: bool) -> bool {
9010    configured || has_container
9011}
9012
9013fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
9014    if !carries_containment(
9015        spec.carry_containment,
9016        containment::current(&roster.containment).is_some(),
9017    ) {
9018        return None;
9019    }
9020    Some(SuperviseError::Spawn {
9021        program: spec.program.clone(),
9022        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9023        cgroup_path: None,
9024    })
9025}
9026
9027fn spawn_child(
9028    spec: &ModuleSpec,
9029    connection_file_path: Option<&std::path::Path>,
9030    handle: Option<&SupervisorHandle>,
9031    ring: &Arc<Mutex<StderrRing>>,
9032    capture_logs_dir: Option<&std::path::Path>,
9033    roster: &ChildRoster,
9034    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9035) -> Result<SupervisedChild, SuperviseError> {
9036    #[cfg(all(test, unix))]
9037    if let Some(source) = containment_tests::join_failure(spec, roster) {
9038        containment::terminate_blocking(&roster.containment);
9039        return Err(SuperviseError::Spawn {
9040            program: spec.program.clone(),
9041            source,
9042            cgroup_path: None,
9043        });
9044    }
9045    let result = spawn_child_in_slot(
9046        spec,
9047        connection_file_path,
9048        handle,
9049        ring,
9050        capture_logs_dir,
9051        roster,
9052        #[cfg(target_os = "linux")]
9053        cgroup_placement,
9054        SpawnRole::Plain,
9055        false,
9056    );
9057    if result.is_err() {
9058        if roster.admission_pending(&spec.module_id) {
9059            containment::terminate_deferred(&roster.containment);
9060        } else {
9061            containment::terminate_blocking(&roster.containment);
9062        }
9063    }
9064    result
9065}
9066
9067/// Spawn one process of `spec` into a slot.
9068///
9069/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
9070/// A swap candidate needs a different cgroup from the process it is replacing,
9071/// which is still alive: in the same cgroup the two would be one kill domain,
9072/// and killing a failed candidate could take the incumbent with it.
9073///
9074/// The stderr capture file is `<module_id>.stderr.log` for every process of
9075/// the module, whichever slot it is in, because that is the one file
9076/// `ck module logs` reads. During a swap's overlap both processes append to it;
9077/// the daemon writes whole lines, so the two interleave by line, which is also
9078/// the merged view an operator wants while a swap runs.
9079#[allow(clippy::too_many_arguments)]
9080fn spawn_child_in_slot(
9081    spec: &ModuleSpec,
9082    connection_file_path: Option<&std::path::Path>,
9083    handle: Option<&SupervisorHandle>,
9084    ring: &Arc<Mutex<StderrRing>>,
9085    capture_logs_dir: Option<&std::path::Path>,
9086    roster: &ChildRoster,
9087    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9088    role: SpawnRole,
9089    alternate_slot: bool,
9090) -> Result<SupervisedChild, SuperviseError> {
9091    if let Some(error) = recovery_block_error(spec, roster) {
9092        return Err(error);
9093    }
9094    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9095        return Err(previous_instance_error(spec, pid, reason));
9096    }
9097    if roster.is_closed() {
9098        return Err(SuperviseError::Spawn {
9099            program: spec.program.clone(),
9100            source: io::Error::other("the daemon is shutting down; not starting a new process"),
9101            cgroup_path: None,
9102        });
9103    }
9104    let carried = containment::current(&roster.containment);
9105    let carrying = carries_containment(spec.carry_containment, carried.is_some());
9106    if role != SpawnRole::Plain && carrying {
9107        return Err(SuperviseError::InvalidSpec {
9108            reason: "a carried container cannot overlap a swap candidate".to_owned(),
9109        });
9110    }
9111    #[cfg(unix)]
9112    let launch_identity = if carrying && carried.is_none() {
9113        Some(
9114            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9115                program: spec.program.clone(),
9116                source,
9117                cgroup_path: None,
9118            })?,
9119        )
9120    } else {
9121        None
9122    };
9123    #[cfg(target_os = "linux")]
9124    let cgroup_placement = match &carried {
9125        Some(container) => container.cgroup().map(|(placement, _)| placement),
9126        None => cgroup_placement,
9127    };
9128    #[cfg(unix)]
9129    let group_target = if let Some(container) = carried
9130        .as_ref()
9131        .filter(|container| container.group().is_some())
9132    {
9133        containment::GroupTarget::Pinned(Arc::clone(container))
9134    } else {
9135        #[cfg(target_os = "linux")]
9136        let cgroup_container = carrying && cgroup_placement.is_some();
9137        #[cfg(not(target_os = "linux"))]
9138        let cgroup_container = false;
9139        if cgroup_container {
9140            containment::GroupTarget::NoGroup
9141        } else {
9142            containment::GroupTarget::OwnLeader
9143        }
9144    };
9145    #[cfg(target_os = "linux")]
9146    let cgroup_name = {
9147        // Slot names alone are not kill domains: a retired incumbent may still
9148        // be draining when a later enable/restart spawns into the same slot.
9149        // Decimal entropy keeps the suffix unambiguous; Placement performs
9150        // the module-id escaping and constructs the filesystem path.
9151        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9152            name.to_owned()
9153        } else if cgroup_placement.is_none() {
9154            swap::cgroup_name(&spec.module_id, alternate_slot)
9155        } else {
9156            let nonce = generate_launch_nonce()?;
9157            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9158            // Leave room for byte escaping and the suffix under NAME_MAX. The
9159            // label is only for humans; the nonce identifies the kill domain.
9160            let mut end = spec.module_id.len().min(64);
9161            while !spec.module_id.is_char_boundary(end) {
9162                end -= 1;
9163            }
9164            format!(
9165                "{}_{suffix}",
9166                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9167            )
9168        }
9169    };
9170    #[cfg(not(target_os = "linux"))]
9171    let _ = alternate_slot;
9172    #[cfg(target_os = "macos")]
9173    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9174    #[cfg(windows)]
9175    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9176        program: spec.program.clone(),
9177        source,
9178        cgroup_path: None,
9179    })?;
9180    #[cfg(windows)]
9181    let mut command = Command::new(image_capture.path());
9182    #[cfg(not(any(target_os = "macos", windows)))]
9183    let mut command = Command::new(&spec.program);
9184    command.args(&spec.args);
9185    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
9186    // that is the whole of the intent, so remove that one key rather than the
9187    // environment.
9188    //
9189    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
9190    // and took the POSIX environment with it. Modules spawned that way had no
9191    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
9192    // logging:
9193    //
9194    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
9195    //     both unset it fell back to the temp dir alone and `ck` could not find
9196    //     a daemon running on the same machine from inside any module's process
9197    //     tree — reporting a path the file has never lived at, which reads as
9198    //     "the daemon did not write its file".
9199    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
9200    //     the RELATIVE `.local/share`, so a module deriving its own store path
9201    //     resolved it against its own CWD. That is the store-fragmentation
9202    //     defect the daemon already refuses in config (`parse_doc` rejects a
9203    //     relative `storage.data_home`) arriving by derivation instead.
9204    //   * anything a module spawns inherited it: git without ~/.gitconfig,
9205    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
9206    //     quietly rather than erroring.
9207    //
9208    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
9209    // offered one candidate under /tmp while the file sat in /run/user/1000.
9210    //
9211    // A configured module is unaffected either way: `module_spec()` puts the
9212    // resolved CK_LOG into `spec.env`, which is applied below and therefore
9213    // wins over anything ambient.
9214    apply_child_env(&mut command, spec);
9215    apply_spawn_role(&mut command, role);
9216    let nonce_handoff =
9217        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9218
9219    #[cfg(target_os = "linux")]
9220    let cgroup_path = cgroup_placement
9221        .map(|placement| {
9222            if let Some(container) = &carried {
9223                let path = container
9224                    .cgroup_path()
9225                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9226                placement.existing_module_path(path)?.ok_or_else(|| {
9227                    io::Error::new(
9228                        io::ErrorKind::NotFound,
9229                        "carried cgroup disappeared before replacement",
9230                    )
9231                })
9232            } else {
9233                if carrying {
9234                    containment::prepare_cgroup_path(placement, &cgroup_name)
9235                } else {
9236                    placement.module_path(&cgroup_name)
9237                }
9238            }
9239        })
9240        .transpose()
9241        .map_err(|source| SuperviseError::Cgroup {
9242            module_id: spec.module_id.clone(),
9243            source,
9244        })?;
9245    #[cfg(not(target_os = "linux"))]
9246    let cgroup_path: Option<PathBuf> = None;
9247    #[cfg(unix)]
9248    let mut admission_gate = if carrying {
9249        Some(
9250            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9251                SuperviseError::Spawn {
9252                    program: spec.program.clone(),
9253                    source,
9254                    cgroup_path: cgroup_path.clone(),
9255                }
9256            })?,
9257        )
9258    } else {
9259        None
9260    };
9261    #[cfg(target_os = "macos")]
9262    exec_ack.install(command.as_std_mut());
9263    #[cfg(target_os = "linux")]
9264    if let Some(path) = &cgroup_path {
9265        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9266            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9267                remove_module_cgroup(placement, &cgroup_name);
9268            }
9269            return Err(error);
9270        }
9271    }
9272
9273    let output_sink = if let Some(logs_dir) = capture_logs_dir {
9274        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9275        match ChildOutputSink::open(&path, capture_retention(spec)) {
9276            Ok(sink) => sink,
9277            Err(error) => {
9278                warn!(
9279                    module_id = %spec.module_id,
9280                    path = %path.display(),
9281                    error = %error,
9282                    "could not open child output capture file; forwarding to stderr"
9283                );
9284                ChildOutputSink::Stderr
9285            }
9286        }
9287    } else {
9288        ChildOutputSink::Stderr
9289    };
9290
9291    command.stdout(Stdio::piped());
9292    command.stderr(Stdio::piped());
9293    command.kill_on_drop(!carrying);
9294    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
9295    // before exec). In the daemon's group, a service manager that kills the
9296    // job's process group when the daemon exits (launchd's default) killed
9297    // every module at the same moment its control connection closed, so no
9298    // module ever ran its EOF teardown on a daemon stop. Outside that group a
9299    // module is reached only by the daemon: the EOF it sees when its
9300    // connection closes, and the bounded stop in `child_roster` for anything
9301    // still running after that. On Linux this composes with the cgroup
9302    // placement above: that is a pre_exec write to cgroup.procs, std performs
9303    // setpgid in the child before running pre_exec callbacks, and the two
9304    // change independent process attributes.
9305    //
9306    // stdin is /dev/null because a process outside the terminal's foreground
9307    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
9308    // by hand would otherwise hand down. Under a service manager stdin is
9309    // already /dev/null.
9310    #[cfg(unix)]
9311    command.process_group(
9312        carried
9313            .as_ref()
9314            .map(|container| container.verified_join_group())
9315            .transpose()
9316            .map_err(|source| SuperviseError::Spawn {
9317                program: spec.program.clone(),
9318                source,
9319                cgroup_path: cgroup_path.clone(),
9320            })?
9321            .flatten()
9322            .unwrap_or(0) as i32,
9323    );
9324    command.stdin(Stdio::null());
9325    #[cfg(unix)]
9326    if let Some(gate) = &mut admission_gate {
9327        gate.install_placement(&mut command);
9328    }
9329    // The LAST pre-exec step, after the cgroup placement above: installing the
9330    // nonce at descriptor 3 replaces whatever the child had there, which could
9331    // be the descriptor an earlier step writes through.
9332    #[cfg(unix)]
9333    if let Some(handoff) = nonce_handoff {
9334        handoff.install_last(command.as_std_mut());
9335    }
9336    #[cfg(not(any(unix, windows)))]
9337    let _ = nonce_handoff;
9338
9339    // Containment, step 1 of 3 (issue #109): create the child suspended so it
9340    // cannot run a single instruction -- and therefore cannot spawn a
9341    // grandchild -- before it is in the job. See `contain_spawned_child` for the
9342    // other two steps and why the window matters.
9343    #[cfg(windows)]
9344    {
9345        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9346            program: spec.program.clone(),
9347            source,
9348            cgroup_path: None,
9349        })?;
9350        subc_jobobject::suspend_on_create_async(&mut command);
9351    }
9352    // Timestamp before the spawn call: on Unix the child can serve before that
9353    // call even returns. This conservative bound never labels an older child
9354    // as a replacement for a request received during process creation.
9355    let spawned_at = Instant::now();
9356    #[cfg(all(test, unix))]
9357    containment_tests::record_spawn_attempt(spec, &mut command);
9358    #[cfg(unix)]
9359    let spawned = match admission_gate {
9360        Some(gate) => gate
9361            .spawn(
9362                command,
9363                spec,
9364                roster,
9365                launch_identity,
9366                #[cfg(target_os = "linux")]
9367                cgroup_placement
9368                    .zip(cgroup_path.as_ref())
9369                    .map(|(placement, path)| {
9370                        (placement.clone(), cgroup_name.clone(), path.clone())
9371                    }),
9372            )
9373            .map(|(child, admitted)| (child, Some(admitted))),
9374        None => command.spawn().map(|child| (child, None)),
9375    };
9376    #[cfg(not(unix))]
9377    let spawned = command.spawn();
9378    let spawned = match spawned {
9379        Ok(child) => child,
9380        Err(source) => {
9381            if carried
9382                .as_ref()
9383                .and_then(|container| container.group())
9384                .is_some()
9385            {
9386                containment::terminate_blocking(&roster.containment);
9387            }
9388            #[cfg(target_os = "linux")]
9389            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9390                remove_module_cgroup(placement, &cgroup_name);
9391            }
9392            return Err(SuperviseError::Spawn {
9393                program: spec.program.clone(),
9394                source,
9395                cgroup_path,
9396            });
9397        }
9398    };
9399    #[cfg(unix)]
9400    let (mut child, mut admitted) = spawned;
9401    #[cfg(unix)]
9402    let group_target = admitted
9403        .as_ref()
9404        .map(|attempt| attempt.group_target())
9405        .unwrap_or(group_target);
9406    #[cfg(not(unix))]
9407    let mut child = spawned;
9408    #[cfg(test)]
9409    spawn_record_tests::after_create(&child, spec);
9410    // The parent must close its writer now: the acknowledgement pipe reports EOF
9411    // only when every writer is gone, and the child's copy closes when the
9412    // trampoline replaces itself with the module. Command holds only an integer
9413    // in its pre_exec callback, not another writer.
9414    #[cfg(target_os = "macos")]
9415    drop(exec_ack);
9416
9417    // Bind while the child is still suspended, before job assignment resumes it.
9418    // Keeping the executable handle open across CreateProcessW denies rename
9419    // and deletion, so the file cannot be exchanged during spawn. Bind that
9420    // file ID to the process handle returned by spawn, never a reopened PID.
9421    #[cfg(windows)]
9422    let windows_image = match image_capture.bind_tokio(&child) {
9423        Ok(image) => Arc::new(image),
9424        Err(source) => {
9425            kill_and_reap_rejected_child(&mut child);
9426            return Err(SuperviseError::Spawn {
9427                program: spec.program.clone(),
9428                source,
9429                cgroup_path: None,
9430            });
9431        }
9432    };
9433    // Assign the suspended child to its module job; resume only after its
9434    // process identity has been durably recorded for orphan recovery.
9435    #[cfg(windows)]
9436    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9437    let spawned_at_ms = unix_ms_now();
9438    #[cfg(not(windows))]
9439    let spawned_from = spec.program.clone();
9440    #[cfg(windows)]
9441    let spawned_from = windows_image.path().to_path_buf();
9442    #[cfg(not(windows))]
9443    let spawned_file_identity = spawned_file_identity(&spawned_from);
9444    #[cfg(windows)]
9445    let spawned_file_identity = Some(SpawnedFileIdentity {
9446        device: 0,
9447        inode: 0,
9448        windows_image: Some(Arc::clone(&windows_image)),
9449    });
9450    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9451        program: spec.program.clone(),
9452        source: io::Error::other("spawned child exposed no live pid"),
9453        cgroup_path: cgroup_path.clone(),
9454    })?;
9455    #[cfg(windows)]
9456    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9457    #[cfg(not(windows))]
9458    let process_start_time = admitted
9459        .as_ref()
9460        .map(|admitted| admitted.birth)
9461        .or_else(|| crate::provenance::process_start_time(pid));
9462    #[cfg(windows)]
9463    let process_start_time = Some(windows_image.start_time());
9464    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9465    #[cfg(all(test, target_os = "macos"))]
9466    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9467    // Record the committed kernel image, not the configured pathname: PATH
9468    // lookup and shebang interpretation can select a different executable.
9469    // Linux's posix_spawn can return before that image is installed, so the
9470    // observer also waits for the kernel's first-exec flag to clear.
9471    #[cfg(not(windows))]
9472    let recorded_image = observe_spawned_image(pid);
9473    // spawn() confirms only the first exec, into the trampoline. Never persist
9474    // the trampoline image; the asynchronous acknowledgement publishes the
9475    // module image once the trampoline has replaced itself with the module.
9476    #[cfg(target_os = "macos")]
9477    let recorded_image = if privacy_exec.is_some() {
9478        None
9479    } else {
9480        recorded_image
9481    };
9482    #[cfg(target_os = "linux")]
9483    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9484    #[cfg(not(target_os = "linux"))]
9485    let recorded_cgroup_name = None;
9486    #[cfg(unix)]
9487    let preadmitted = admitted
9488        .as_mut()
9489        .and_then(|admitted| admitted.leader.take());
9490    #[cfg(not(unix))]
9491    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9492    let roster_guard = if let Some(guard) = preadmitted {
9493        Ok(guard)
9494    } else {
9495        roster.admit(
9496            spec.module_id.clone(),
9497            pid,
9498            spec.protocol,
9499            process_start_time,
9500            crate::child_roster::RecordedIdentity {
9501                container: None,
9502                #[cfg(unix)]
9503                carried: None,
9504                #[cfg(not(windows))]
9505                start_time: recorded_image.map(|image| image.start_time),
9506                #[cfg(windows)]
9507                start_time: process_start_time,
9508                #[cfg(not(windows))]
9509                executable: recorded_image
9510                    .and_then(|image| image.executable)
9511                    .map(crate::live_children::ExecutableIdentity::from),
9512                #[cfg(windows)]
9513                executable: Some(crate::live_children::ExecutableIdentity::from(
9514                    windows_image.identity(),
9515                )),
9516                cgroup_name: recorded_cgroup_name,
9517                #[cfg(target_os = "linux")]
9518                cgroup_placement: cgroup_placement.cloned(),
9519            },
9520        )
9521    };
9522    let roster_guard = match roster_guard {
9523        Ok(guard) => guard,
9524        Err(failure) => {
9525            return failure.with_cleanup(|source| {
9526                #[cfg(windows)]
9527                if let Some(job) = &job {
9528                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9529                }
9530                #[cfg(target_os = "linux")]
9531                kill_module_cgroup(cgroup_placement, &cgroup_name);
9532                #[cfg(unix)]
9533                let rejected_group = group_target.clone();
9534                kill_and_reap_rejected_child(
9535                    &mut child,
9536                    #[cfg(unix)]
9537                    rejected_group,
9538                );
9539                #[cfg(target_os = "linux")]
9540                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9541                    remove_module_cgroup(placement, &cgroup_name);
9542                }
9543                Err(SuperviseError::Spawn {
9544                    program: spec.program.clone(),
9545                    source: io::Error::other(format!(
9546                        "could not durably record module child: {source}"
9547                    )),
9548                    cgroup_path,
9549                })
9550            });
9551        }
9552    };
9553    #[cfg(unix)]
9554    if admitted.is_some() {
9555        if let Some(image) = recorded_image {
9556            if let Err(source) = roster_guard.confirm_executable(image) {
9557                kill_and_reap_rejected_child(&mut child, group_target.clone());
9558                return Err(SuperviseError::Spawn {
9559                    program: spec.program.clone(),
9560                    source,
9561                    cgroup_path,
9562                });
9563            }
9564        }
9565    }
9566    // The pre-spawn check can pass just before shutdown begins. Ownership is
9567    // published at the start of admission, even while a record write is queued.
9568    // If shutdown already saw an empty roster before that publication, this
9569    // check sees the roster closed and ends the child before starting a module.
9570    if roster.is_closed() {
9571        if carrying {
9572            #[cfg(unix)]
9573            let rejected_group = admitted
9574                .as_ref()
9575                .map(|admitted| admitted.rejection_target())
9576                .unwrap_or_else(|| group_target.clone());
9577            #[cfg(windows)]
9578            if let Some(job) = &job {
9579                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9580            }
9581            kill_and_reap_rejected_child(
9582                &mut child,
9583                #[cfg(unix)]
9584                rejected_group,
9585            );
9586            #[cfg(all(test, unix))]
9587            let _ =
9588                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9589            drop(roster_guard);
9590            return Err(SuperviseError::Spawn {
9591                program: spec.program.clone(),
9592                source: io::Error::other(
9593                    "the daemon began shutting down during carried admission; ended the container",
9594                ),
9595                cgroup_path,
9596            });
9597        }
9598        // This child was never admitted, so there is no module protocol shutdown to wait for.
9599        #[cfg(target_os = "linux")]
9600        kill_module_cgroup(cgroup_placement, &cgroup_name);
9601        if let Err(error) = child.start_kill() {
9602            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9603        }
9604        #[cfg(target_os = "linux")]
9605        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9606            // This spawn was never admitted, so shutdown has no roster entry
9607            // to await. Do not detach its cleanup: the runtime could exit
9608            // before that task reaps the rejected child and removes its group.
9609            while matches!(child.try_wait(), Ok(None)) {
9610                std::thread::yield_now();
9611            }
9612            if matches!(
9613                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9614                subc_cgroup::KillOutcome::Killed
9615            ) {
9616                if let Ok(path) = placement.module_path(&cgroup_name) {
9617                    while std::fs::read_to_string(path.join("cgroup.events"))
9618                        .ok()
9619                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9620                    {
9621                        std::thread::yield_now();
9622                    }
9623                }
9624            }
9625            remove_module_cgroup(placement, &cgroup_name);
9626        }
9627        drop(roster_guard);
9628        return Err(SuperviseError::Spawn {
9629            program: spec.program.clone(),
9630            source: io::Error::other(
9631                "the daemon began shutting down while this process was starting; ended it",
9632            ),
9633            cgroup_path,
9634        });
9635    }
9636
9637    let stdout_pump = match child.stdout.take() {
9638        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9639        None => {
9640            warn!(
9641                module_id = %spec.module_id,
9642                "spawned child exposed no stdout pipe; file capture will be incomplete"
9643            );
9644            None
9645        }
9646    };
9647    let stderr_pump = match child.stderr.take() {
9648        Some(stderr) => {
9649            let generation = ring
9650                .lock()
9651                .unwrap_or_else(|poisoned| poisoned.into_inner())
9652                .begin_process();
9653            Some(StderrPump {
9654                task: tokio::spawn(pump_stderr_to(
9655                    stderr,
9656                    Arc::clone(ring),
9657                    generation,
9658                    output_sink,
9659                )),
9660                generation,
9661            })
9662        }
9663        None => {
9664            // Spawning succeeded but the pipe did not materialise. Recording it as
9665            // uncaptured keeps the tail honest: the alternative is an empty tail
9666            // that reads as a module which printed nothing.
9667            ring.lock()
9668                .unwrap_or_else(|poisoned| poisoned.into_inner())
9669                .mark_not_captured("stderr pipe was not available on spawn");
9670            warn!(
9671                module_id = %spec.module_id,
9672                "spawned child exposed no stderr pipe; tail will be unavailable"
9673            );
9674            None
9675        }
9676    };
9677
9678    let exec = Arc::new(crate::activity::ExecOutcome::new());
9679    #[cfg(target_os = "macos")]
9680    let pending_exec = privacy_exec.is_some();
9681    #[cfg(not(target_os = "macos"))]
9682    let pending_exec = false;
9683    if !pending_exec {
9684        exec.finish(crate::activity::ExecState::CommittedExec);
9685    }
9686    #[cfg(windows)]
9687    let subtree_target = job
9688        .as_ref()
9689        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9690        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9691    #[cfg(target_os = "linux")]
9692    let subtree_target = cgroup_path
9693        .map(crate::subtree_resources::Target::Cgroup)
9694        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9695    #[cfg(not(any(target_os = "linux", windows)))]
9696    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9697    let mut supervised = SupervisedChild {
9698        subtree_target,
9699        child: containment::ProcessChild::plain(child, pid),
9700        exec: crate::activity::ExecOwner(exec),
9701        #[cfg(windows)]
9702        nonce_delivery,
9703        protocol: spec.protocol,
9704        #[cfg(target_os = "linux")]
9705        module_id: cgroup_name,
9706        #[cfg(target_os = "linux")]
9707        cgroup_placement: cgroup_placement.cloned(),
9708        #[cfg(windows)]
9709        job,
9710        stdout_pump,
9711        stderr_pump,
9712        stderr_ring: Arc::clone(ring),
9713        spawned_at,
9714        registration_deadline: None,
9715        registration_watch: None,
9716        spawned_at_ms,
9717        spawned_from,
9718        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9719        spawned_file_identity,
9720        process_start_time,
9721        process_identity,
9722        pid,
9723        roster_guard: Some(roster_guard),
9724        #[cfg(target_os = "macos")]
9725        privacy_exec,
9726        #[cfg(target_os = "macos")]
9727        report_ready: Arc::new(OnceLock::new()),
9728        spawn_failure: None,
9729    };
9730    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9731        supervised.child.reject(
9732            #[cfg(unix)]
9733            group_target,
9734        );
9735        #[cfg(all(test, unix))]
9736        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9737        return Err(error);
9738    }
9739    #[cfg(unix)]
9740    if let Some(admitted) = admitted.take() {
9741        admitted.disarm();
9742    }
9743    #[cfg(windows)]
9744    if let Err(error) = resume_suspended_child(pid, spec) {
9745        containment::terminate_blocking(&roster.containment);
9746        let _ = supervised.child.start_kill();
9747        return Err(error);
9748    }
9749    Ok(supervised)
9750}
9751
9752#[cfg(test)]
9753mod spawn_record_tests {
9754    use super::*;
9755
9756    thread_local! {
9757        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9758    }
9759
9760    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9761        LAST_PID.set(child.id());
9762        #[cfg(windows)]
9763        if let Some((_, address)) = _spec
9764            .env
9765            .iter()
9766            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9767        {
9768            use std::io::{Read, Write};
9769            let mut stream = std::net::TcpStream::connect(address).unwrap();
9770            stream
9771                .write_all(&child.id().unwrap().to_ne_bytes())
9772                .unwrap();
9773            let mut release = [0];
9774            stream.read_exact(&mut release).unwrap();
9775        }
9776    }
9777
9778    #[cfg(windows)]
9779    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9780        if spec
9781            .env
9782            .iter()
9783            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9784        {
9785            Err(io::Error::other("injected module job assignment failure"))
9786        } else {
9787            Ok(())
9788        }
9789    }
9790
9791    fn spec() -> ModuleSpec {
9792        ModuleSpec {
9793            module_id: "spawn-record-worker".into(),
9794            #[cfg(unix)]
9795            program: "/bin/sleep".into(),
9796            #[cfg(windows)]
9797            program: std::env::current_exe().unwrap(),
9798            #[cfg(unix)]
9799            args: vec!["60".into()],
9800            #[cfg(windows)]
9801            args: vec![
9802                "--exact".into(),
9803                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9804                "--nocapture".into(),
9805            ],
9806            env: vec![],
9807            reserved: false,
9808            reserved_prefixes: vec![],
9809            protocol: ModuleProtocol::None,
9810            overlap: Default::default(),
9811            carry_containment: false,
9812        }
9813    }
9814
9815    #[tokio::test]
9816    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9817        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9818        let supervisor =
9819            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9820                .with_live_children_record(home.join("live-children.json"));
9821        crate::live_children::fail_next_file_sync();
9822        let result = supervisor.spawn(spec());
9823        let failed = result.is_err();
9824        if let Ok(module) = result {
9825            module.stop().await.unwrap();
9826        }
9827        assert!(failed, "record failure must fail spawn");
9828        let pid = LAST_PID.get().expect("the real OS child was created");
9829        assert!(
9830            subc_os::Process::open(pid).unwrap().is_none(),
9831            "child {pid} survived rejected admission or was not reaped"
9832        );
9833    }
9834
9835    #[cfg(target_os = "macos")]
9836    #[test]
9837    #[ignore = "manual on-device latency measurement"]
9838    fn durable_record_write_latency_50() {
9839        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9840        let path = home.join("live-children.json");
9841        let children = vec![crate::live_children::LiveChild {
9842            module_id: "latency-worker".into(),
9843            pid: std::process::id(),
9844            protocol: ModuleProtocol::None,
9845            start_time: Some(1),
9846            executable: None,
9847            cgroup_name: None,
9848            container: None,
9849        }];
9850        let mut samples = Vec::new();
9851        for _ in 0..50 {
9852            let start = std::time::Instant::now();
9853            crate::live_children::write_record(&path, &children).unwrap();
9854            samples.push(start.elapsed());
9855        }
9856        samples.sort();
9857        eprintln!(
9858            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9859            (samples[24] + samples[25]) / 2,
9860            samples[47]
9861        );
9862    }
9863
9864    #[cfg(windows)]
9865    #[test]
9866    fn windows_store_touch_fixture() {
9867        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9868            std::fs::write(path, b"module ran").unwrap();
9869        }
9870    }
9871
9872    #[cfg(windows)]
9873    #[tokio::test]
9874    async fn windows_daemon_fixture() {
9875        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9876            return;
9877        };
9878        let mut spec = spec();
9879        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9880        let supervisor =
9881            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9882        let _module = supervisor.spawn(spec).unwrap();
9883    }
9884
9885    #[cfg(windows)]
9886    #[test]
9887    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9888        use std::io::Read;
9889        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9890        // An outer job models a scheduler/service host. The daemon's root and
9891        // root job must nest without requiring breakaway from this job.
9892        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9893        command
9894            .args([
9895                "--exact",
9896                "supervise::spawn_record_tests::windows_daemon_fixture",
9897                "--nocapture",
9898            ])
9899            .env(
9900                "SUBC_TEST_DAEMON_BARRIER",
9901                listener.local_addr().unwrap().to_string(),
9902            );
9903        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9904        listener.set_nonblocking(true).unwrap();
9905        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9906        let mut stream = loop {
9907            if let Ok((stream, _)) = listener.accept() {
9908                break stream;
9909            }
9910            assert!(
9911                daemon.child.try_wait().unwrap().is_none(),
9912                "daemon failed before rendezvous"
9913            );
9914            assert!(
9915                std::time::Instant::now() < deadline,
9916                "daemon did not reach CreateProcess failpoint"
9917            );
9918            std::thread::yield_now();
9919        };
9920        stream
9921            .set_read_timeout(Some(Duration::from_secs(30)))
9922            .unwrap();
9923        let mut pid = [0; 4];
9924        stream.read_exact(&mut pid).unwrap();
9925        let pid = u32::from_ne_bytes(pid);
9926        assert!(subc_jobobject::process_exists(pid));
9927        // Terminate only the daemon, not its outer job: otherwise the host job
9928        // would also kill a child whose daemon never joined its root job.
9929        daemon.child.kill().unwrap();
9930        daemon.child.wait().unwrap();
9931        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9932        if !exited {
9933            daemon.job.terminate().unwrap();
9934        }
9935        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9936    }
9937
9938    #[cfg(windows)]
9939    #[tokio::test]
9940    async fn windows_containment_failure_never_resumes() {
9941        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9942        let sentinel = home.join("store-touched");
9943        let mut spec = spec();
9944        spec.env
9945            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9946        spec.env.push((
9947            "SUBC_TEST_STORE_TOUCH".into(),
9948            sentinel.display().to_string(),
9949        ));
9950        let supervisor =
9951            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9952        let result = supervisor.spawn(spec);
9953        // A version that resumes despite assignment failure must have time to
9954        // execute its sentinel before dropping the returned module kills it.
9955        if let Ok(module) = result {
9956            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9957                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9958                    tokio::task::yield_now().await;
9959                }
9960            })
9961            .await;
9962            let _ = module.stop().await;
9963            panic!(
9964                "assignment failure resumed a module; store touched={}",
9965                sentinel.exists()
9966            );
9967        }
9968        assert!(
9969            !sentinel.exists(),
9970            "refused module executed store-touch sentinel"
9971        );
9972        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9973    }
9974
9975    #[cfg(windows)]
9976    #[tokio::test]
9977    async fn windows_nested_daemon_fixture() {
9978        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9979            return;
9980        };
9981        let mut spec = spec();
9982        spec.env.push((
9983            "SUBC_TEST_STORE_TOUCH".into(),
9984            sentinel.to_str().unwrap().into(),
9985        ));
9986        let supervisor =
9987            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9988        let module = supervisor
9989            .spawn(spec)
9990            .expect("per-module job must nest below daemon and host jobs");
9991        tokio::time::timeout(Duration::from_secs(10), async {
9992            while !PathBuf::from(&sentinel).exists() {
9993                tokio::task::yield_now().await;
9994            }
9995        })
9996        .await
9997        .unwrap();
9998        module.stop().await.unwrap();
9999    }
10000
10001    #[cfg(windows)]
10002    #[test]
10003    fn windows_module_job_nests_under_host_job() {
10004        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
10005        let sentinel = home.join("nested-store-touched");
10006        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
10007        command
10008            .args([
10009                "--exact",
10010                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
10011                "--nocapture",
10012            ])
10013            .env("SUBC_TEST_NESTED_STORE", &sentinel);
10014        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
10015        assert!(daemon.child.wait().unwrap().success());
10016        assert!(sentinel.exists(), "nested protocol:none module never ran");
10017    }
10018}
10019
10020#[cfg(target_os = "linux")]
10021pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10022    use subc_cgroup::KillOutcome;
10023    match subc_cgroup::kill_module(placement, module_id) {
10024        KillOutcome::Killed => {}
10025        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10026            debug!(
10027                module_id,
10028                "cgroup tree kill unavailable; using direct-child kill"
10029            );
10030        }
10031        KillOutcome::IoError { path, error } => {
10032            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10033        }
10034    }
10035}
10036
10037/// Assign a freshly spawned Windows child without ever resuming on failure.
10038///
10039/// Steps 2 and 3 of the suspended-create contract: the job is created and the
10040/// child assigned **while it is still suspended** (step 1 is
10041/// `suspend_on_create_async` at the spawn site). The caller resumes only after
10042/// the child's roster admission is durable. Failed containment kills and reaps
10043/// the suspended child instead of running an unprotected module.
10044#[cfg(windows)]
10045fn contain_spawned_child(
10046    child: &mut Child,
10047    spec: &ModuleSpec,
10048    carried: Option<&Arc<containment::CarriedContainment>>,
10049) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10050    let assignment = || {
10051        let job = match carried {
10052            Some(container) => container.job(),
10053            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10054        };
10055        #[cfg(test)]
10056        spawn_record_tests::assignment_failpoint(spec)?;
10057        job.lock()
10058            .unwrap_or_else(|p| p.into_inner())
10059            .assign(child)?;
10060        Ok::<_, io::Error>(job)
10061    };
10062    match assignment() {
10063        Ok(job) => Ok(Some(job)),
10064        Err(source) => {
10065            kill_and_reap_rejected_child(child);
10066            Err(SuperviseError::Spawn {
10067                program: spec.program.clone(),
10068                source: io::Error::other(format!(
10069                    "Windows module job containment failed: {source}"
10070                )),
10071                cgroup_path: None,
10072            })
10073        }
10074    }
10075}
10076
10077/// Rejection is synchronous: do not return a spawn error while the child can
10078/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
10079fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10080    #[cfg(unix)]
10081    group.signal_rejected(child);
10082    let _ = child.start_kill();
10083    while matches!(child.try_wait(), Ok(None)) {
10084        std::thread::yield_now();
10085    }
10086}
10087
10088/// Resume a suspended child, killing it if it cannot be started.
10089///
10090/// A suspended process holds a pid and does nothing, so there is no useful
10091/// state to return: the caller gets an error and the spawn fails.
10092#[cfg(windows)]
10093fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10094    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10095        // Kill it here rather than leaving a suspended process for the caller
10096        // to notice; `kill_on_drop` would eventually do this, but the module
10097        // would have been reported as running in between.
10098        let _ = std::process::Command::new("taskkill.exe")
10099            .args(["/PID", &pid.to_string(), "/T", "/F"])
10100            .stdin(Stdio::null())
10101            .stdout(Stdio::null())
10102            .stderr(Stdio::null())
10103            .status();
10104        return Err(SuperviseError::Spawn {
10105            program: spec.program.clone(),
10106            source,
10107            cgroup_path: None,
10108        });
10109    }
10110    Ok(())
10111}
10112
10113#[cfg(target_os = "linux")]
10114fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10115    match placement.remove_module(module_id) {
10116        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10117        Err(error) => warn!(
10118            module_id,
10119            error = %error,
10120            "could not remove module cgroup after process exit; continuing teardown"
10121        ),
10122    }
10123}
10124
10125#[cfg(target_os = "linux")]
10126const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10127#[cfg(target_os = "linux")]
10128const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10129
10130#[cfg(target_os = "linux")]
10131async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10132    let path = placement.module_path(module_id).ok();
10133    if path.as_ref().is_some_and(|path| {
10134        std::fs::read_to_string(path.join("cgroup.events"))
10135            .ok()
10136            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10137    }) {
10138        remove_module_cgroup(placement, module_id);
10139        return;
10140    }
10141
10142    // Freeze before enumerating: a frozen task cannot exit and release its PID
10143    // for reuse by a process outside this module while SIGTERM is being sent.
10144    if terminate_reaped_cgroup(placement, module_id).await {
10145        if let Some(path) = &path {
10146            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10147            while cgroup_is_populated(path) && Instant::now() < deadline {
10148                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10149            }
10150        }
10151    }
10152
10153    // Reaping the direct child is not proof its descendants exited. Kill any
10154    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
10155    // cgroup empty before rmdir.
10156    if matches!(
10157        subc_cgroup::kill_module(Some(placement), module_id),
10158        subc_cgroup::KillOutcome::Killed
10159    ) {
10160        if let Some(path) = &path {
10161            while cgroup_is_populated(path) {
10162                sleep(Duration::from_millis(1)).await;
10163            }
10164        }
10165    }
10166    remove_module_cgroup(placement, module_id);
10167}
10168
10169#[cfg(target_os = "linux")]
10170fn cgroup_is_populated(path: &std::path::Path) -> bool {
10171    std::fs::read_to_string(path.join("cgroup.events"))
10172        .ok()
10173        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10174}
10175
10176#[cfg(target_os = "linux")]
10177async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10178    let frozen =
10179        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10180            .await
10181        {
10182            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10183            outcome => {
10184                tracing::debug!(
10185                    module_id,
10186                    ?outcome,
10187                    "cgroup freeze failed; skipping SIGTERM grace"
10188                );
10189                return false;
10190            }
10191        };
10192    let listed = match frozen.processes() {
10193        subc_cgroup::ProcsOutcome::Listed(pids) => {
10194            for pid in pids.into_iter().filter(|pid| *pid > 1) {
10195                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10196                    if let Err(error) =
10197                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10198                    {
10199                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10200                    }
10201                }
10202            }
10203            true
10204        }
10205        outcome => {
10206            tracing::debug!(
10207                module_id,
10208                ?outcome,
10209                "cgroup process listing failed; skipping SIGTERM grace"
10210            );
10211            false
10212        }
10213    };
10214    match frozen.thaw() {
10215        subc_cgroup::ThawOutcome::Thawed => listed,
10216        outcome => {
10217            tracing::debug!(
10218                module_id,
10219                ?outcome,
10220                "cgroup thaw failed; proceeding to cgroup.kill"
10221            );
10222            false
10223        }
10224    }
10225}
10226
10227#[cfg(target_os = "linux")]
10228fn apply_cgroup_placement(
10229    command: &mut Command,
10230    spec: &ModuleSpec,
10231    path: &std::path::Path,
10232) -> Result<(), SuperviseError> {
10233    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10234        module_id: spec.module_id.clone(),
10235        source,
10236    })
10237}
10238
10239fn capture_retention(spec: &ModuleSpec) -> Retention {
10240    let defaults = Retention::default();
10241    let value = |name: &str| {
10242        spec.env
10243            .iter()
10244            .rev()
10245            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10246    };
10247    Retention {
10248        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10249            .and_then(|value| value.parse().ok())
10250            .unwrap_or(defaults.max_file_mb),
10251        keep: value(CAPTURE_KEEP_ENV)
10252            .and_then(|value| value.parse().ok())
10253            .unwrap_or(defaults.keep),
10254        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10255            .and_then(|value| value.parse().ok())
10256            .unwrap_or(defaults.max_age_days),
10257    }
10258}
10259
10260/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
10261/// module's registration to the exact process the supervisor spawned.
10262fn generate_launch_nonce() -> Result<String, SuperviseError> {
10263    let mut bytes = [0u8; 32];
10264    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10265        reason: source.to_string(),
10266    })?;
10267    let mut hex = String::with_capacity(64);
10268    for b in bytes {
10269        use std::fmt::Write;
10270        let _ = write!(hex, "{b:02x}");
10271    }
10272    Ok(hex)
10273}
10274
10275/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
10276/// signal about how many leading bytes matched.
10277fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10278    if a.len() != b.len() {
10279        return false;
10280    }
10281    let mut diff = 0u8;
10282    for (x, y) in a.iter().zip(b.iter()) {
10283        diff |= x ^ y;
10284    }
10285    diff == 0
10286}
10287
10288/// The kernel's image after an acknowledged exec, shared by ordinary launches
10289/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
10290/// not identities inferred from a configured pathname.
10291#[cfg(not(windows))]
10292fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10293    let process = subc_os::Process::open(pid).ok().flatten()?;
10294    #[cfg(target_os = "linux")]
10295    return sample_after_linux_exec(
10296        || {
10297            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10298            let committed = linux_exec_committed(&stat);
10299            #[cfg(test)]
10300            spawn_image_tests::report_exec_state(committed);
10301            committed
10302        },
10303        || process.observe(),
10304    );
10305    #[cfg(not(target_os = "linux"))]
10306    process.observe()
10307}
10308
10309/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
10310/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
10311/// installation. Until it clears, /proc/pid/exe can still name the daemon.
10312#[cfg(target_os = "linux")]
10313fn linux_exec_committed(stat: &str) -> Option<bool> {
10314    const PF_FORKNOEXEC: u64 = 0x40;
10315    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10316    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10317}
10318
10319#[cfg(target_os = "linux")]
10320fn sample_after_linux_exec(
10321    mut exec_committed: impl FnMut() -> Option<bool>,
10322    observe: impl FnOnce() -> Option<subc_os::Observation>,
10323) -> Option<subc_os::Observation> {
10324    let deadline = std::time::Instant::now() + Duration::from_secs(1);
10325    while !exec_committed()? {
10326        // A failed or indefinitely stalled exec must not publish the inherited
10327        // image as authority to signal a future orphan. Missing identity keeps
10328        // the existing conservative orphan-sweep behavior.
10329        if std::time::Instant::now() >= deadline {
10330            return None;
10331        }
10332        std::thread::sleep(Duration::from_micros(50));
10333    }
10334    observe()
10335}
10336
10337#[cfg(all(test, target_os = "linux"))]
10338mod spawn_image_tests {
10339    use super::*;
10340    use std::cell::{Cell, RefCell};
10341
10342    enum ExecEvent {
10343        Uncommitted,
10344        Image(Option<subc_os::Observation>),
10345    }
10346
10347    thread_local! {
10348        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10349    }
10350
10351    pub(super) fn report_exec_state(committed: Option<bool>) {
10352        if committed == Some(false) {
10353            EXEC_EVENTS.with(|events| {
10354                if let Some(events) = events.borrow_mut().take() {
10355                    let _ = events.send(ExecEvent::Uncommitted);
10356                }
10357            });
10358        }
10359    }
10360
10361    #[test]
10362    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10363        use std::io::{Read, Write};
10364        use std::os::fd::AsRawFd;
10365
10366        let descriptor = std::fs::File::open("/dev/null").unwrap();
10367        let mut command = std::process::Command::new("/bin/sleep");
10368        command.arg("30");
10369        let (mut ready, mut release) =
10370            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10371                .unwrap();
10372        let spawn = std::thread::spawn(move || command.spawn());
10373        let mut message = [0u8; 8];
10374        ready.read_exact(&mut message).unwrap();
10375        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10376        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10377        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10378        let (events, received) = std::sync::mpsc::channel();
10379        let observer = std::thread::spawn(move || {
10380            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10381            let image = observe_spawned_image(pid);
10382            let _ = events.send(ExecEvent::Image(image));
10383        });
10384        // Release only once the production observer has seen the uncommitted
10385        // exec, or has incorrectly returned the inherited image. Either way,
10386        // release and reap before asserting so a regression cannot leak a child.
10387        let first = received.recv_timeout(Duration::from_secs(5));
10388        release.write_all(&[1]).unwrap();
10389        let mut child = spawn.join().unwrap().unwrap();
10390        let after = match first {
10391            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10392                Ok(ExecEvent::Image(image)) => image,
10393                _ => None,
10394            },
10395            Ok(ExecEvent::Image(image)) => image,
10396            Err(_) => None,
10397        };
10398        observer.join().unwrap();
10399        child.kill().unwrap();
10400        child.wait().unwrap();
10401
10402        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10403        assert_eq!(
10404            inherited.unwrap().executable,
10405            subc_os::file_identity(&std::env::current_exe().unwrap())
10406        );
10407        assert_eq!(
10408            after.unwrap().executable,
10409            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10410        );
10411    }
10412
10413    #[test]
10414    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10415        let committed = Cell::new(false);
10416        let inherited = subc_os::Observation {
10417            start_time: 23,
10418            executable: Some(subc_os::FileIdentity {
10419                device: 29,
10420                inode: 31,
10421            }),
10422        };
10423        let module = subc_os::Observation {
10424            start_time: 23,
10425            executable: Some(subc_os::FileIdentity {
10426                device: 29,
10427                inode: 37,
10428            }),
10429        };
10430        let polls = Cell::new(0);
10431        let image = sample_after_linux_exec(
10432            || {
10433                polls.set(polls.get() + 1);
10434                // Hold the inherited image through the first poll, as in the
10435                // posix_spawn parent-wake window, then commit the module image.
10436                committed.set(polls.get() == 2);
10437                Some(committed.get())
10438            },
10439            || Some(if committed.get() { module } else { inherited }),
10440        );
10441        assert_eq!(image, Some(module));
10442        assert_eq!(polls.get(), 2);
10443    }
10444
10445    #[test]
10446    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10447        assert_eq!(
10448            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10449            Some(false)
10450        );
10451        assert_eq!(
10452            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10453            Some(true)
10454        );
10455        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10456        assert_eq!(linux_exec_committed("17 (a) R"), None);
10457    }
10458
10459    #[test]
10460    fn unreadable_exec_state_never_samples_an_inherited_image() {
10461        assert_eq!(
10462            sample_after_linux_exec(
10463                || None,
10464                || panic!("must not sample before exec confirmation")
10465            ),
10466            None
10467        );
10468    }
10469
10470    #[test]
10471    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10472        assert_eq!(
10473            sample_after_linux_exec(
10474                || Some(false),
10475                || panic!("must not sample after an exec timeout")
10476            ),
10477            None
10478        );
10479    }
10480}
10481
10482/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
10483/// different executable after the no-delete capture was taken.
10484#[cfg(windows)]
10485fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10486    let mut program = spec.program.clone();
10487    if program.is_absolute() || program.components().count() > 1 {
10488        // An explicit path can name a PE image with no extension. Try that
10489        // exact file first, as Command does, before falling back to .exe.
10490        match subc_os::ExecutableCapture::open(&program) {
10491            Ok(capture) => return Ok(capture),
10492            Err(error)
10493                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10494            Err(error) => return Err(error),
10495        }
10496        program.set_extension("exe");
10497        return subc_os::ExecutableCapture::open(&program);
10498    }
10499    if program.extension().is_none() {
10500        program.set_extension("exe");
10501    }
10502    let custom_path = spec
10503        .env
10504        .iter()
10505        .rev()
10506        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10507    let mut directories = Vec::new();
10508    // These are Windows' default executable search locations. A configured
10509    // PATH still overrides the inherited PATH, not the system directories.
10510    if let Some(parent) = std::env::current_exe()?.parent() {
10511        directories.push(parent.to_path_buf());
10512    }
10513    directories.push(std::env::current_dir()?);
10514    if let Some(windows) = std::env::var_os("SystemRoot") {
10515        let windows = PathBuf::from(windows);
10516        directories.push(windows.join("System32"));
10517        directories.push(windows.clone());
10518    }
10519    let path = custom_path
10520        .map(|(_, value)| std::ffi::OsString::from(value))
10521        .or_else(|| std::env::var_os("PATH"));
10522    if let Some(path) = path {
10523        directories.extend(std::env::split_paths(&path));
10524    }
10525    for directory in directories {
10526        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10527            Ok(capture) => return Ok(capture),
10528            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10529            Err(error) => return Err(error),
10530        }
10531    }
10532    Err(io::Error::new(
10533        io::ErrorKind::NotFound,
10534        "resolved executable not found",
10535    ))
10536}
10537
10538fn spawn_and_mark_running(
10539    spec: &ModuleSpec,
10540    runtime: &SupervisorRuntimeConfig,
10541    snapshot: &SharedSnapshot,
10542) -> Result<SupervisedChild, SuperviseError> {
10543    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10544    let registration_watch = watch_registration(&runtime.registry, spec);
10545    #[cfg(unix)]
10546    let joining_group = containment::current(&runtime.child_roster.containment)
10547        .is_some_and(|container| container.group().is_some());
10548    let spawn = || {
10549        spawn_child(
10550            spec,
10551            runtime.connection_file_path.as_deref(),
10552            runtime.supervisor_handle.as_ref(),
10553            &runtime.stderr_ring,
10554            runtime.capture_logs_dir.as_deref(),
10555            &runtime.child_roster,
10556            #[cfg(target_os = "linux")]
10557            runtime.cgroup_placement.as_ref(),
10558        )
10559    };
10560    let result = spawn();
10561    #[cfg(unix)]
10562    let result = if result.is_err()
10563        && joining_group
10564        && !runtime.child_roster.is_closed()
10565        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10566        && runtime
10567            .child_roster
10568            .previous_instance(&spec.module_id)
10569            .is_none()
10570    {
10571        // The failed join already destroyed and verified the old container.
10572        // Try one fresh group as part of this same restart, without spending
10573        // another crash-budget entry. Cgroups and Windows jobs do not retry.
10574        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10575        spawn()
10576    } else {
10577        result
10578    };
10579    let child = result?;
10580    let mut child = child;
10581    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10582    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10583        containment::terminate_blocking(&runtime.child_roster.containment);
10584        return Err(error);
10585    }
10586    if let Some(spawn) = activity_spawn {
10587        spawn.complete(snapshot);
10588    }
10589    Ok(child)
10590}
10591
10592struct RegistrationWatch {
10593    changes: watch::Receiver<u64>,
10594    baseline: u64,
10595}
10596
10597// Capture before spawning: a fast child's accepted HELLO can disappear from the
10598// active registry before the monitor first runs. A pre-existing registration
10599// must not satisfy a new process's deadline either.
10600fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10601    if spec.protocol != ModuleProtocol::Subc {
10602        return None;
10603    }
10604    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10605    let baseline = *changes.borrow();
10606    Some(RegistrationWatch { changes, baseline })
10607}
10608
10609fn arm_registration_deadline(
10610    child: &mut SupervisedChild,
10611    budget: Duration,
10612    registration_watch: Option<RegistrationWatch>,
10613) {
10614    child.registration_deadline =
10615        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10616    child.registration_watch = registration_watch;
10617}
10618
10619async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10620    if let Some(container) = child.child.container() {
10621        container.stop_members().await;
10622        child.kill_leader()?;
10623        child.wait().await?;
10624        return Ok(());
10625    }
10626    #[cfg(unix)]
10627    {
10628        // Children lead their own process groups. Signal the whole group so a
10629        // helper stuck before HELLO cannot survive its supervised parent.
10630        let group = unreaped_registration_group(child)?;
10631        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10632        // Process-group safety rule: only signal a group while its leader is
10633        // our unreaped child. Even a zombie leader reserves its pid/group id;
10634        // reaping it before KILL would let that id name an unrelated group.
10635        // Sleep without waiting so TERM-ignoring helpers get the full grace.
10636        sleep(crate::child_roster::TERM_TO_KILL).await;
10637        let group = unreaped_registration_group(child)?;
10638        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10639    }
10640    // Windows terminates the containing Job Object as well as the direct
10641    // child. Linux also kills the child's cgroup when placement is enabled.
10642    child.start_kill()?;
10643    child.wait().await?;
10644    Ok(())
10645}
10646
10647#[cfg(unix)]
10648fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10649    if let Some(container) = child.child.container() {
10650        return container
10651            .group()
10652            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10653            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10654    }
10655    // The group id comes only from the pid recorded at spawn, when the child
10656    // created its own group. Never admit init's group or a negative group id.
10657    let group = i32::try_from(child.pid)
10658        .ok()
10659        .filter(|pid| *pid > 1)
10660        .and_then(rustix::process::Pid::from_raw)
10661        .ok_or_else(|| {
10662            io::Error::new(
10663                io::ErrorKind::InvalidInput,
10664                "invalid supervised process group",
10665            )
10666        })?;
10667    if child.child.id() != Some(child.pid) {
10668        return Err(io::Error::new(
10669            io::ErrorKind::InvalidInput,
10670            "refusing process-group signal after supervised child was reaped",
10671        ));
10672    }
10673    Ok(group)
10674}
10675
10676enum RegistrationWaitOutcome {
10677    Registered,
10678    Exited(ExitReport),
10679    TimedOut,
10680}
10681
10682struct ReloadRegistrationFailure {
10683    exit_report: ExitReport,
10684    reason: String,
10685}
10686
10687#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10688enum BusyGaugeObservation {
10689    Quiescent,
10690    Busy,
10691    Omitted,
10692}
10693
10694/// What one health answer said about one declared busy gauge.
10695#[derive(Debug, Clone, PartialEq, Eq)]
10696enum GaugeVerdict {
10697    /// The gauge read 0: this gauge is not what holds the drain.
10698    Zero,
10699    /// The gauge read a positive count.
10700    NonZero(u64),
10701    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10702    /// a negative number). It counts as busy, because the daemon cannot tell
10703    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10704    NotACount(String),
10705    /// The health answer had no metrics object, or one without this gauge.
10706    Missing,
10707    /// The probe produced no health answer to read; carries the probe error's
10708    /// evidence label (`no-answer` for a probe that timed out).
10709    ProbeFailed(&'static str),
10710}
10711
10712impl GaugeVerdict {
10713    fn reads_quiet(&self) -> bool {
10714        matches!(self, Self::Zero)
10715    }
10716}
10717
10718impl fmt::Display for GaugeVerdict {
10719    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10720        match self {
10721            Self::Zero => f.write_str("zero"),
10722            Self::NonZero(value) => write!(f, "non_zero({value})"),
10723            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10724            Self::Missing => f.write_str("missing_from_health_body"),
10725            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10726        }
10727    }
10728}
10729
10730/// The longest JSON text kept for a gauge value that is not a count, so a
10731/// module reporting a large object as a gauge cannot flood the drain log.
10732const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10733
10734/// Read each declared gauge out of one health answer's metrics.
10735fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10736    let metrics = metrics.and_then(Value::as_object);
10737    gauges
10738        .iter()
10739        .map(
10740            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10741                None => GaugeVerdict::Missing,
10742                Some(value) => match value.as_u64() {
10743                    Some(0) => GaugeVerdict::Zero,
10744                    Some(count) => GaugeVerdict::NonZero(count),
10745                    None => {
10746                        let mut text = value.to_string();
10747                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10748                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10749                            while !text.is_char_boundary(end) {
10750                                end -= 1;
10751                            }
10752                            text.truncate(end);
10753                            text.push_str("...");
10754                        }
10755                        GaugeVerdict::NotACount(text)
10756                    }
10757                },
10758            },
10759        )
10760        .collect()
10761}
10762
10763/// The drain's overall reading of one health answer: omitted when any
10764/// declared gauge is missing, quiescent only when every gauge reads 0.
10765fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10766    if verdicts
10767        .iter()
10768        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10769    {
10770        BusyGaugeObservation::Omitted
10771    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10772        BusyGaugeObservation::Quiescent
10773    } else {
10774        BusyGaugeObservation::Busy
10775    }
10776}
10777
10778/// One declared gauge's history over a drain: the latest verdict on it, and
10779/// the latest value actually read for it (which a later failed probe or
10780/// missing gauge does not erase). Times are offsets from the start of the
10781/// drain's wait.
10782#[derive(Debug, Clone, Default, PartialEq, Eq)]
10783struct GaugeTrace {
10784    latest: Option<(GaugeVerdict, Duration)>,
10785    last_value: Option<(GaugeVerdict, Duration)>,
10786}
10787
10788/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10789/// its ceiling can say in its timeout line which declared gauge held it and
10790/// why. Without it, a gauge that read busy, a gauge the module never
10791/// reported, and a probe that never got an answer all looked the same: a
10792/// drain that ended at its ceiling with nothing in flight.
10793#[derive(Debug, Clone, Default, PartialEq, Eq)]
10794struct DrainGaugeTrace {
10795    /// Health probes the drain sent.
10796    probes: u32,
10797    /// Each declared gauge, in declaration order.
10798    gauges: Vec<(String, GaugeTrace)>,
10799}
10800
10801impl DrainGaugeTrace {
10802    fn new(gauges: &[String]) -> Self {
10803        Self {
10804            probes: 0,
10805            gauges: gauges
10806                .iter()
10807                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10808                .collect(),
10809        }
10810    }
10811
10812    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10813    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10814        self.probes = self.probes.saturating_add(1);
10815        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10816            trace.latest = Some((verdict.clone(), at));
10817            if matches!(
10818                verdict,
10819                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10820            ) {
10821                trace.last_value = Some((verdict.clone(), at));
10822            }
10823        }
10824    }
10825}
10826
10827/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10828/// declared gauge, separated by ` | `. A value is shown as its verdict
10829/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10830/// and whether that counted as busy.
10831impl fmt::Display for DrainGaugeTrace {
10832    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10833        if self.gauges.is_empty() {
10834            return f.write_str("none declared");
10835        }
10836        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10837            if index > 0 {
10838                f.write_str(" | ")?;
10839            }
10840            write!(f, "{name}: ")?;
10841            match &trace.latest {
10842                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10843                None => f.write_str("never probed")?,
10844            }
10845            match &trace.last_value {
10846                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10847                None => f.write_str("; no value ever read")?,
10848            }
10849        }
10850        Ok(())
10851    }
10852}
10853
10854pub(crate) fn declared_busy_gauges(
10855    registry: &Registry,
10856    module_id: &str,
10857) -> Result<Vec<String>, SuperviseError> {
10858    busy_gauges_of(
10859        registry
10860            .get_module(module_id)
10861            .map_err(SuperviseError::Registry)?,
10862    )
10863}
10864
10865/// [`declared_busy_gauges`] for the registration a connection holds, in any
10866/// slot: after cutover the incumbent is no longer the id's active
10867/// registration, and its own manifest is the one that names its gauges.
10868fn declared_busy_gauges_for_connection(
10869    registry: &Registry,
10870    connection_id: ConnectionId,
10871) -> Result<Vec<String>, SuperviseError> {
10872    busy_gauges_of(
10873        registry
10874            .get_module_by_connection(connection_id)
10875            .map_err(SuperviseError::Registry)?,
10876    )
10877}
10878
10879fn busy_gauges_of(
10880    registration: Option<crate::registry::ModuleRegistration>,
10881) -> Result<Vec<String>, SuperviseError> {
10882    let Some(registration) = registration else {
10883        return Ok(Vec::new());
10884    };
10885    let Some(self_signals) = registration.manifest.self_signals else {
10886        return Ok(Vec::new());
10887    };
10888
10889    let mut gauges = Vec::new();
10890    for declaration in self_signals {
10891        if declaration.kind != SelfSignalKind::Busy {
10892            continue;
10893        }
10894        match declaration.anchored_to {
10895            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10896                gauges.extend(declared)
10897            }
10898            _ => {
10899                // An invalid Busy anchor is fail-safe: the empty name cannot be
10900                // present in a conforming health report, so this drain stays busy.
10901                gauges.push(String::new());
10902            }
10903        }
10904    }
10905    Ok(gauges)
10906}
10907
10908/// Wait for `endpoint` to have nothing in flight and, when the module declares
10909/// busy gauges, for a health probe to report them quiet. The probe is addressed
10910/// by `scope`: a swap's superseded incumbent must be asked about its own
10911/// gauges, and by module id the probe would reach the promoted candidate.
10912/// While the gauges read busy, the probe repeats every
10913/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10914/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10915/// timeout line.
10916#[allow(clippy::too_many_arguments)]
10917async fn wait_for_forwarding_quiescence(
10918    forwarding: &ForwardingTable,
10919    module_id: &str,
10920    runtime: &SupervisorRuntimeConfig,
10921    endpoint: crate::ModuleEndpointId,
10922    deadline: Instant,
10923    busy_gauges: &[String],
10924    scope: DrainScope,
10925    trace: &mut DrainGaugeTrace,
10926) -> Result<bool, SuperviseError> {
10927    let wait_started = Instant::now();
10928    let mut gauges_quiescent = busy_gauges.is_empty();
10929    let mut next_probe_at = wait_started;
10930    let mut omission_reported = false;
10931
10932    loop {
10933        let now = Instant::now();
10934        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10935            let report = match scope {
10936                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10937                DrainScope::Endpoint(endpoint) => {
10938                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10939                }
10940            };
10941            let verdicts = match &report {
10942                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10943                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10944            };
10945            trace.record(&verdicts, wait_started.elapsed());
10946            gauges_quiescent = match report {
10947                Ok(_) => match busy_gauge_observation(&verdicts) {
10948                    BusyGaugeObservation::Quiescent => true,
10949                    BusyGaugeObservation::Busy => false,
10950                    BusyGaugeObservation::Omitted => {
10951                        // Once per drain: a module that never reports a gauge
10952                        // it declared would otherwise say so on every probe.
10953                        if !omission_reported {
10954                            forwarding
10955                                .counters()
10956                                .increment_drains_with_undeclared_gauge();
10957                            let missing = busy_gauges
10958                                .iter()
10959                                .zip(&verdicts)
10960                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10961                                .map(|(gauge, _)| gauge.as_str())
10962                                .collect::<Vec<_>>()
10963                                .join(",");
10964                            warn!(
10965                                module_id,
10966                                missing_gauges = %missing,
10967                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10968                            );
10969                            omission_reported = true;
10970                        }
10971                        false
10972                    }
10973                },
10974                Err(err) => {
10975                    warn!(
10976                        module_id,
10977                        error = %err,
10978                        "drain health.check did not produce declared busy gauges; treating module as busy"
10979                    );
10980                    false
10981                }
10982            };
10983            // Counted from the probe's completion: the probe above was awaited,
10984            // so no second probe can start while one is still outstanding.
10985            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10986        }
10987
10988        let in_flight = forwarding
10989            .endpoint_in_flight_count(endpoint)
10990            .map_err(SuperviseError::Forwarding)?;
10991        if in_flight == 0 && gauges_quiescent {
10992            return Ok(true);
10993        }
10994
10995        let now = Instant::now();
10996        if now >= deadline {
10997            return Ok(false);
10998        }
10999        let mut wait = deadline
11000            .saturating_duration_since(now)
11001            .min(REGISTRY_RELEASE_POLL);
11002        if !busy_gauges.is_empty() {
11003            wait = wait.min(next_probe_at.saturating_duration_since(now));
11004        }
11005        sleep(wait).await;
11006    }
11007}
11008
11009/// The wait between drain busy-gauge probes: the larger of
11010/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
11011/// drain's own in-flight poll interval.
11012fn drain_busy_reprobe_interval() -> Duration {
11013    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
11014}
11015
11016/// Logs why a drain gave up: the drain waited its full timeout without the
11017/// module going quiet, and teardown is being forced.
11018///
11019/// The line names two kinds of evidence, because a bare "timed out" hides
11020/// causes that need different fixes:
11021/// - `holdouts`: the requests and routes still open when the timeout hit,
11022///   and on which connections. One stuck call is a module bug; a held stream
11023///   on every session is a module that should end its streams when it
11024///   receives `module.draining`.
11025/// - `gauges`: for each busy gauge the module declares, its latest reading
11026///   and how many probes ran. This separates a module that kept reporting
11027///   itself busy, one whose health answer never contained the gauge, and one
11028///   that never answered the probes, which otherwise all look like a timeout
11029///   with nothing in flight.
11030fn warn_drain_timed_out(
11031    module_id: &str,
11032    drain_timeout: Duration,
11033    reason: RouteCloseReason,
11034    holdouts: &crate::forwarding::DrainHoldouts,
11035    gauges: &DrainGaugeTrace,
11036) {
11037    warn!(
11038        module_id = %module_id,
11039        waited = ?drain_timeout,
11040        ?reason,
11041        held_requests = holdouts.requests,
11042        held_routes = holdouts.routes,
11043        total_routes = holdouts.total_routes,
11044        top_connections = ?holdouts.top_connections,
11045        // `module_channel:corr`, so the module can find each held request
11046        // in its own log; capped, so `held_requests` is the full count.
11047        held = %holdouts
11048            .held
11049            .iter()
11050            .map(|(channel, corr)| format!("{channel}:{corr}"))
11051            .collect::<Vec<_>>()
11052            .join(","),
11053        gauge_probes = gauges.probes,
11054        busy_gauges = %gauges,
11055        "route drain timed out before request quiescence; forcing teardown"
11056    );
11057}
11058
11059/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
11060///
11061/// `Ok` is always honest and passed straight through -- the wait actually measured
11062/// in-flight state. `Err` means the wait produced no measurement at all (the
11063/// forwarding table's lock was poisoned), so `false` is reported as the one honest
11064/// constant: the drain did not complete. Never recomputed from route state, never a
11065/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
11066fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11067    match wait_result {
11068        Ok(drained) => *drained,
11069        Err(_) => false,
11070    }
11071}
11072
11073fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11074    for released in released_routes {
11075        let frame = match Frame::build_with_version(
11076            released.negotiated_ver,
11077            FrameType::Goodbye,
11078            control_flags(),
11079            released.channel,
11080            released.epoch,
11081            0,
11082            Vec::new(),
11083        ) {
11084            Ok(frame) => frame,
11085            Err(err) => {
11086                warn!(
11087                    route_channel = released.channel,
11088                    error = %err,
11089                    "failed to build supervisor drain route GOODBYE frame"
11090                );
11091                continue;
11092            }
11093        };
11094        if !released.close_on_delivery_failure() {
11095            crate::forwarding::send_module_route_goodbye(
11096                &forwarding.counters(),
11097                &released.sink,
11098                frame,
11099                released.module_id.as_deref(),
11100                "supervisor drain",
11101            );
11102            continue;
11103        }
11104        if let Err(err) = released.sink.try_send(frame) {
11105            warn!(
11106                target_connection_id = released.connection_id.get(),
11107                route_channel = released.channel,
11108                error = %err,
11109                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11110            );
11111            let _ = forwarding.escalate_client_delivery_failure(
11112                released.connection_id,
11113                released.channel,
11114                released.epoch,
11115                CloseReason::new(
11116                    "route_goodbye_delivery_failed",
11117                    format!(
11118                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11119                        released.channel
11120                    ),
11121                ),
11122                crate::forwarding::UndeliveredFrame {
11123                    module_id: released.module_id.as_deref(),
11124                    sink: &released.sink,
11125                },
11126            );
11127        }
11128    }
11129}
11130
11131async fn send_module_draining(
11132    module_id: &str,
11133    reason: RouteCloseReason,
11134    deadline_ms: u64,
11135    target: &ModuleDrainTarget,
11136    _wait_for_flush: bool,
11137) -> bool {
11138    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11139        reason,
11140        deadline_ms,
11141    }) {
11142        Ok(body) => body,
11143        Err(err) => {
11144            warn!(
11145                module_id,
11146                error = %err,
11147                "failed to encode module draining command"
11148            );
11149            return false;
11150        }
11151    };
11152    let frame = match Frame::build_with_version(
11153        target.negotiated_ver,
11154        FrameType::Push,
11155        control_flags(),
11156        0,
11157        0,
11158        0,
11159        body,
11160    ) {
11161        Ok(frame) => frame,
11162        Err(err) => {
11163            warn!(
11164                module_id,
11165                error = %err,
11166                "failed to build module draining command frame"
11167            );
11168            return false;
11169        }
11170    };
11171    #[cfg(unix)]
11172    if _wait_for_flush {
11173        return target.sink.send_flushed(frame).await.is_ok();
11174    }
11175    if let Err(err) = target.sink.try_send(frame) {
11176        warn!(
11177            module_id,
11178            target_connection_id = target.endpoint.connection_id.get(),
11179            error = %err,
11180            "module draining command was not delivered to peer"
11181        );
11182        return false;
11183    }
11184    true
11185}
11186
11187/// The channel-0 GOODBYE that tells a module its stop is planned.
11188fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11189    match Frame::build_with_version(
11190        negotiated_ver,
11191        FrameType::Goodbye,
11192        control_flags(),
11193        0,
11194        0,
11195        0,
11196        Vec::new(),
11197    ) {
11198        Ok(frame) => Some(frame),
11199        Err(err) => {
11200            warn!(
11201                module_id,
11202                error = %err,
11203                "failed to build module GOODBYE frame"
11204            );
11205            None
11206        }
11207    }
11208}
11209
11210/// Send every registered module connection its module GOODBYE at daemon
11211/// shutdown, then request that connection's close.
11212///
11213/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
11214/// before EOF, so the GOODBYE must reach the socket before the close. A close
11215/// request does not wait for the connection's queued frames: its writer gets a
11216/// bounded grace after the close, is aborted if it overruns it, and the daemon
11217/// process may exit before that grace ends. So with `wait_for_flush`, each
11218/// connection is closed only after its writer has acknowledged writing the
11219/// GOODBYE, or once a short shared budget runs out, so one module that is not
11220/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
11221/// are only queued, for a shutdown the operator has told to stop waiting.
11222/// A connection that is already gone is skipped.
11223#[cfg(unix)]
11224async fn send_module_goodbyes_for_daemon_shutdown(
11225    forwarding: &Arc<ForwardingTable>,
11226    reason: &CloseReason,
11227    wait_for_flush: bool,
11228) {
11229    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11230    let targets = match forwarding.module_connections() {
11231        Ok(targets) => targets,
11232        Err(err) => {
11233            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11234            return;
11235        }
11236    };
11237    let deadline = Instant::now() + GOODBYE_BUDGET;
11238    let mut sends = tokio::task::JoinSet::new();
11239    for target in targets {
11240        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11241            continue;
11242        };
11243        if !wait_for_flush {
11244            if let Err(err) = target.sink.try_send(frame) {
11245                debug!(
11246                    module_id = %target.module_id,
11247                    error = %err,
11248                    "shutdown module GOODBYE was not queued"
11249                );
11250            }
11251            continue;
11252        }
11253        let forwarding = Arc::clone(forwarding);
11254        let reason = reason.clone();
11255        sends.spawn(async move {
11256            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11257                Ok(Ok(())) => {}
11258                Ok(Err(err)) => debug!(
11259                    module_id = %target.module_id,
11260                    error = %err,
11261                    "module connection closed before its shutdown GOODBYE was written"
11262                ),
11263                Err(_) => warn!(
11264                    module_id = %target.module_id,
11265                    budget = ?GOODBYE_BUDGET,
11266                    "shutdown module GOODBYE was not written within its budget; closing anyway"
11267                ),
11268            }
11269            forwarding.request_connection_close(target.endpoint.connection_id, reason);
11270        });
11271    }
11272    // Every task ends by the shared deadline, so this wait is bounded too.
11273    while sends.join_next().await.is_some() {}
11274}
11275
11276fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11277    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11278        return;
11279    };
11280    if let Err(err) = target.sink.try_send(frame) {
11281        warn!(
11282            module_id,
11283            target_connection_id = target.endpoint.connection_id.get(),
11284            error = %err,
11285            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11286        );
11287        forwarding.request_connection_close(
11288            target.endpoint.connection_id,
11289            CloseReason::new(
11290                "module_goodbye_delivery_failed",
11291                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11292            ),
11293        );
11294    }
11295}
11296
11297#[derive(Clone, Copy)]
11298struct ForwardingDrainContext<'a> {
11299    spec: &'a ModuleSpec,
11300    runtime: &'a SupervisorRuntimeConfig,
11301    registry: &'a Registry,
11302    scope: DrainScope,
11303}
11304
11305/// Which process a forwarding drain addresses.
11306#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11307enum DrainScope {
11308    /// Whatever endpoint is active for the module id: every plain stop,
11309    /// restart and reload. Also moves the module's state to `Draining`.
11310    Active,
11311    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
11312    /// module id would resolve to the promoted candidate and leave neither
11313    /// process routable. The module's state is left alone, since the promoted
11314    /// candidate is what it describes and that process is running.
11315    Endpoint(crate::ModuleEndpointId),
11316}
11317
11318/// Whether a child being drained has already been asked to stop by the time
11319/// its drain wait starts.
11320///
11321/// The drain wait is the same budget whatever this says. What it decides is
11322/// whether the supervisor must ask by signal before that wait begins: a child
11323/// that nobody asked will sit out the whole budget and then be SIGKILLed,
11324/// healthy or not.
11325#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11326enum StopNotice {
11327    /// The module was sent `module.draining` and a module GOODBYE over its own
11328    /// registered connection, and stops itself.
11329    SentOverConnection,
11330    /// The forwarding drain found no registered connection for the module: a
11331    /// subc child spawned moments ago that has not sent HELLO yet, or a
11332    /// `protocol: "none"` child, which never registers.
11333    NoConnection,
11334    /// This path sends nothing over the module's connection: the supervisor has
11335    /// no forwarding table, or the caller stops the child without a forwarding
11336    /// drain.
11337    NotSent,
11338}
11339
11340async fn begin_forwarding_drain(
11341    spec: &ModuleSpec,
11342    runtime: &SupervisorRuntimeConfig,
11343    registry: &Registry,
11344    snapshot: &SharedSnapshot,
11345    enabled: Option<bool>,
11346    reason: RouteCloseReason,
11347) -> Result<StopNotice, SuperviseError> {
11348    let Some(forwarding) = runtime.forwarding.as_ref() else {
11349        return Err(SuperviseError::ReloadUnavailable {
11350            module_id: spec.module_id.clone(),
11351            reason: "supervisor was not configured with a forwarding table".to_string(),
11352        });
11353    };
11354
11355    begin_forwarding_drain_with(
11356        forwarding,
11357        ForwardingDrainContext {
11358            spec,
11359            runtime,
11360            registry,
11361            scope: DrainScope::Active,
11362        },
11363        snapshot,
11364        enabled,
11365        reason,
11366        runtime.drain_timeout,
11367    )
11368    .await
11369}
11370
11371async fn begin_forwarding_drain_if_configured(
11372    spec: &ModuleSpec,
11373    runtime: &SupervisorRuntimeConfig,
11374    registry: &Registry,
11375    snapshot: &SharedSnapshot,
11376    enabled: Option<bool>,
11377    reason: RouteCloseReason,
11378) -> Result<StopNotice, SuperviseError> {
11379    begin_forwarding_drain_with_timeout(
11380        spec,
11381        runtime,
11382        registry,
11383        snapshot,
11384        enabled,
11385        reason,
11386        runtime.drain_timeout,
11387    )
11388    .await
11389}
11390
11391/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
11392/// budget, for paths where the operator overrides the module's configured one
11393/// (`supervisor.restart{drain_timeout_ms}`).
11394async fn begin_forwarding_drain_with_timeout(
11395    spec: &ModuleSpec,
11396    runtime: &SupervisorRuntimeConfig,
11397    registry: &Registry,
11398    snapshot: &SharedSnapshot,
11399    enabled: Option<bool>,
11400    reason: RouteCloseReason,
11401    drain_timeout: Duration,
11402) -> Result<StopNotice, SuperviseError> {
11403    let Some(forwarding) = runtime.forwarding.as_ref() else {
11404        return Ok(StopNotice::NotSent);
11405    };
11406
11407    begin_forwarding_drain_with(
11408        forwarding,
11409        ForwardingDrainContext {
11410            spec,
11411            runtime,
11412            registry,
11413            scope: DrainScope::Active,
11414        },
11415        snapshot,
11416        enabled,
11417        reason,
11418        drain_timeout,
11419    )
11420    .await
11421}
11422
11423async fn begin_forwarding_drain_with(
11424    forwarding: &ForwardingTable,
11425    context: ForwardingDrainContext<'_>,
11426    snapshot: &SharedSnapshot,
11427    enabled: Option<bool>,
11428    reason: RouteCloseReason,
11429    drain_timeout: Duration,
11430) -> Result<StopNotice, SuperviseError> {
11431    let ForwardingDrainContext {
11432        spec,
11433        runtime,
11434        registry,
11435        scope,
11436    } = context;
11437    debug_assert_ne!(reason, RouteCloseReason::Crash);
11438    let terminal = matches!(reason, RouteCloseReason::Disable);
11439    let drain_started_at = Instant::now();
11440    let drain_deadline = drain_started_at + drain_timeout;
11441    let deadline_ms =
11442        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11443    let busy_gauges = match scope {
11444        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11445        DrainScope::Endpoint(endpoint) => {
11446            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11447        }
11448    };
11449
11450    // Admission gate first: route.open/commit and route REQUEST admission are closed
11451    // before the first quiescence check, so the outstanding count can only fall.
11452    let gate_started = Instant::now();
11453    let drain_target = match scope {
11454        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11455        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11456    }
11457    .map_err(SuperviseError::Forwarding)?;
11458    // The instant admission closed, and how long taking the forwarding write
11459    // lock to close it took. The timeout line reports only the quiescence
11460    // wait, so without this a drain that started late looked like one that
11461    // started on time.
11462    info!(
11463        module_id = %spec.module_id,
11464        ?reason,
11465        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11466        connected = drain_target.is_some(),
11467        "module drain began; route admission closed"
11468    );
11469    if scope == DrainScope::Active {
11470        update_snapshot(snapshot, Some(&spec.module_id), |state| {
11471            state.state = ModuleState::Draining;
11472            state.draining_to_replace =
11473                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11474            if let Some(enabled) = enabled {
11475                state.enabled = enabled;
11476            }
11477        })?;
11478    }
11479
11480    let Some(target) = drain_target.as_ref() else {
11481        // Nothing was sent: the module has no registered connection to carry
11482        // `module.draining` or a GOODBYE. The caller must not assume the child
11483        // was asked to stop.
11484        return Ok(StopNotice::NoConnection);
11485    };
11486    {
11487        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11488        #[cfg(feature = "test-support")]
11489        forwarding.wait_before_route_closing_for_test().await;
11490        let routes = forwarding
11491            .endpoint_routes(target.endpoint)
11492            .map_err(SuperviseError::Forwarding)?;
11493        let routes_notified = routes.len();
11494        crate::control::send_route_control_pushes(
11495            forwarding,
11496            routes.clone(),
11497            ClientControlPush::RouteClosing {
11498                module_id: spec.module_id.clone(),
11499                channels: Vec::new(),
11500                reason,
11501            },
11502        );
11503        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11504
11505        // `route.closing` was just sent above: from here on every return path,
11506        // including an early one, MUST send `route.closed` before propagating
11507        // anything else. A client holds `closing` as a promise that a verdict is
11508        // coming; leaving early without `closed` strands it waiting forever, since
11509        // `closing` carries no timeout of its own.
11510        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11511        let wait_result = wait_for_forwarding_quiescence(
11512            forwarding,
11513            &spec.module_id,
11514            runtime,
11515            target.endpoint,
11516            drain_deadline,
11517            &busy_gauges,
11518            scope,
11519            &mut gauge_trace,
11520        )
11521        .await;
11522        let drained = drained_after_quiescence_wait(&wait_result);
11523        if let Err(err) = &wait_result {
11524            error!(
11525                module_id = %spec.module_id,
11526                ?reason,
11527                error = %err,
11528                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11529            );
11530        } else if !drained {
11531            // Read before teardown releases the routes.
11532            let holdouts = forwarding
11533                .endpoint_drain_holdouts(target.endpoint)
11534                .unwrap_or_default();
11535            warn_drain_timed_out(
11536                &spec.module_id,
11537                drain_timeout,
11538                reason,
11539                &holdouts,
11540                &gauge_trace,
11541            );
11542        }
11543        crate::control::send_route_control_pushes(
11544            forwarding,
11545            routes,
11546            ClientControlPush::RouteClosed {
11547                module_id: spec.module_id.clone(),
11548                channels: Vec::new(),
11549                reason,
11550                drained,
11551                abandoned: target.abandoned_bindings.len() as u32,
11552                excluded_subscriptions: target.excluded_subscriptions,
11553                terminal: Some(terminal),
11554            },
11555        );
11556        wait_result?;
11557
11558        // `route.closed` has now been sent unconditionally above. From here the
11559        // remaining steps are cleanup (route + module GOODBYE) rather than a
11560        // promise the client is waiting on, but a lock-poisoned
11561        // `release_module_endpoint_routes` would otherwise skip the module
11562        // GOODBYE silently too -- send it before propagating the error.
11563        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11564            Ok(routes) => routes,
11565            Err(err) => {
11566                warn!(
11567                    module_id = %spec.module_id,
11568                    ?reason,
11569                    error = %err,
11570                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11571                );
11572                send_module_goodbye(&spec.module_id, forwarding, target);
11573                return Err(SuperviseError::Forwarding(err));
11574            }
11575        };
11576        let route_goodbye_count = released_routes.len();
11577        send_route_goodbyes(forwarding, released_routes);
11578        send_module_goodbye(&spec.module_id, forwarding, target);
11579
11580        // The drain's happy path was previously silent: every emission above is
11581        // best-effort with only its failure arm logged, so "were consumers told"
11582        // was unprovable from the daemon log (surfaced by a 30-minute consumer
11583        // hang where the open question was exactly whether teardown notice went
11584        // out). One summary line makes that class decidable in one grep.
11585        info!(
11586            module_id = %spec.module_id,
11587            ?reason,
11588            routes_notified,
11589            route_goodbyes = route_goodbye_count,
11590            abandoned_reservations = target.abandoned_bindings.len(),
11591            excluded_subscriptions = target.excluded_subscriptions,
11592            drained,
11593            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11594        );
11595    }
11596
11597    Ok(StopNotice::SentOverConnection)
11598}
11599
11600/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
11601/// the only slot a plain (non-swap) spawn can register into.
11602async fn wait_for_registration_after_reload(
11603    registry: &Registry,
11604    module_id: &str,
11605    snapshot: &SharedSnapshot,
11606    child: &mut SupervisedChild,
11607    wait: Duration,
11608) -> Result<RegistrationWaitOutcome, SuperviseError> {
11609    wait_for_slot_registration(
11610        registry,
11611        crate::registry::RegistrationSlot::Active(module_id),
11612        module_id,
11613        snapshot,
11614        child,
11615        wait,
11616    )
11617    .await
11618}
11619
11620/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
11621///
11622/// Keyed on the slot rather than the bare module id because during a swap the
11623/// id's active slot is already held by the incumbent: an id-keyed wait would
11624/// report the incumbent's registration as the candidate's and a candidate that
11625/// never registers would look registered. A swap candidate waits on
11626/// `crate::registry::RegistrationSlot::Candidate`.
11627async fn wait_for_slot_registration(
11628    registry: &Registry,
11629    slot: crate::registry::RegistrationSlot<'_>,
11630    module_id: &str,
11631    snapshot: &SharedSnapshot,
11632    child: &mut SupervisedChild,
11633    wait: Duration,
11634) -> Result<RegistrationWaitOutcome, SuperviseError> {
11635    let deadline = Instant::now() + wait;
11636    loop {
11637        if registry
11638            .registration(slot)
11639            .map_err(SuperviseError::Registry)?
11640            .is_some()
11641        {
11642            return Ok(RegistrationWaitOutcome::Registered);
11643        }
11644
11645        let now = Instant::now();
11646        if now >= deadline {
11647            return Ok(RegistrationWaitOutcome::TimedOut);
11648        }
11649        let remaining = deadline.saturating_duration_since(now);
11650        let poll = remaining.min(REGISTRY_RELEASE_POLL);
11651
11652        tokio::select! {
11653            wait_result = child.wait() => {
11654                let status = wait_result.map_err(|source| SuperviseError::Wait {
11655                    module_id: module_id.to_string(),
11656                    source,
11657                })?;
11658                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11659                    snapshot,
11660                    child,
11661                    &status,
11662                )));
11663            }
11664            _ = sleep(poll) => {}
11665        }
11666    }
11667}
11668
11669fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11670    // A process stopped for missing its first HELLO did not provide service,
11671    // even if its TERM handler exits zero. Reload also uses this classification
11672    // when a replacement exits before registering.
11673    if exit_report.kind != ExitKind::DeliberateSeverance {
11674        exit_report.kind = ExitKind::Crash;
11675    }
11676    exit_report
11677}
11678
11679async fn handle_reload_child_registration_failure(
11680    spec: &ModuleSpec,
11681    runtime: &SupervisorRuntimeConfig,
11682    registry: &Registry,
11683    process_liveness: &SupervisorProcessLiveness,
11684    snapshot: &SharedSnapshot,
11685    _child: &mut Option<SupervisedChild>,
11686    failure: ReloadRegistrationFailure,
11687) -> Result<(), SuperviseError> {
11688    let ReloadRegistrationFailure {
11689        exit_report,
11690        reason,
11691    } = failure;
11692    match on_child_exit(
11693        spec,
11694        runtime.restart_policy,
11695        registry,
11696        snapshot,
11697        &runtime.terminal_ring,
11698        &runtime.spawn_events,
11699        &runtime.child_roster,
11700        exit_report,
11701        runtime.recovery_is_daemon,
11702    )
11703    .await
11704    {
11705        NextAction::Stop {
11706            registration_released,
11707        } => {
11708            if registration_released {
11709                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11710            }
11711        }
11712        NextAction::Restart { schedule } => {
11713            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11714                schedule.delay
11715            });
11716            if let Some(schedule) = schedule {
11717                log_crash_respawn(&spec.module_id, schedule);
11718            }
11719            schedule_respawn(
11720                runtime,
11721                snapshot,
11722                &spec.module_id,
11723                delay,
11724                RespawnKind::Spawn,
11725                true,
11726            )?;
11727        }
11728    }
11729    Err(SuperviseError::ReloadFailed {
11730        module_id: spec.module_id.clone(),
11731        reason,
11732    })
11733}
11734
11735async fn handle_reload_spawn_failure(
11736    spec: &ModuleSpec,
11737    runtime: &SupervisorRuntimeConfig,
11738    process_liveness: &SupervisorProcessLiveness,
11739    snapshot: &SharedSnapshot,
11740    _child: &mut Option<SupervisedChild>,
11741    reason: String,
11742) -> Result<(), SuperviseError> {
11743    let now = Instant::now();
11744    let mut schedule = None;
11745    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11746        clear_current_process_facts(state);
11747        if state.enabled {
11748            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11749            state.state = if schedule.is_some() {
11750                ModuleState::Restarting
11751            } else {
11752                if runtime
11753                    .failure_context(runtime.recovery_is_daemon)
11754                    .may_retry()
11755                {
11756                    state.failed_retry.schedule(&spec.module_id);
11757                }
11758                ModuleState::Failed
11759            };
11760        } else {
11761            state.state = ModuleState::Disabled;
11762        }
11763    })?;
11764    if let Some(schedule) = schedule {
11765        schedule_respawn(
11766            runtime,
11767            snapshot,
11768            &spec.module_id,
11769            schedule.delay,
11770            RespawnKind::Spawn,
11771            true,
11772        )?;
11773    } else {
11774        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11775    }
11776    Err(SuperviseError::ReloadFailed {
11777        module_id: spec.module_id.clone(),
11778        reason,
11779    })
11780}
11781
11782fn control_flags() -> Flags {
11783    Flags::new(false, Priority::Passive, false)
11784}
11785
11786#[allow(clippy::too_many_arguments)]
11787async fn drain_optional_child(
11788    module_id: &str,
11789    protocol: ModuleProtocol,
11790    stop_notice: StopNotice,
11791    registry: &Registry,
11792    forwarding: Option<&ForwardingTable>,
11793    snapshot: &SharedSnapshot,
11794    terminal_ring: &Arc<Mutex<TerminalRing>>,
11795    spawn_events: &SpawnEventFeed,
11796    child: &mut Option<SupervisedChild>,
11797    drain_timeout: Duration,
11798    final_state: ModuleState,
11799    enabled: Option<bool>,
11800    failure: &FailureContext,
11801) -> Result<(), SuperviseError> {
11802    if let Some(child) = child.take() {
11803        drain_child_to_state(
11804            module_id,
11805            protocol,
11806            stop_notice,
11807            registry,
11808            forwarding,
11809            snapshot,
11810            terminal_ring,
11811            spawn_events,
11812            child,
11813            drain_timeout,
11814            final_state,
11815            enabled,
11816            failure,
11817        )
11818        .await
11819    } else {
11820        update_snapshot(snapshot, Some(module_id), |state| {
11821            state.state = final_state;
11822            if final_state == ModuleState::Failed {
11823                state.failed_retry.failed_at = Some(Instant::now());
11824            }
11825            if let Some(enabled) = enabled {
11826                state.enabled = enabled;
11827            }
11828            clear_current_process_facts(state);
11829        })?;
11830        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11831    }
11832}
11833
11834#[allow(clippy::too_many_arguments)]
11835async fn drain_child_to_state(
11836    module_id: &str,
11837    _protocol: ModuleProtocol,
11838    stop_notice: StopNotice,
11839    registry: &Registry,
11840    forwarding: Option<&ForwardingTable>,
11841    snapshot: &SharedSnapshot,
11842    terminal_ring: &Arc<Mutex<TerminalRing>>,
11843    spawn_events: &SpawnEventFeed,
11844    mut child: SupervisedChild,
11845    drain_timeout: Duration,
11846    final_state: ModuleState,
11847    enabled: Option<bool>,
11848    failure: &FailureContext,
11849) -> Result<(), SuperviseError> {
11850    let protocol = child.protocol;
11851    update_snapshot(snapshot, Some(module_id), |state| {
11852        state.state = ModuleState::Draining;
11853        state.draining_to_replace = final_state == ModuleState::Restarting;
11854        if let Some(enabled) = enabled {
11855            state.enabled = enabled;
11856        }
11857    })?;
11858
11859    // The wait below is the same budget in every case; what differs is
11860    // whether anything has ASKED the child to stop before it starts. Only a
11861    // forwarding drain that reached the module's registered connection has
11862    // (`module.draining`, then a module GOODBYE). Every other child was told
11863    // nothing: a `protocol: "none"` module, which never registers; a subc
11864    // module spawned moments ago that has not sent HELLO yet; or a stop that
11865    // runs no forwarding drain. Without a signal the budget is only a delay
11866    // in front of SIGKILL -- and the not-yet-registered child is the worst
11867    // case, because it registers into a module that is already draining,
11868    // is never told, and is killed while healthy.
11869    if stop_notice != StopNotice::SentOverConnection {
11870        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11871            info!(
11872                module_id,
11873                pid = child.pid,
11874                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11875                "module has no connection yet; requesting stop by signal"
11876            );
11877        }
11878        request_graceful_stop(module_id, &child);
11879    }
11880
11881    let exit_report = match timeout(drain_timeout, child.wait()).await {
11882        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11883        Ok(Err(source)) => {
11884            fail_after_wait_error(
11885                module_id,
11886                snapshot,
11887                terminal_ring,
11888                spawn_events,
11889                format!("failed to wait during drain: {source}"),
11890                failure,
11891            );
11892            return Err(SuperviseError::Wait {
11893                module_id: module_id.to_string(),
11894                source,
11895            });
11896        }
11897        Err(_) => {
11898            // Mirror the sibling arm above: state is already `Draining`, and an
11899            // error propagated from here would strand it there -- a state
11900            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11901            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11902            // `Failed` before `?` keeps the module operator-visible and
11903            // revivable. Trigger is an ESRCH race (process exits between the
11904            // drain timeout firing and the kill) or a post-kill wait failure
11905            // (issue #34).
11906            //
11907            // Logged because the kill is otherwise visible only as signal 9 in
11908            // the terminal ring, and the budget it follows can be long enough
11909            // that consumers see a stretch of refusals with no stated cause.
11910            warn!(
11911                module_id,
11912                pid = child.pid,
11913                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11914                reason = ?final_state,
11915                ?stop_notice,
11916                "drain budget expired before the module exited; killing it"
11917            );
11918            let kill =
11919                if final_state == ModuleState::Restarting && child.child.container().is_some() {
11920                    child.kill_leader()
11921                } else {
11922                    child.start_kill()
11923                };
11924            kill.map_err(|source| {
11925                fail_after_wait_error(
11926                    module_id,
11927                    snapshot,
11928                    terminal_ring,
11929                    spawn_events,
11930                    format!("failed to kill during drain: {source}"),
11931                    failure,
11932                );
11933                SuperviseError::Kill {
11934                    module_id: module_id.to_string(),
11935                    source,
11936                }
11937            })?;
11938            let status = child.wait().await.map_err(|source| {
11939                fail_after_wait_error(
11940                    module_id,
11941                    snapshot,
11942                    terminal_ring,
11943                    spawn_events,
11944                    format!("failed to wait after drain kill: {source}"),
11945                    failure,
11946                );
11947                SuperviseError::Wait {
11948                    module_id: module_id.to_string(),
11949                    source,
11950                }
11951            })?;
11952            classify_reaped_child_exit(snapshot, &child, &status)
11953        }
11954    };
11955
11956    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11957    publish_after_terminal_record(
11958        module_id,
11959        || {
11960            record_terminal_with_detail(
11961                module_id,
11962                terminal_ring,
11963                spawn_events,
11964                &exit_report,
11965                terminal_disposition(final_state),
11966                detail,
11967            )
11968        },
11969        || {
11970            update_snapshot(snapshot, Some(module_id), |state| {
11971                state.state = final_state;
11972                if final_state == ModuleState::Failed {
11973                    state.failed_retry.failed_at = Some(Instant::now());
11974                }
11975                if let Some(enabled) = enabled {
11976                    state.enabled = enabled;
11977                }
11978                clear_current_process_facts(state);
11979                state.last_exit = Some(exit_report.clone());
11980                if exit_report.kind == ExitKind::DeliberateSeverance {
11981                    state.lifetime_restarts += 1;
11982                }
11983            })
11984        },
11985    )?;
11986    child.drain_stderr(module_id).await;
11987
11988    let released =
11989        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11990    child
11991        .finish_exit(
11992            released.is_ok()
11993                && final_state == ModuleState::Restarting
11994                && child.spawn_failure.is_none(),
11995        )
11996        .await
11997        .map_err(|source| SuperviseError::Wait {
11998            module_id: module_id.to_owned(),
11999            source,
12000        })?;
12001    released
12002}
12003
12004/// Ask a child that nothing else has asked to stop, by signal.
12005///
12006/// A registered subc module is asked over its own connection: the drain sends
12007/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
12008/// module GOODBYE, and the module stops itself. A module that speaks no subc
12009/// wire receives none of that, and neither does a subc module that has not
12010/// registered yet, so for them the drain budget would be pure delay in front of
12011/// a SIGKILL -- and for a process with a store to flush (JetStream is the
12012/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
12013/// into a recovery on the next start.
12014///
12015/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
12016/// rule rather than an optimisation: that module's graceful stop is already
12017/// running by the time its child is drained, and a signal would race it.
12018///
12019/// Best-effort by construction. A child that has already exited is the ordinary
12020/// case rather than an error (the kill lands on a reaped or exiting pid), so a
12021/// failure is logged at debug and the wait-then-kill below still decides the
12022/// outcome.
12023#[cfg(unix)]
12024fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12025    let Some(pid) = child
12026        .id()
12027        .and_then(|pid| i32::try_from(pid).ok())
12028        .and_then(rustix::process::Pid::from_raw)
12029    else {
12030        debug!(
12031            module_id,
12032            "no pid to signal for teardown; falling through to the drain wait"
12033        );
12034        return;
12035    };
12036    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12037        Ok(()) => debug!(
12038            module_id,
12039            "sent SIGTERM to a module nothing else asked to stop"
12040        ),
12041        Err(err) => debug!(
12042            module_id,
12043            error = %err,
12044            "SIGTERM to module failed; the drain wait and kill still apply"
12045        ),
12046    }
12047}
12048
12049/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
12050/// Windows does offer need cooperation this supervisor cannot assume: a console
12051/// control event requires sharing a console with the child, and `WM_CLOSE`
12052/// requires the child to pump a message loop. A supervised server process does
12053/// neither, so there is nothing to send and teardown is the wait followed by the
12054/// kill. Emulating a signal here would mean inventing a stop protocol, which is
12055/// the thing `protocol: "none"` exists to avoid.
12056#[cfg(not(unix))]
12057fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12058    debug!(
12059        module_id,
12060        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12061    );
12062}
12063
12064fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12065    match final_state {
12066        ModuleState::Stopped => TerminalDisposition::Stopped,
12067        ModuleState::Disabled => TerminalDisposition::Disabled,
12068        ModuleState::Restarting => TerminalDisposition::Restarting,
12069        ModuleState::Failed => TerminalDisposition::Failed,
12070        ModuleState::Starting
12071        | ModuleState::Running
12072        | ModuleState::Unresponsive
12073        | ModuleState::Draining => {
12074            unreachable!("terminal exits only finish in terminal or restarting states")
12075        }
12076    }
12077}
12078
12079/// Release a reaped child's registration before allowing another spawn.
12080///
12081/// EOF is not a process-lifetime signal: an inherited socket can stay open
12082/// indefinitely, and serial frame dispatch can be waiting on egress instead of
12083/// reading EOF. After the normal release grace, request connection close (which
12084/// cancels both reads and dispatch), then allow one more release grace for the
12085/// connection guard's forwarding cleanup. Never evict a different connection.
12086async fn release_dead_registration(
12087    registry: &Registry,
12088    forwarding: Option<&ForwardingTable>,
12089    snapshot: &SharedSnapshot,
12090    module_id: &str,
12091    failure: &FailureContext,
12092) -> Result<(), SuperviseError> {
12093    let result = async {
12094        let registration = registry
12095            .get_module(module_id)
12096            .map_err(SuperviseError::Registry)?;
12097        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12098            Ok(()) => return Ok(()),
12099            Err(SuperviseError::RegistrationStillActive { .. }) => {}
12100            Err(err) => return Err(err),
12101        }
12102        let pid = lock_snapshot(snapshot)?.reaped_pid;
12103        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12104            warn!(
12105                module_id,
12106                pid,
12107                connection_id = registration.connection_id.get(),
12108                "reaped module registration outlived release grace; closing dead connection"
12109            );
12110            forwarding.request_connection_close(
12111                registration.connection_id,
12112                CloseReason::new(
12113                    "supervised_process_reaped",
12114                    format!("module '{module_id}' pid {pid} exited"),
12115                ),
12116            );
12117            wait_for_slot_registration_release(
12118                registry,
12119                crate::registry::RegistrationSlot::Connection(registration.connection_id),
12120                REGISTRY_RELEASE_TIMEOUT,
12121            )
12122            .await?;
12123        }
12124        wait_for_registration_release(registry, module_id, Duration::ZERO).await
12125    }
12126    .await;
12127    if let Err(err) = &result {
12128        fail_snapshot(snapshot, Some(module_id), None, failure);
12129        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12130    }
12131    result
12132}
12133
12134/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
12135/// plain stop or restart waits for before it spawns a replacement.
12136async fn wait_for_registration_release(
12137    registry: &Registry,
12138    module_id: &str,
12139    wait: Duration,
12140) -> Result<(), SuperviseError> {
12141    wait_for_slot_registration_release(
12142        registry,
12143        crate::registry::RegistrationSlot::Active(module_id),
12144        wait,
12145    )
12146    .await
12147}
12148
12149/// Wait for the registration in `slot` to go away.
12150///
12151/// Keyed on the slot rather than the bare module id because a successful swap
12152/// never empties the id's active slot (the promoted candidate is in it), so an
12153/// id-keyed wait for the incumbent's release would always time out. Draining a
12154/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
12155/// incumbent's connection instead.
12156async fn wait_for_slot_registration_release(
12157    registry: &Registry,
12158    slot: crate::registry::RegistrationSlot<'_>,
12159    wait: Duration,
12160) -> Result<(), SuperviseError> {
12161    let deadline = Instant::now() + wait;
12162    let mut release_events = registration_release_events().subscribe();
12163    let still_active = |registration: &crate::registry::ModuleRegistration| {
12164        SuperviseError::RegistrationStillActive {
12165            module_id: registration.manifest.module_id.clone(),
12166            waited: wait,
12167        }
12168    };
12169    loop {
12170        let _observed_generation = *release_events.borrow_and_update();
12171        let Some(registration) = registry
12172            .registration(slot)
12173            .map_err(SuperviseError::Registry)?
12174        else {
12175            return Ok(());
12176        };
12177
12178        let now = Instant::now();
12179        if now >= deadline {
12180            return Err(still_active(&registration));
12181        }
12182
12183        let remaining = deadline.saturating_duration_since(now);
12184        match timeout(remaining, release_events.changed()).await {
12185            Ok(Ok(())) | Ok(Err(_)) => {}
12186            Err(_) => return Err(still_active(&registration)),
12187        }
12188    }
12189}
12190
12191#[cfg(test)]
12192mod slot_registration_wait_tests {
12193    use super::*;
12194    use crate::registry::{ConnectionId, RegistrationSlot};
12195    use subc_protocol::manifest::ModuleManifest;
12196
12197    #[tokio::test]
12198    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12199        let registry = Arc::new(Registry::default());
12200        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
12201        let runtime = supervisor.runtime_config();
12202        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12203        let spec = ModuleSpec {
12204            module_id: "enable-stale-registration".to_string(),
12205            program: PathBuf::from("/missing/enable-retry-test"),
12206            args: Vec::new(),
12207            env: Vec::new(),
12208            reserved: false,
12209            reserved_prefixes: Vec::new(),
12210            protocol: ModuleProtocol::Subc,
12211            overlap: Default::default(),
12212            carry_containment: false,
12213        };
12214        let connection = ConnectionId::new(90);
12215        registry
12216            .register_with_control_ops(
12217                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12218                1,
12219                connection,
12220                Vec::new(),
12221            )
12222            .unwrap();
12223        let mut child = None;
12224        let err = set_child_enabled(
12225            &spec,
12226            &runtime,
12227            &registry,
12228            &supervisor.process_liveness,
12229            &snapshot,
12230            &mut child,
12231            true,
12232            &runtime.failure_context(false),
12233        )
12234        .await
12235        .unwrap_err();
12236        assert!(matches!(
12237            err,
12238            SuperviseError::RegistrationStillActive { .. }
12239        ));
12240        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12241        assert!(child.is_none());
12242        registry.deregister_connection(connection).unwrap();
12243        let err = set_child_enabled(
12244            &spec,
12245            &runtime,
12246            &registry,
12247            &supervisor.process_liveness,
12248            &snapshot,
12249            &mut child,
12250            true,
12251            &runtime.failure_context(false),
12252        )
12253        .await
12254        .unwrap_err();
12255        assert!(
12256            matches!(err, SuperviseError::Spawn { .. }),
12257            "second enable must attempt a spawn: {err}"
12258        );
12259        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12260    }
12261
12262    const INCUMBENT: u64 = 1;
12263    const CANDIDATE: u64 = 2;
12264
12265    fn swapped_registry() -> Arc<Registry> {
12266        let registry = Arc::new(Registry::default());
12267        let manifest = ModuleManifest::builder("m", "0.1.0").build();
12268        registry
12269            .register_with_control_ops(
12270                manifest.clone(),
12271                1,
12272                ConnectionId::new(INCUMBENT),
12273                Vec::new(),
12274            )
12275            .unwrap();
12276        registry
12277            .register_candidate_with_control_ops(
12278                manifest,
12279                1,
12280                ConnectionId::new(CANDIDATE),
12281                Vec::new(),
12282            )
12283            .unwrap();
12284        registry
12285    }
12286
12287    /// After a promotion the id's active slot is held by the new process, so an
12288    /// id-keyed wait for the incumbent's release can never succeed; the
12289    /// connection-keyed wait completes as soon as the incumbent deregisters.
12290    #[tokio::test]
12291    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12292        let registry = swapped_registry();
12293        registry.promote_candidate("m").unwrap().unwrap();
12294
12295        assert!(matches!(
12296            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
12297            Err(SuperviseError::RegistrationStillActive { .. })
12298        ));
12299
12300        // Still held while the incumbent's connection has not deregistered.
12301        assert!(matches!(
12302            wait_for_slot_registration_release(
12303                &registry,
12304                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12305                Duration::from_millis(50),
12306            )
12307            .await,
12308            Err(SuperviseError::RegistrationStillActive { .. })
12309        ));
12310
12311        let releaser = Arc::clone(&registry);
12312        let release = tokio::spawn(async move {
12313            sleep(Duration::from_millis(20)).await;
12314            releaser
12315                .deregister_connection(ConnectionId::new(INCUMBENT))
12316                .unwrap();
12317            notify_registration_release();
12318        });
12319        wait_for_slot_registration_release(
12320            &registry,
12321            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12322            Duration::from_secs(5),
12323        )
12324        .await
12325        .expect("the incumbent's own registration is released");
12326        release.await.unwrap();
12327        assert!(registry.get_module("m").unwrap().is_some());
12328    }
12329
12330    /// The candidate slot is waited on separately from the active slot: the
12331    /// incumbent's registration neither holds up nor stands in for it.
12332    #[tokio::test]
12333    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12334        let registry = swapped_registry();
12335        assert!(matches!(
12336            wait_for_slot_registration_release(
12337                &registry,
12338                RegistrationSlot::Candidate("m"),
12339                Duration::from_millis(50),
12340            )
12341            .await,
12342            Err(SuperviseError::RegistrationStillActive { .. })
12343        ));
12344        registry
12345            .deregister_connection(ConnectionId::new(CANDIDATE))
12346            .unwrap();
12347        wait_for_slot_registration_release(
12348            &registry,
12349            RegistrationSlot::Candidate("m"),
12350            Duration::from_millis(50),
12351        )
12352        .await
12353        .expect("a candidate slot with no candidate is released");
12354        assert!(registry
12355            .registration(RegistrationSlot::Active("m"))
12356            .unwrap()
12357            .is_some());
12358    }
12359}
12360
12361fn classify_exit(status: &ExitStatus) -> ExitReport {
12362    ExitReport {
12363        kind: if status.success() {
12364            ExitKind::Clean
12365        } else {
12366            ExitKind::Crash
12367        },
12368        code: status.code(),
12369        signal: exit_signal(status),
12370        at_ms: unix_ms_now(),
12371    }
12372}
12373
12374/// The terminal record for a module whose `wait()` call itself errored (e.g. the
12375/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
12376/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
12377/// disposition still must be `Failed` so the terminal ring is not silently missing
12378/// an entry, matching what `fail_snapshot` records for this same arm.
12379fn wait_error_exit_report() -> ExitReport {
12380    ExitReport {
12381        kind: ExitKind::Crash,
12382        code: None,
12383        signal: None,
12384        at_ms: unix_ms_now(),
12385    }
12386}
12387
12388#[cfg(unix)]
12389fn exit_signal(status: &ExitStatus) -> Option<i32> {
12390    use std::os::unix::process::ExitStatusExt;
12391
12392    status.signal()
12393}
12394
12395#[cfg(not(unix))]
12396fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12397    None
12398}
12399
12400/// Give an operator-touched module its full crash budget back.
12401///
12402/// Named for the counter it used to zero; it now empties the in-window ring,
12403/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
12404/// ledger of what happened survives every operator action.
12405fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12406    update_snapshot(snapshot, Some(module_id), |state| {
12407        state.clear_crash_restarts();
12408    })
12409}
12410
12411fn set_running(
12412    snapshot: &SharedSnapshot,
12413    child: &SupervisedChild,
12414    module_id: &str,
12415    spawn_events: &SpawnEventFeed,
12416) -> Result<(), SuperviseError> {
12417    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12418        module_id: Some(module_id.to_string()),
12419    })?;
12420    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12421    if std::mem::take(&mut state.coalesced_restart_pending) {
12422        let generation = state.spawn_generation;
12423        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12424    }
12425    state.drain_disposition_detail = None;
12426    state.spawn_failure = None;
12427    state.waiting_for_previous_pid = None;
12428    state.waiting_for_previous_detail = None;
12429    // Every caller of this is a plain spawn, which always uses the primary key;
12430    // a promoted swap candidate sets the flag itself after this returns.
12431    state.in_alternate_slot = false;
12432    state.configuration_updated_since_spawn = false;
12433    state.spawned_protocol = Some(child.protocol);
12434    state.state = ModuleState::Running;
12435    state.enabled = true;
12436    state.process_alive = true;
12437    state.pid = child.id();
12438    state.subtree_target = child.subtree_target.clone();
12439    // Probe samples belong to one process. Keep module-level escalation and
12440    // late-answer history, but never charge a new child for its predecessor's misses.
12441    state.health.status = SupervisorHealthStatus::Unknown;
12442    state.last_probe_evidence = None;
12443    state.health.consecutive_failures = 0;
12444    state.health.detail = None;
12445    state.health.metrics = None;
12446    state.health.last_probe_ms = None;
12447    state.failed_retry.healthy_since = None;
12448    state.failed_retry.failed_at = None;
12449    state.failed_retry.running_since = Some(child.spawned_at);
12450    #[cfg(windows)]
12451    {
12452        state.pipe_consumed = child
12453            .nonce_delivery
12454            .as_ref()
12455            .map(|delivery| delivery.consumed());
12456    }
12457    #[cfg(target_os = "macos")]
12458    {
12459        state.report_ready = Some(Arc::clone(&child.report_ready));
12460    }
12461    state.spawned_at_ms = Some(child.spawned_at_ms);
12462    state.exec_outcome = Some(Arc::clone(&child.exec.0));
12463    state.spawned_from = Some(child.spawned_from.clone());
12464    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12465    state.spawned_file_identity = child.spawned_file_identity.clone();
12466    state.process_start_time = child.process_start_time;
12467    Ok(())
12468}
12469
12470fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12471    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12472    state.failed_retry.healthy_since = None;
12473    state.failed_retry.running_since = None;
12474    state.waiting_for_previous_pid = None;
12475    state.waiting_for_previous_detail = None;
12476    state.process_alive = false;
12477    state.spawned_protocol = None;
12478    state.pid = None;
12479    #[cfg(windows)]
12480    {
12481        state.pipe_consumed = None;
12482    }
12483    #[cfg(target_os = "macos")]
12484    {
12485        state.report_ready = None;
12486    }
12487    state.spawned_at_ms = None;
12488    state.exec_outcome = None;
12489    state.spawned_from = None;
12490    state.launch_spec_digest = None;
12491    state.spawned_file_identity = None;
12492    state.process_start_time = None;
12493    state.deliberate_severance = None;
12494}
12495
12496#[cfg(test)]
12497fn record_deliberate_severance(
12498    snapshot: &SharedSnapshot,
12499    identity: ProcessIdentity,
12500) -> Result<(), SuperviseError> {
12501    update_snapshot(snapshot, None, |state| {
12502        state.deliberate_severance = Some(identity);
12503    })
12504}
12505
12506fn apply_deliberate_severance_marker(
12507    snapshot: &SharedSnapshot,
12508    exited_identity: Option<ProcessIdentity>,
12509    mut exit_report: ExitReport,
12510) -> ExitReport {
12511    let marker = lock_snapshot(snapshot)
12512        .ok()
12513        .and_then(|mut state| state.deliberate_severance.take());
12514    if marker.is_some() && marker == exited_identity {
12515        exit_report.kind = ExitKind::DeliberateSeverance;
12516    }
12517    exit_report
12518}
12519
12520fn classify_reaped_child_exit(
12521    snapshot: &SharedSnapshot,
12522    child: &SupervisedChild,
12523    status: &ExitStatus,
12524) -> ExitReport {
12525    let _ = update_snapshot(snapshot, None, |state| {
12526        state.reaped_pid = Some(child.pid);
12527        state.spawn_failure = child.spawn_failure.clone();
12528    });
12529    if child.spawn_failure.as_deref() == Some("registration_timeout") {
12530        registration_failure_exit_report(classify_exit(status))
12531    } else {
12532        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12533    }
12534}
12535
12536fn fail_snapshot(
12537    snapshot: &SharedSnapshot,
12538    module_id: Option<&str>,
12539    last_exit: Option<ExitReport>,
12540    failure: &FailureContext,
12541) {
12542    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12543        state.state = ModuleState::Failed;
12544        clear_current_process_facts(state);
12545        if failure.may_retry() && state.enabled {
12546            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12547        } else if failure.shutdown.is_set() {
12548            state.failed_retry.cancel();
12549        }
12550        if let Some(last_exit) = last_exit {
12551            state.last_exit = Some(last_exit);
12552        }
12553    }) {
12554        error!(error = %err, "failed to mark supervisor state failed");
12555    }
12556}
12557
12558fn update_snapshot(
12559    snapshot: &SharedSnapshot,
12560    module_id: Option<&str>,
12561    update: impl FnOnce(&mut SupervisorSnapshot),
12562) -> Result<(), SuperviseError> {
12563    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12564        module_id: module_id.map(ToOwned::to_owned),
12565    })?;
12566    update(&mut state);
12567    Ok(())
12568}
12569
12570const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12571
12572fn lock_snapshot_for_control<'a>(
12573    snapshot: &'a SharedSnapshot,
12574    module_id: &str,
12575    caller: &'static str,
12576) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12577    let started_at = Instant::now();
12578    let guard = lock_snapshot(snapshot)?;
12579    let waited = started_at.elapsed();
12580    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12581        warn!(
12582            module_id = %module_id,
12583            waited_ms = waited.as_millis() as u64,
12584            caller = %caller,
12585            "slow snapshot lock"
12586        );
12587    }
12588    Ok(guard)
12589}
12590
12591fn lock_snapshot(
12592    snapshot: &SharedSnapshot,
12593) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12594    snapshot
12595        .lock()
12596        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12597}
12598
12599#[cfg(test)]
12600mod terminal_history_tests {
12601    use std::{
12602        path::PathBuf,
12603        sync::Arc,
12604        time::{Duration, Instant},
12605    };
12606
12607    use tokio::{sync::mpsc, time::sleep};
12608
12609    use super::{
12610        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12611        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12612        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12613        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12614        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12615        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12616        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12617        SupervisorHealthStatus, SupervisorSnapshot,
12618    };
12619    // The supervisor's clock, distinct from the `std::time::Instant` these tests
12620    // use for their own wall-clock deadlines: crash-restart instants must be on
12621    // the same clock the production code stamps them with, which is tokio's (and
12622    // is what `start_paused` tests can move).
12623    use super::Instant as ClockInstant;
12624    use crate::{
12625        registry::Registry,
12626        terminal_ring::{TerminalRing, TerminalRingConfig},
12627    };
12628    use std::sync::Mutex;
12629    use subc_control::TerminalDisposition;
12630
12631    /// See the twin in `control.rs` for why this derives the path from
12632    /// `current_exe()` and why the existence check is here: `--lib` alone does
12633    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
12634    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
12635    pub(super) fn fake_aft_stub_path() -> PathBuf {
12636        let mut path = std::env::current_exe().expect("current_exe available in tests");
12637        path.pop();
12638        path.pop();
12639        path.push(if cfg!(windows) {
12640            "fake-aft-stub.exe"
12641        } else {
12642            "fake-aft-stub"
12643        });
12644        assert!(
12645            path.exists(),
12646            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12647             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12648            path.display()
12649        );
12650        path
12651    }
12652
12653    #[test]
12654    fn reserved_never_spawned_refuses_every_hello() {
12655        // The canary hole: a reserved id whose module has never spawned had NO
12656        // gate entry and admitted anyone -- the reservation protected the nonce
12657        // holder, not the NAME. Now the entry is present with no legitimate
12658        // holder and refuses all comers.
12659        let supervisor = SupervisorHandle::default();
12660        supervisor.apply_identity_configuration(&ModuleSpec {
12661            module_id: "never-spawned".to_string(),
12662            program: PathBuf::from("/usr/bin/false"),
12663            args: Vec::new(),
12664            env: Vec::new(),
12665            reserved: true,
12666            reserved_prefixes: Vec::new(),
12667            protocol: ModuleProtocol::Subc,
12668            overlap: Default::default(),
12669            carry_containment: false,
12670        });
12671        assert!(
12672            supervisor
12673                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12674                .is_err(),
12675            "forged nonce must refuse on a reserved never-spawned id"
12676        );
12677        assert!(
12678            supervisor
12679                .reserved_hello_verification("never-spawned", None)
12680                .is_err(),
12681            "absent nonce must refuse on a reserved never-spawned id"
12682        );
12683        // And a real spawn nonce minted later admits exactly that nonce.
12684        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12685        supervisor.apply_identity_configuration(&ModuleSpec {
12686            module_id: "never-spawned".to_string(),
12687            program: PathBuf::from("/usr/bin/false"),
12688            args: Vec::new(),
12689            env: Vec::new(),
12690            reserved: true,
12691            reserved_prefixes: Vec::new(),
12692            protocol: ModuleProtocol::Subc,
12693            overlap: Default::default(),
12694            carry_containment: false,
12695        });
12696        assert!(supervisor
12697            .reserved_hello_verification("never-spawned", Some("minted"))
12698            .is_ok());
12699        assert!(supervisor
12700            .reserved_hello_verification("never-spawned", Some("forged"))
12701            .is_err());
12702    }
12703
12704    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12705    /// happened, which is what "spent budget" looks like to every reader.
12706    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12707        let now = ClockInstant::now();
12708        for _ in 0..count {
12709            state.crash_restarts.push_back(now);
12710        }
12711    }
12712
12713    /// Age the oldest recorded restart out of `window`, standing in for the hours
12714    /// that would otherwise have to pass. Injecting the instant is the point: a
12715    /// test that slept a real window would take ten minutes and still prove less.
12716    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12717        let aged = state
12718            .crash_restarts
12719            .front()
12720            .expect("a crash restart must be recorded before it can be aged")
12721            .checked_sub(window + Duration::from_secs(1))
12722            .expect("the test clock is far enough from its origin to age an instant");
12723        state.crash_restarts[0] = aged;
12724    }
12725
12726    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12727        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12728        seed_crash_restarts(&mut state, count);
12729        state
12730    }
12731
12732    #[test]
12733    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12734        let policy = RestartPolicy::new(3, Duration::ZERO);
12735        let now = ClockInstant::now();
12736        assert!(daemon_will_restart(
12737            &mut snapshot_with_restarts(true, 2),
12738            &policy,
12739            now
12740        ));
12741        assert!(!daemon_will_restart(
12742            &mut snapshot_with_restarts(true, 3),
12743            &policy,
12744            now
12745        ));
12746        assert!(!daemon_will_restart(
12747            &mut snapshot_with_restarts(false, 0),
12748            &policy,
12749            now
12750        ));
12751    }
12752
12753    #[test]
12754    fn crash_restart_backoff_escalates_with_in_window_count() {
12755        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12756            .with_max_backoff(Duration::from_secs(30));
12757        let now = ClockInstant::now();
12758        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12759        let schedules = (0..4)
12760            .map(|_| {
12761                state
12762                    .next_crash_restart(&policy, now)
12763                    .expect("the test policy allows four crash restarts")
12764            })
12765            .collect::<Vec<_>>();
12766
12767        assert_eq!(
12768            schedules
12769                .iter()
12770                .map(|schedule| schedule.restart_in_window)
12771                .collect::<Vec<_>>(),
12772            vec![0, 1, 2, 3]
12773        );
12774        assert_eq!(
12775            schedules
12776                .iter()
12777                .map(|schedule| schedule.delay)
12778                .collect::<Vec<_>>(),
12779            vec![
12780                Duration::from_millis(100),
12781                Duration::from_secs(1),
12782                Duration::from_secs(10),
12783                Duration::from_secs(30),
12784            ]
12785        );
12786    }
12787
12788    #[test]
12789    fn crash_restart_backoff_resets_after_ring_clear() {
12790        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12791        let now = ClockInstant::now();
12792        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12793        assert_eq!(
12794            state.next_crash_restart(&policy, now).unwrap().delay,
12795            Duration::from_millis(100)
12796        );
12797        assert_eq!(
12798            state.next_crash_restart(&policy, now).unwrap().delay,
12799            Duration::from_secs(1)
12800        );
12801
12802        state.clear_crash_restarts();
12803        let schedule = state
12804            .next_crash_restart(&policy, now)
12805            .expect("a cleared ring must allow another restart");
12806        assert_eq!(schedule.restart_in_window, 0);
12807        assert_eq!(schedule.delay, Duration::from_millis(100));
12808    }
12809
12810    #[test]
12811    fn crash_restart_backoff_ignores_aged_restarts() {
12812        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12813        let now = ClockInstant::now();
12814        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12815        state
12816            .next_crash_restart(&policy, now)
12817            .expect("the first restart is allowed");
12818        state
12819            .next_crash_restart(&policy, now)
12820            .expect("the second restart is allowed");
12821        state.crash_restarts[0] = now
12822            .checked_sub(policy.window + Duration::from_secs(1))
12823            .expect("the fake clock can age a restart past the window");
12824
12825        let schedule = state
12826            .next_crash_restart(&policy, now)
12827            .expect("an aged restart must release its slot");
12828        assert_eq!(schedule.restart_in_window, 1);
12829        assert_eq!(schedule.delay, Duration::from_secs(1));
12830        assert_eq!(state.crash_restarts.len(), 2);
12831    }
12832
12833    /// The budget is a rate: the same three spent restarts refuse a respawn
12834    /// while they are recent and allow one once they have aged past the window.
12835    /// Nothing about the module changed in between, which is the whole point.
12836    #[test]
12837    fn a_budget_spent_before_the_window_no_longer_refuses() {
12838        let policy = RestartPolicy::new(3, Duration::ZERO);
12839        let mut state = snapshot_with_restarts(true, 3);
12840        let now = ClockInstant::now();
12841        assert!(!daemon_will_restart(&mut state, &policy, now));
12842
12843        assert!(daemon_will_restart(
12844            &mut state,
12845            &policy,
12846            now + policy.window + Duration::from_secs(1)
12847        ));
12848        assert!(
12849            state.crash_restarts.is_empty(),
12850            "reading the budget must drop the instants that left the window"
12851        );
12852    }
12853
12854    fn module_with_recovery_snapshot(
12855        state: ModuleState,
12856        enabled: bool,
12857        restart_count: u32,
12858    ) -> SupervisedModule {
12859        let registry = Arc::new(Registry::default());
12860        let supervisor =
12861            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12862        let runtime = supervisor.runtime_config();
12863        let spec = ModuleSpec {
12864            module_id: "recovery-snapshot".to_string(),
12865            program: fake_aft_stub_path(),
12866            args: Vec::new(),
12867            env: Vec::new(),
12868            reserved: false,
12869            reserved_prefixes: Vec::new(),
12870            protocol: ModuleProtocol::Subc,
12871            overlap: Default::default(),
12872            carry_containment: false,
12873        };
12874        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12875        seed_crash_restarts(&mut snapshot, restart_count);
12876        // These tests read synthetic snapshots. A real child and monitor would
12877        // race those reads by replacing the requested state during startup.
12878        let (commands, _rx) = mpsc::channel(4);
12879        SupervisedModule {
12880            inner: Arc::new(SupervisedModuleInner {
12881                module_id: spec.module_id.clone(),
12882                registry,
12883                snapshot: Arc::new(Mutex::new(snapshot)),
12884                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12885                    spec,
12886                    health: runtime.health,
12887                })),
12888                stderr_ring: runtime.stderr_ring,
12889                terminal_ring: runtime.terminal_ring,
12890                commands,
12891                monitor: Mutex::new(None),
12892                restart_policy: runtime.restart_policy,
12893                effective_drain_timeout: runtime.effective_drain_timeout,
12894                provenance_probe: supervisor.provenance_probe.clone(),
12895                retained: runtime.child_roster.retained_view(),
12896            }),
12897        }
12898    }
12899
12900    #[cfg(target_os = "linux")]
12901    #[tokio::test]
12902    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12903        let supervisor =
12904            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12905                .with_cgroup_placement(None);
12906        let result = supervisor.spawn(ModuleSpec {
12907            module_id: "no-cgroup-placement".to_string(),
12908            program: fake_aft_stub_path(),
12909            args: Vec::new(),
12910            env: Vec::new(),
12911            reserved: false,
12912            reserved_prefixes: Vec::new(),
12913            protocol: ModuleProtocol::Subc,
12914            overlap: Default::default(),
12915            carry_containment: false,
12916        });
12917
12918        assert!(
12919            result.is_ok(),
12920            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12921        );
12922    }
12923
12924    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925    async fn undecided_snapshot_uses_shared_restart_predicate() {
12926        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12927            .will_recover_after_connection_loss()
12928            .unwrap());
12929        assert!(
12930            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12931                .will_recover_after_connection_loss()
12932                .unwrap()
12933        );
12934    }
12935
12936    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12937    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12938        assert!(
12939            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12940                .will_recover_after_connection_loss()
12941                .unwrap()
12942        );
12943    }
12944
12945    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12946    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12947        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12948            .will_recover_after_connection_loss()
12949            .unwrap());
12950        assert!(
12951            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12952                .will_recover_after_connection_loss()
12953                .unwrap()
12954        );
12955    }
12956
12957    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12958    async fn warming_snapshot_is_limited_to_startup_phases() {
12959        for state in [
12960            ModuleState::Starting,
12961            ModuleState::Running,
12962            ModuleState::Restarting,
12963        ] {
12964            assert!(
12965                module_with_recovery_snapshot(state, true, 0)
12966                    .is_warming()
12967                    .unwrap(),
12968                "{state:?} should be warming"
12969            );
12970        }
12971        for state in [
12972            ModuleState::Unresponsive,
12973            ModuleState::Draining,
12974            ModuleState::Stopped,
12975            ModuleState::Failed,
12976            ModuleState::Disabled,
12977        ] {
12978            assert!(
12979                !module_with_recovery_snapshot(state, true, 0)
12980                    .is_warming()
12981                    .unwrap(),
12982                "{state:?} should not be warming"
12983            );
12984        }
12985    }
12986
12987    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12988    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12989        let registry = Arc::new(Registry::default());
12990        let supervisor =
12991            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12992        let module = supervisor
12993            .spawn(ModuleSpec {
12994                module_id: "terminal-history".to_string(),
12995                program: fake_aft_stub_path(),
12996                args: Vec::new(),
12997                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12998                reserved: false,
12999                reserved_prefixes: Vec::new(),
13000                protocol: ModuleProtocol::Subc,
13001                overlap: Default::default(),
13002                carry_containment: false,
13003            })
13004            .unwrap();
13005
13006        let deadline = Instant::now() + Duration::from_secs(5);
13007        loop {
13008            let history = module.terminal_history();
13009            if history.entries.len() == 2 {
13010                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
13011                assert_eq!(history.dropped, 0);
13012                assert_eq!(
13013                    history
13014                        .entries
13015                        .iter()
13016                        .map(|entry| entry.exit_code)
13017                        .collect::<Vec<_>>(),
13018                    vec![Some(23), Some(23)]
13019                );
13020                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13021                return;
13022            }
13023            assert!(
13024                Instant::now() < deadline,
13025                "module did not retain two terminal exits: {history:?}"
13026            );
13027            sleep(Duration::from_millis(10)).await;
13028        }
13029    }
13030
13031    /// A disable issued while a crash respawn is still backing off must preempt
13032    /// that respawn: the operator's stop wins, the disable must not queue behind
13033    /// the backoff, and the module must never come back up afterwards.
13034    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13035    async fn disable_during_crash_backoff_cancels_pending_respawn() {
13036        let backoff = Duration::from_secs(2);
13037        let supervisor = Supervisor::new_for_test(
13038            Arc::new(Registry::default()),
13039            RestartPolicy::new(10, backoff),
13040        );
13041        let module = supervisor
13042            .spawn(ModuleSpec {
13043                module_id: "disable-during-backoff".to_string(),
13044                program: fake_aft_stub_path(),
13045                args: Vec::new(),
13046                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13047                reserved: false,
13048                reserved_prefixes: Vec::new(),
13049                protocol: ModuleProtocol::Subc,
13050                overlap: Default::default(),
13051                carry_containment: false,
13052            })
13053            .unwrap();
13054
13055        // Wait for the first crash to put the module into its backoff window.
13056        let deadline = Instant::now() + Duration::from_secs(5);
13057        loop {
13058            if module.status().unwrap().state == ModuleState::Restarting {
13059                break;
13060            }
13061            assert!(
13062                Instant::now() < deadline,
13063                "module never entered the crash backoff"
13064            );
13065            sleep(Duration::from_millis(10)).await;
13066        }
13067
13068        let started = Instant::now();
13069        module.set_enabled(false).await.unwrap();
13070        let waited = started.elapsed();
13071
13072        assert!(
13073            waited < backoff / 2,
13074            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13075        );
13076        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13077
13078        // Outlast the backoff: the respawn it was counting down to must never run.
13079        sleep(backoff + Duration::from_millis(500)).await;
13080        let status = module.status().unwrap();
13081        assert_eq!(status.state, ModuleState::Disabled);
13082        assert_eq!(
13083            status.spawn_generation, 1,
13084            "module respawned after the operator disabled it"
13085        );
13086    }
13087
13088    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
13089    /// the shape of nats-server, the program this rule exists for.
13090    #[cfg(unix)]
13091    fn protocol_none_sigterm_exits_clean_spec(
13092        module_id: &str,
13093        dir: &std::path::Path,
13094    ) -> (ModuleSpec, PathBuf, PathBuf) {
13095        let ready = dir.join("ready");
13096        let marker = dir.join("sigterm");
13097        let spec = ModuleSpec {
13098            module_id: module_id.to_string(),
13099            program: fake_aft_stub_path(),
13100            args: Vec::new(),
13101            env: vec![
13102                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13103                (
13104                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13105                    marker.display().to_string(),
13106                ),
13107                (
13108                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13109                    ready.display().to_string(),
13110                ),
13111            ],
13112            reserved: false,
13113            reserved_prefixes: Vec::new(),
13114            protocol: ModuleProtocol::None,
13115            overlap: Default::default(),
13116            carry_containment: false,
13117        };
13118        (spec, ready, marker)
13119    }
13120
13121    /// Wait for a file the child writes, so a signal is never sent before the
13122    /// child's SIGTERM handler is installed (the default disposition would
13123    /// kill it by signal and the exit would not be clean).
13124    #[cfg(unix)]
13125    async fn wait_for_file(path: &std::path::Path) {
13126        let deadline = Instant::now() + Duration::from_secs(10);
13127        while !path.exists() {
13128            assert!(
13129                Instant::now() < deadline,
13130                "{} never appeared",
13131                path.display()
13132            );
13133            sleep(Duration::from_millis(10)).await;
13134        }
13135    }
13136
13137    /// A protocol-none module that exits 0 because something OUTSIDE the
13138    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
13139    /// the crash-path disposition rather than `stopped`.
13140    #[cfg(unix)]
13141    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13142    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13143        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13144        let (spec, ready, marker) =
13145            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13146        let supervisor = Supervisor::new_for_test(
13147            Arc::new(Registry::default()),
13148            RestartPolicy::new(3, Duration::ZERO),
13149        );
13150        let module = supervisor.spawn(spec).unwrap();
13151        wait_for_file(&ready).await;
13152        // The ready file proves the child installed its SIGTERM handler, not
13153        // that the supervisor has processed the privacy trampoline's exec
13154        // acknowledgement. On macOS status withholds the pid until then.
13155        let deadline = Instant::now() + Duration::from_secs(10);
13156        let first_pid = loop {
13157            let status = module.status().unwrap();
13158            if status.state == ModuleState::Running {
13159                if let Some(pid) = status.pid {
13160                    break pid;
13161                }
13162            }
13163            assert!(
13164                Instant::now() < deadline,
13165                "a running module must report its pid after exec confirmation: {status:?}"
13166            );
13167            sleep(Duration::from_millis(10)).await;
13168        };
13169
13170        rustix::process::kill_process(
13171            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13172            rustix::process::Signal::TERM,
13173        )
13174        .unwrap();
13175
13176        let deadline = Instant::now() + Duration::from_secs(10);
13177        let respawned = loop {
13178            let status = module.status().unwrap();
13179            if status.state == ModuleState::Running
13180                && status.pid.is_some_and(|pid| pid != first_pid)
13181            {
13182                break status;
13183            }
13184            assert!(
13185                Instant::now() < deadline,
13186                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13187            );
13188            sleep(Duration::from_millis(10)).await;
13189        };
13190        assert_eq!(respawned.spawn_generation, 2);
13191        assert!(
13192            marker.exists(),
13193            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13194        );
13195
13196        let history = module.terminal_history();
13197        assert_eq!(history.entries.len(), 1, "{history:?}");
13198        let entry = &history.entries[0];
13199        assert_eq!(entry.exit_code, Some(0));
13200        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13201        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13202
13203        module.stop().await.unwrap();
13204    }
13205
13206    /// Repeated unrequested clean exits of a protocol-none module spend the
13207    /// restart budget exactly as crashes do, and the module ends `failed` with
13208    /// the budget named.
13209    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13210    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13211        let supervisor = Supervisor::new_for_test(
13212            Arc::new(Registry::default()),
13213            RestartPolicy::new(1, Duration::ZERO),
13214        );
13215        let module = supervisor
13216            .spawn(ModuleSpec {
13217                module_id: "none-clean-exit-budget".to_string(),
13218                program: fake_aft_stub_path(),
13219                args: Vec::new(),
13220                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13221                reserved: false,
13222                reserved_prefixes: Vec::new(),
13223                protocol: ModuleProtocol::None,
13224                overlap: Default::default(),
13225                carry_containment: false,
13226            })
13227            .unwrap();
13228
13229        // Failed follows the terminal write; this deadline only bounds a hang,
13230        // not an assumed duration for the two launches or their exit recording.
13231        let deadline = Instant::now() + Duration::from_secs(10);
13232        loop {
13233            let status = module.status().unwrap();
13234            if status.state == ModuleState::Failed {
13235                break;
13236            }
13237            assert!(
13238                Instant::now() < deadline,
13239                "module never exhausted its budget: {status:?} {:?}",
13240                module.terminal_history()
13241            );
13242            sleep(Duration::from_millis(10)).await;
13243        }
13244        let history = module.terminal_history();
13245        assert_eq!(
13246            history
13247                .entries
13248                .iter()
13249                .map(|entry| (entry.exit_code, entry.disposition.clone()))
13250                .collect::<Vec<_>>(),
13251            vec![
13252                (Some(0), TerminalDisposition::Restarting),
13253                (Some(0), TerminalDisposition::Failed),
13254            ]
13255        );
13256        let detail = history.entries[1]
13257            .disposition_detail
13258            .as_deref()
13259            .expect("a budget failure names the budget");
13260        assert!(detail.contains("max_restarts=1"), "{detail}");
13261        assert_eq!(module.status().unwrap().spawn_generation, 2);
13262    }
13263
13264    #[test]
13265    fn restart_budget_failure_is_published_after_its_terminal_record() {
13266        use std::io::{Read, Write};
13267        let supervisor = Supervisor::new_for_test(
13268            Arc::new(Registry::default()),
13269            RestartPolicy::new(0, Duration::ZERO),
13270        );
13271        let runtime = supervisor.runtime_config();
13272        let spec = ModuleSpec {
13273            protocol: ModuleProtocol::None,
13274            ..windowed_crash_spec("budget-publication-order")
13275        };
13276        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13277            ModuleState::Running,
13278            true,
13279        )));
13280        lock_snapshot(&snapshot).unwrap().process_alive = true;
13281        let reaped_snapshot = snapshot.clone();
13282        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13283        let history = runtime.terminal_ring.clone();
13284        let reap = std::thread::spawn(move || {
13285            tokio::runtime::Builder::new_current_thread()
13286                .enable_all()
13287                .build()
13288                .unwrap()
13289                .block_on(on_child_exit(
13290                    &spec,
13291                    runtime.restart_policy,
13292                    &supervisor.registry,
13293                    &reaped_snapshot,
13294                    &runtime.terminal_ring,
13295                    &runtime.spawn_events,
13296                    &runtime.child_roster,
13297                    ExitReport {
13298                        kind: ExitKind::Clean,
13299                        code: Some(0),
13300                        signal: None,
13301                        at_ms: 1,
13302                    },
13303                    true,
13304                ))
13305        });
13306        let mut reached = [0];
13307        barrier.read_exact(&mut reached).unwrap();
13308        assert_eq!(&reached, b"R");
13309        let (before_state, before_alive) = {
13310            let state = lock_snapshot(&snapshot).unwrap();
13311            (state.state, state.process_alive)
13312        };
13313        let before_history = history.lock().unwrap().snapshot();
13314        barrier.write_all(b"X").unwrap();
13315        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13316        assert!(before_history.entries.is_empty());
13317        assert_ne!(
13318            before_state,
13319            ModuleState::Failed,
13320            "Failed was visible before its terminal record could be written"
13321        );
13322        assert!(before_alive);
13323        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13324        let history = history.lock().unwrap().snapshot();
13325        assert_eq!(history.entries.len(), 1);
13326        assert_eq!(history.entries[0].exit_code, Some(0));
13327        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13328    }
13329
13330    #[test]
13331    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13332        use std::io::{Read, Write};
13333
13334        let supervisor = Supervisor::new_for_test(
13335            Arc::new(Registry::default()),
13336            RestartPolicy::new(3, Duration::ZERO),
13337        );
13338        let mut runtime = supervisor.runtime_config();
13339        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13340        let snapshot = module.inner.snapshot.clone();
13341        lock_snapshot(&snapshot).unwrap().process_alive = true;
13342        runtime.terminal_ring = module.inner.terminal_ring.clone();
13343        let spec = windowed_crash_spec(module.module_id());
13344        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13345        let reaped_snapshot = snapshot.clone();
13346        let reap = std::thread::spawn(move || {
13347            tokio::runtime::Builder::new_current_thread()
13348                .enable_all()
13349                .build()
13350                .unwrap()
13351                .block_on(on_child_exit(
13352                    &spec,
13353                    runtime.restart_policy,
13354                    &supervisor.registry,
13355                    &reaped_snapshot,
13356                    &runtime.terminal_ring,
13357                    &runtime.spawn_events,
13358                    &runtime.child_roster,
13359                    ExitReport {
13360                        kind: ExitKind::Crash,
13361                        code: Some(2),
13362                        signal: None,
13363                        at_ms: 1,
13364                    },
13365                    true,
13366                ))
13367        });
13368
13369        // Hold the crash handler between its budget decision and publication.
13370        // Connection cleanup must still see the recovery that this crash chose,
13371        // even when that recovery consumes the last available restart slot.
13372        let mut reached = [0];
13373        barrier.read_exact(&mut reached).unwrap();
13374        assert_eq!(&reached, b"R");
13375        let before_status = module.status().unwrap();
13376        let before_history = module.terminal_history();
13377        let before_recovers = module.will_recover_after_connection_loss().unwrap();
13378        barrier.write_all(b"X").unwrap();
13379        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13380
13381        assert_eq!(before_status.state, ModuleState::Running);
13382        assert!(before_status.process_alive);
13383        assert_eq!(before_status.restart_count, 3);
13384        assert!(before_history.entries.is_empty());
13385        assert!(
13386            before_recovers,
13387            "connection cleanup would report a terminal close for a scheduled crash restart"
13388        );
13389        let restarting = module.status().unwrap();
13390        assert_eq!(restarting.state, ModuleState::Restarting);
13391        assert_eq!(restarting.restart_count, 3);
13392        assert!(module.will_recover_after_connection_loss().unwrap());
13393        let history = module.terminal_history();
13394        assert_eq!(history.entries.len(), 1);
13395        assert_eq!(
13396            history.entries[0].disposition,
13397            TerminalDisposition::Restarting
13398        );
13399
13400        // Once the replacement is running, its next crash must be judged
13401        // against the exhausted budget rather than the previous recovery.
13402        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13403            state.state = ModuleState::Running;
13404            state.respawn_pending = false;
13405        })
13406        .unwrap();
13407        assert!(!module.will_recover_after_connection_loss().unwrap());
13408        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13409            state.enabled = false;
13410            state.respawn_pending = true;
13411        })
13412        .unwrap();
13413        assert!(!module.will_recover_after_connection_loss().unwrap());
13414    }
13415
13416    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13417        use std::net::{TcpListener, TcpStream};
13418        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13419        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13420        let (barrier, _) = listener.accept().unwrap();
13421        for stream in [&writer, &barrier] {
13422            stream
13423                .set_read_timeout(Some(Duration::from_secs(10)))
13424                .unwrap();
13425            stream
13426                .set_write_timeout(Some(Duration::from_secs(10)))
13427                .unwrap();
13428        }
13429        *events.1.lock().unwrap() = Some(writer);
13430        barrier
13431    }
13432
13433    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13434        use std::io::{Read, Write};
13435        let barrier = events.1.lock().unwrap().take();
13436        if let Some(mut barrier) = barrier {
13437            barrier.write_all(b"R").unwrap();
13438            let mut release = [0];
13439            barrier.read_exact(&mut release).unwrap();
13440            assert_eq!(&release, b"X");
13441        }
13442    }
13443
13444    #[test]
13445    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13446        use std::io::{Read, Write};
13447
13448        let supervisor = Supervisor::new_for_test(
13449            Arc::new(Registry::default()),
13450            RestartPolicy::new(0, Duration::ZERO),
13451        );
13452        let mut runtime = supervisor.runtime_config();
13453        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13454        let snapshot = module.inner.snapshot.clone();
13455        runtime.terminal_ring = module.inner.terminal_ring.clone();
13456        let spec = ModuleSpec {
13457            module_id: module.module_id().into(),
13458            program: fake_aft_stub_path(),
13459            protocol: ModuleProtocol::None,
13460            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13461            ..windowed_crash_spec(module.module_id())
13462        };
13463        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13464        let reap = std::thread::spawn(move || {
13465            tokio::runtime::Builder::new_current_thread()
13466                .enable_all()
13467                .build()
13468                .unwrap()
13469                .block_on(async {
13470                    let mut child =
13471                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13472                    health_restart_child(
13473                        &spec,
13474                        &runtime,
13475                        &supervisor.registry,
13476                        &supervisor.process_liveness,
13477                        &snapshot,
13478                        &mut child,
13479                        SupervisorHealthStatus::Failing,
13480                        Some("health probe did not answer"),
13481                        1,
13482                    )
13483                    .await
13484                    .unwrap();
13485                    assert!(child.is_none());
13486                });
13487        });
13488        // The loopback barrier acknowledges entry into terminal recording by
13489        // health_restart_child's drain, then waits for the test's release byte.
13490        // Status and history are observed while that writer is held, not after
13491        // an elapsed-time guess about when the health action has completed.
13492        let mut reached = [0];
13493        barrier.read_exact(&mut reached).unwrap();
13494        assert_eq!(&reached, b"R");
13495        let before_status = module.status().unwrap();
13496        let before_history = module.terminal_history();
13497        barrier.write_all(b"X").unwrap();
13498        reap.join().unwrap();
13499        assert!(before_history.entries.is_empty());
13500        assert_ne!(
13501            before_status.state,
13502            ModuleState::Failed,
13503            "Failed was visible before its terminal record could be written"
13504        );
13505        assert!(
13506            before_status.process_alive,
13507            "process death was visible before its terminal record could be written"
13508        );
13509        let failed = module.status().unwrap();
13510        assert_eq!(failed.state, ModuleState::Failed);
13511        assert!(!failed.process_alive);
13512        let history = module.terminal_history();
13513        assert_eq!(history.entries.len(), 1);
13514        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13515        assert_eq!(
13516            history.entries[0].disposition_detail.as_deref(),
13517            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13518        );
13519    }
13520
13521    #[test]
13522    fn observed_exits_publish_process_death_after_terminal_history() {
13523        use std::io::{Read, Write};
13524        for (kind, enabled, shutdown, wait_error, expected_state) in [
13525            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13526            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13527            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13528            (
13529                ExitKind::DeliberateSeverance,
13530                true,
13531                false,
13532                false,
13533                ModuleState::Restarting,
13534            ),
13535            #[cfg(unix)]
13536            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13537            (ExitKind::Crash, true, false, true, ModuleState::Failed),
13538        ] {
13539            let supervisor = Supervisor::new_for_test(
13540                Arc::new(Registry::default()),
13541                RestartPolicy::new(1, Duration::ZERO),
13542            );
13543            let runtime = supervisor.runtime_config();
13544            #[cfg(unix)]
13545            if shutdown {
13546                runtime.child_roster.close();
13547            }
13548            #[cfg(not(unix))]
13549            let _ = shutdown;
13550            let spec = windowed_crash_spec("exit-publication-order");
13551            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13552                ModuleState::Running,
13553                enabled,
13554            )));
13555            lock_snapshot(&snapshot).unwrap().process_alive = true;
13556            let reaped_snapshot = snapshot.clone();
13557            let history = runtime.terminal_ring.clone();
13558            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13559            let reap = std::thread::spawn(move || {
13560                if wait_error {
13561                    super::fail_after_wait_error(
13562                        &spec.module_id,
13563                        &reaped_snapshot,
13564                        &runtime.terminal_ring,
13565                        &runtime.spawn_events,
13566                        "wait failed".into(),
13567                        &runtime.failure_context(true),
13568                    );
13569                } else {
13570                    tokio::runtime::Builder::new_current_thread()
13571                        .enable_all()
13572                        .build()
13573                        .unwrap()
13574                        .block_on(on_child_exit(
13575                            &spec,
13576                            runtime.restart_policy,
13577                            &supervisor.registry,
13578                            &reaped_snapshot,
13579                            &runtime.terminal_ring,
13580                            &runtime.spawn_events,
13581                            &runtime.child_roster,
13582                            ExitReport {
13583                                kind,
13584                                code: Some(0),
13585                                signal: None,
13586                                at_ms: 1,
13587                            },
13588                            true,
13589                        ));
13590                }
13591            });
13592            let mut reached = [0];
13593            barrier.read_exact(&mut reached).unwrap();
13594            assert_eq!(&reached, b"R");
13595            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13596            let before_history = history.lock().unwrap().snapshot();
13597            barrier.write_all(b"X").unwrap();
13598            reap.join().unwrap();
13599            assert!(
13600                before_alive,
13601                "{kind:?}: death was published before recording"
13602            );
13603            assert!(before_history.entries.is_empty());
13604            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13605            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13606            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13607        }
13608    }
13609
13610    #[tokio::test]
13611    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13612        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13613        let path = dir.join("terminals.jsonl");
13614        std::fs::create_dir(&path).unwrap();
13615        let supervisor = Supervisor::new_for_test(
13616            Arc::new(Registry::default()),
13617            RestartPolicy::new(0, Duration::ZERO),
13618        )
13619        .with_terminal_journal(path, "health-budget-journal-failure".into());
13620        let runtime = supervisor.runtime_config();
13621        let spec = ModuleSpec {
13622            program: fake_aft_stub_path(),
13623            protocol: ModuleProtocol::None,
13624            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13625            ..windowed_crash_spec("health-budget-journal-failure")
13626        };
13627        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13628        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13629        health_restart_child(
13630            &spec,
13631            &runtime,
13632            &supervisor.registry,
13633            &supervisor.process_liveness,
13634            &snapshot,
13635            &mut child,
13636            SupervisorHealthStatus::Failing,
13637            None,
13638            1,
13639        )
13640        .await
13641        .unwrap();
13642        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13643        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13644        let history = runtime
13645            .terminal_ring
13646            .lock()
13647            .unwrap()
13648            .durable_history(&spec.module_id);
13649        assert!(history.journal_write_failures > 0);
13650        assert_eq!(history.entries.len(), 1);
13651        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13652    }
13653
13654    #[test]
13655    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13656        let supervisor = Supervisor::new_for_test(
13657            Arc::new(Registry::default()),
13658            RestartPolicy::new(0, Duration::ZERO),
13659        );
13660        let runtime = supervisor.runtime_config();
13661        let spec = ModuleSpec {
13662            program: fake_aft_stub_path(),
13663            protocol: ModuleProtocol::None,
13664            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13665            ..windowed_crash_spec("health-budget-recording-panic")
13666        };
13667        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13668        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13669            tokio::runtime::Builder::new_current_thread()
13670                .enable_all()
13671                .build()
13672                .unwrap()
13673                .block_on(async {
13674                    let mut child =
13675                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13676                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13677                    health_restart_child(
13678                        &spec,
13679                        &runtime,
13680                        &supervisor.registry,
13681                        &supervisor.process_liveness,
13682                        &snapshot,
13683                        &mut child,
13684                        SupervisorHealthStatus::Failing,
13685                        None,
13686                        1,
13687                    )
13688                    .await
13689                    .unwrap();
13690                });
13691        }));
13692        let panic = result.expect_err("recording must still unwind");
13693        assert_eq!(
13694            panic.downcast_ref::<String>().map(String::as_str),
13695            Some("spawn event sequence exhausted")
13696        );
13697        let state = lock_snapshot(&snapshot).unwrap();
13698        assert_eq!(state.state, ModuleState::Failed);
13699        assert!(!state.process_alive);
13700        assert!(state.last_exit.is_some());
13701    }
13702
13703    #[tokio::test]
13704    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13705        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13706        let path = dir.join("terminals.jsonl");
13707        std::fs::create_dir(&path).unwrap();
13708        let supervisor = Supervisor::new_for_test(
13709            Arc::new(Registry::default()),
13710            RestartPolicy::new(0, Duration::ZERO),
13711        )
13712        .with_terminal_journal(path, "budget-journal-failure".into());
13713        let runtime = supervisor.runtime_config();
13714        let spec = windowed_crash_spec("budget-journal-failure");
13715        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13716        assert!(matches!(
13717            on_child_exit(
13718                &spec,
13719                runtime.restart_policy,
13720                &supervisor.registry,
13721                &snapshot,
13722                &runtime.terminal_ring,
13723                &runtime.spawn_events,
13724                &runtime.child_roster,
13725                crash_exit_report(1),
13726                true,
13727            )
13728            .await,
13729            NextAction::Stop { .. }
13730        ));
13731        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13732        let history = runtime
13733            .terminal_ring
13734            .lock()
13735            .unwrap()
13736            .durable_history(&spec.module_id);
13737        assert!(history.journal_write_failures > 0);
13738        assert_eq!(history.entries.len(), 1);
13739        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13740    }
13741
13742    #[test]
13743    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13744        let supervisor = Supervisor::new_for_test(
13745            Arc::new(Registry::default()),
13746            RestartPolicy::new(0, Duration::ZERO),
13747        );
13748        let runtime = supervisor.runtime_config();
13749        let spec = windowed_crash_spec("budget-recording-panic");
13750        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13751        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13752        // Exhausting the event sequence makes emit_exited panic before the
13753        // terminal write, exercising publication on the recording unwind.
13754        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13755        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13756            tokio::runtime::Builder::new_current_thread()
13757                .enable_all()
13758                .build()
13759                .unwrap()
13760                .block_on(on_child_exit(
13761                    &spec,
13762                    runtime.restart_policy,
13763                    &supervisor.registry,
13764                    &snapshot,
13765                    &runtime.terminal_ring,
13766                    &runtime.spawn_events,
13767                    &runtime.child_roster,
13768                    crash_exit_report(1),
13769                    true,
13770                ))
13771        }));
13772        let panic = result.err().expect("recording must still unwind");
13773        assert_eq!(
13774            panic.downcast_ref::<String>().map(String::as_str),
13775            Some("spawn event sequence exhausted")
13776        );
13777        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13778    }
13779
13780    /// A stop the supervisor itself requests still stops a protocol-none
13781    /// module, even though the child answers the SIGTERM with exit 0.
13782    #[cfg(unix)]
13783    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13784    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13785        for disable in [false, true] {
13786            let label = if disable {
13787                "none-requested-disable"
13788            } else {
13789                "none-requested-stop"
13790            };
13791            let dir = cortexkit_test_support::ScratchDir::new(label);
13792            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13793            let supervisor = Supervisor::new_for_test(
13794                Arc::new(Registry::default()),
13795                RestartPolicy::new(3, Duration::ZERO),
13796            );
13797            let module = supervisor.spawn(spec).unwrap();
13798            wait_for_file(&ready).await;
13799
13800            if disable {
13801                module.set_enabled(false).await.unwrap();
13802            } else {
13803                module.stop().await.unwrap();
13804            }
13805            assert!(
13806                marker.exists(),
13807                "{label}: the child must have left through its SIGTERM handler with exit 0"
13808            );
13809
13810            // Long enough for a zero-backoff respawn to have happened if the
13811            // exit had been treated as a crash.
13812            sleep(Duration::from_millis(500)).await;
13813            let status = module.status().unwrap();
13814            let expected = if disable {
13815                ModuleState::Disabled
13816            } else {
13817                ModuleState::Stopped
13818            };
13819            assert_eq!(status.state, expected, "{label}");
13820            assert_eq!(
13821                status.spawn_generation, 1,
13822                "{label}: respawned after a requested stop"
13823            );
13824            let history = module.terminal_history();
13825            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13826            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13827            assert_ne!(
13828                history.entries[0].disposition,
13829                TerminalDisposition::Restarting,
13830                "{label}"
13831            );
13832        }
13833    }
13834
13835    /// A subc-wire module that exits 0 on its own is still a stop: the
13836    /// protocol-none rule must not reach it.
13837    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13838    async fn subc_wire_clean_exit_is_still_a_stop() {
13839        let supervisor = Supervisor::new_for_test(
13840            Arc::new(Registry::default()),
13841            RestartPolicy::new(3, Duration::ZERO),
13842        );
13843        let module = supervisor
13844            .spawn(ModuleSpec {
13845                module_id: "wire-clean-exit".to_string(),
13846                program: fake_aft_stub_path(),
13847                args: Vec::new(),
13848                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13849                reserved: false,
13850                reserved_prefixes: Vec::new(),
13851                protocol: ModuleProtocol::Subc,
13852                overlap: Default::default(),
13853                carry_containment: false,
13854            })
13855            .unwrap();
13856
13857        let deadline = Instant::now() + Duration::from_secs(10);
13858        while module.terminal_history().entries.is_empty() {
13859            assert!(Instant::now() < deadline, "module never exited");
13860            sleep(Duration::from_millis(10)).await;
13861        }
13862        // Long enough for a zero-backoff respawn to have happened.
13863        sleep(Duration::from_millis(500)).await;
13864        let status = module.status().unwrap();
13865        assert_eq!(status.state, ModuleState::Stopped);
13866        assert_eq!(status.spawn_generation, 1);
13867        let history = module.terminal_history();
13868        assert_eq!(history.entries.len(), 1, "{history:?}");
13869        assert_eq!(history.entries[0].exit_code, Some(0));
13870        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13871    }
13872
13873    #[cfg(unix)]
13874    #[tokio::test]
13875    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13876        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13877        let record = dir.join("live-children.json");
13878        let supervisor = Supervisor::new_for_test(
13879            Arc::new(Registry::default()),
13880            RestartPolicy::new(0, Duration::ZERO),
13881        );
13882        let mut runtime = supervisor.runtime_config();
13883        runtime.child_roster.record_to(record.clone());
13884        let gate = Arc::new(super::ReloadExitRecordGate::default());
13885        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13886        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13887        let spec = ModuleSpec {
13888            module_id: "reload-exit-roster".into(),
13889            program: fake_aft_stub_path(),
13890            args: Vec::new(),
13891            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13892            reserved: false,
13893            reserved_prefixes: Vec::new(),
13894            protocol: ModuleProtocol::Subc,
13895            overlap: Default::default(),
13896            carry_containment: false,
13897        };
13898        let mut child = None;
13899        let reload = super::finish_reload_child(
13900            &spec,
13901            &runtime,
13902            &supervisor.registry,
13903            &supervisor.process_liveness,
13904            &snapshot,
13905            &mut child,
13906        );
13907        tokio::pin!(reload);
13908        tokio::select! {
13909            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13910            _ = gate.reached.notified() => {}
13911        }
13912        assert!(runtime
13913            .terminal_ring
13914            .lock()
13915            .unwrap()
13916            .snapshot()
13917            .entries
13918            .is_empty());
13919        assert_eq!(
13920            crate::live_children::read_record(&record).unwrap().len(),
13921            1,
13922            "shutdown must still wait for the reaped child until its terminal record exists"
13923        );
13924        runtime.child_roster.close();
13925        gate.resume.notify_one();
13926        assert!(reload.await.is_err());
13927        assert!(crate::live_children::read_record(&record)
13928            .unwrap()
13929            .is_empty());
13930        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13931        assert_eq!(history.entries.len(), 1);
13932        assert_eq!(
13933            history.entries[0].disposition,
13934            TerminalDisposition::DaemonShutdown
13935        );
13936    }
13937
13938    /// Each restart-producing arm has its own state transition. Keeping their
13939    /// lifetime count assertions adjacent prevents a later new arm from silently
13940    /// spending budget without recording the historical restart.
13941    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13942    async fn every_restart_increment_path_advances_lifetime_count() {
13943        let supervisor = Supervisor::new_for_test(
13944            Arc::new(Registry::default()),
13945            RestartPolicy::new(1, Duration::ZERO),
13946        );
13947        let runtime = supervisor.runtime_config();
13948        let spec = ModuleSpec {
13949            module_id: "lifetime-increment-path".to_string(),
13950            program: PathBuf::from("/unused/lifetime-increment-path"),
13951            args: Vec::new(),
13952            env: Vec::new(),
13953            reserved: false,
13954            reserved_prefixes: Vec::new(),
13955            protocol: ModuleProtocol::Subc,
13956            overlap: Default::default(),
13957            carry_containment: false,
13958        };
13959
13960        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13961        assert!(matches!(
13962            on_child_exit(
13963                &spec,
13964                runtime.restart_policy,
13965                &supervisor.registry,
13966                &crash_snapshot,
13967                &runtime.terminal_ring,
13968                &runtime.spawn_events,
13969                &runtime.child_roster,
13970                ExitReport {
13971                    kind: ExitKind::Crash,
13972                    code: Some(1),
13973                    signal: None,
13974                    at_ms: 1,
13975                },
13976                true,
13977            )
13978            .await,
13979            NextAction::Restart { schedule: _ }
13980        ));
13981        let (crash_restarts, crash_lifetime) = {
13982            let state = lock_snapshot(&crash_snapshot).unwrap();
13983            (state.crash_restarts.len(), state.lifetime_restarts)
13984        };
13985        assert_eq!(crash_restarts, 1);
13986        assert_eq!(crash_lifetime, 1);
13987
13988        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13989        let mut health_child = None;
13990        assert!(matches!(
13991            health_restart_child(
13992                &spec,
13993                &runtime,
13994                &supervisor.registry,
13995                &supervisor.process_liveness,
13996                &health_snapshot,
13997                &mut health_child,
13998                SupervisorHealthStatus::Failing,
13999                None,
14000                2,
14001            )
14002            .await,
14003            Ok(())
14004        ));
14005        assert!(health_child.is_none());
14006        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
14007        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
14008        let (health_restarts, health_lifetime) = {
14009            let state = lock_snapshot(&health_snapshot).unwrap();
14010            (state.crash_restarts.len(), state.lifetime_restarts)
14011        };
14012        assert_eq!(health_restarts, 1);
14013        assert_eq!(health_lifetime, 1);
14014
14015        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14016        let mut reload_child = None;
14017        assert!(matches!(
14018            handle_reload_spawn_failure(
14019                &spec,
14020                &runtime,
14021                &supervisor.process_liveness,
14022                &reload_snapshot,
14023                &mut reload_child,
14024                "forced reload spawn failure".to_string(),
14025            )
14026            .await,
14027            Err(SuperviseError::ReloadFailed { .. })
14028        ));
14029        let (reload_restarts, reload_lifetime) = {
14030            let state = lock_snapshot(&reload_snapshot).unwrap();
14031            (state.crash_restarts.len(), state.lifetime_restarts)
14032        };
14033        assert_eq!(reload_restarts, 1);
14034        assert_eq!(reload_lifetime, 1);
14035    }
14036
14037    #[tokio::test]
14038    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14039        let supervisor = Supervisor::new_for_test(
14040            Arc::new(Registry::default()),
14041            RestartPolicy::new(3, Duration::ZERO),
14042        );
14043        let runtime = supervisor.runtime_config();
14044        let spec = ModuleSpec {
14045            module_id: "deliberately-severed".to_string(),
14046            program: PathBuf::from("/unused/deliberately-severed"),
14047            args: Vec::new(),
14048            env: Vec::new(),
14049            reserved: false,
14050            reserved_prefixes: Vec::new(),
14051            protocol: ModuleProtocol::Subc,
14052            overlap: Default::default(),
14053            carry_containment: false,
14054        };
14055        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14056        let process = ProcessIdentity {
14057            pid: 41,
14058            start_time: 101,
14059        };
14060        record_deliberate_severance(&snapshot, process).unwrap();
14061        let exit_report = apply_deliberate_severance_marker(
14062            &snapshot,
14063            Some(process),
14064            ExitReport {
14065                kind: ExitKind::Crash,
14066                code: Some(1),
14067                signal: None,
14068                at_ms: 1,
14069            },
14070        );
14071        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14072
14073        assert!(matches!(
14074            on_child_exit(
14075                &spec,
14076                runtime.restart_policy,
14077                &supervisor.registry,
14078                &snapshot,
14079                &runtime.terminal_ring,
14080                &runtime.spawn_events,
14081                &runtime.child_roster,
14082                exit_report,
14083                true,
14084            )
14085            .await,
14086            NextAction::Restart { schedule: _ }
14087        ));
14088        let state = lock_snapshot(&snapshot).unwrap();
14089        assert_eq!(state.lifetime_restarts, 1);
14090        assert_eq!(state.crash_restarts.len(), 0);
14091    }
14092
14093    #[tokio::test]
14094    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14095        let supervisor = Supervisor::new_for_test(
14096            Arc::new(Registry::default()),
14097            RestartPolicy::new(3, Duration::ZERO),
14098        );
14099        let runtime = supervisor.runtime_config();
14100        let spec = ModuleSpec {
14101            module_id: "genuine-crash".to_string(),
14102            program: PathBuf::from("/unused/genuine-crash"),
14103            args: Vec::new(),
14104            env: Vec::new(),
14105            reserved: false,
14106            reserved_prefixes: Vec::new(),
14107            protocol: ModuleProtocol::Subc,
14108            overlap: Default::default(),
14109            carry_containment: false,
14110        };
14111        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14112
14113        assert!(matches!(
14114            on_child_exit(
14115                &spec,
14116                runtime.restart_policy,
14117                &supervisor.registry,
14118                &snapshot,
14119                &runtime.terminal_ring,
14120                &runtime.spawn_events,
14121                &runtime.child_roster,
14122                ExitReport {
14123                    kind: ExitKind::Crash,
14124                    code: Some(1),
14125                    signal: None,
14126                    at_ms: 1,
14127                },
14128                true,
14129            )
14130            .await,
14131            NextAction::Restart { schedule: _ }
14132        ));
14133        let state = lock_snapshot(&snapshot).unwrap();
14134        assert_eq!(state.lifetime_restarts, 1);
14135        assert_eq!(state.crash_restarts.len(), 1);
14136    }
14137
14138    fn crash_exit_report(at_ms: u64) -> ExitReport {
14139        ExitReport {
14140            kind: ExitKind::Crash,
14141            code: Some(1),
14142            signal: None,
14143            at_ms,
14144        }
14145    }
14146
14147    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14148        ModuleSpec {
14149            module_id: module_id.to_string(),
14150            program: PathBuf::from("/unused").join(module_id),
14151            args: Vec::new(),
14152            env: Vec::new(),
14153            reserved: false,
14154            reserved_prefixes: Vec::new(),
14155            protocol: ModuleProtocol::Subc,
14156            overlap: Default::default(),
14157            carry_containment: false,
14158        }
14159    }
14160
14161    /// A real crash loop still stops. Three crashes with nothing aging out spend
14162    /// a budget of two and the third respawn is refused, and both surfaces an
14163    /// operator has -- the log line and the retained terminal record -- name the
14164    /// window rather than only the cap, because `max_restarts=2` alone is what
14165    /// this budget used to mean.
14166    #[tokio::test]
14167    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14168        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14169        let supervisor = Supervisor::new_for_test(
14170            Arc::new(Registry::default()),
14171            RestartPolicy::new(2, Duration::ZERO),
14172        );
14173        let runtime = supervisor.runtime_config();
14174        let spec = windowed_crash_spec("crash-loop-in-window");
14175        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14176
14177        for attempt in 1..=2 {
14178            assert!(
14179                matches!(
14180                    on_child_exit(
14181                        &spec,
14182                        runtime.restart_policy,
14183                        &supervisor.registry,
14184                        &snapshot,
14185                        &runtime.terminal_ring,
14186                        &runtime.spawn_events,
14187                        &runtime.child_roster,
14188                        crash_exit_report(attempt),
14189                        true,
14190                    )
14191                    .await,
14192                    NextAction::Restart { schedule: _ }
14193                ),
14194                "crash {attempt} is inside the budget and must respawn"
14195            );
14196        }
14197
14198        assert!(matches!(
14199            on_child_exit(
14200                &spec,
14201                runtime.restart_policy,
14202                &supervisor.registry,
14203                &snapshot,
14204                &runtime.terminal_ring,
14205                &runtime.spawn_events,
14206                &runtime.child_roster,
14207                crash_exit_report(3),
14208                true,
14209            )
14210            .await,
14211            NextAction::Stop { .. }
14212        ));
14213
14214        {
14215            let state = lock_snapshot(&snapshot).unwrap();
14216            assert_eq!(state.state, ModuleState::Failed);
14217            assert_eq!(state.crash_restarts.len(), 2);
14218            assert_eq!(state.lifetime_restarts, 2);
14219        }
14220
14221        let history = runtime
14222            .terminal_ring
14223            .lock()
14224            .expect("terminal ring is not poisoned")
14225            .snapshot();
14226        let last = history
14227            .entries
14228            .last()
14229            .expect("the refused crash is retained");
14230        assert_eq!(last.disposition, TerminalDisposition::Failed);
14231        assert_eq!(
14232            last.disposition_detail.as_deref(),
14233            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14234        );
14235
14236        let captured = crate::router::test_log::captured_logs(&logs);
14237        assert!(
14238            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14239            "the stop must be logged with its window: {captured}"
14240        );
14241    }
14242
14243    /// The rate, stated as a test: three crashes where the first has aged past
14244    /// the window are two crashes as far as the budget is concerned, so the
14245    /// third respawn is allowed and the ring holds only the two recent ones.
14246    ///
14247    /// This is the case a lifetime counter got wrong -- and the case the daemon
14248    /// now hits routinely, since a module exits non-zero every time its
14249    /// connection to the daemon drops.
14250    #[tokio::test]
14251    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14252        let supervisor = Supervisor::new_for_test(
14253            Arc::new(Registry::default()),
14254            RestartPolicy::new(2, Duration::ZERO),
14255        );
14256        let runtime = supervisor.runtime_config();
14257        let spec = windowed_crash_spec("crash-across-windows");
14258        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14259
14260        for attempt in 1..=2 {
14261            assert!(matches!(
14262                on_child_exit(
14263                    &spec,
14264                    runtime.restart_policy,
14265                    &supervisor.registry,
14266                    &snapshot,
14267                    &runtime.terminal_ring,
14268                    &runtime.spawn_events,
14269                    &runtime.child_roster,
14270                    crash_exit_report(attempt),
14271                    true,
14272                )
14273                .await,
14274                NextAction::Restart { schedule: _ }
14275            ));
14276        }
14277
14278        // The oldest crash moves out of the window; nothing else about the
14279        // module changes.
14280        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14281            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14282        })
14283        .unwrap();
14284
14285        assert!(
14286            matches!(
14287                on_child_exit(
14288                    &spec,
14289                    runtime.restart_policy,
14290                    &supervisor.registry,
14291                    &snapshot,
14292                    &runtime.terminal_ring,
14293                    &runtime.spawn_events,
14294                    &runtime.child_roster,
14295                    crash_exit_report(3),
14296                    true,
14297                )
14298                .await,
14299                NextAction::Restart { schedule: _ }
14300            ),
14301            "a crash older than the window must not hold a budget slot"
14302        );
14303
14304        let state = lock_snapshot(&snapshot).unwrap();
14305        assert_eq!(state.state, ModuleState::Restarting);
14306        assert_eq!(
14307            state.crash_restarts.len(),
14308            2,
14309            "the aged instant is dropped and the new one takes its place"
14310        );
14311        assert_eq!(
14312            state.lifetime_restarts, 3,
14313            "the ledger counts every restart, including the ones the window forgot"
14314        );
14315    }
14316
14317    /// An operator restart hands the budget back whole, and the ledger keeps
14318    /// counting. Those are different questions -- "how close is this module to
14319    /// being stopped" and "how many times has it been replaced" -- and the
14320    /// operator action answers only the first.
14321    #[tokio::test]
14322    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14323        let supervisor = Supervisor::new_for_test(
14324            Arc::new(Registry::default()),
14325            RestartPolicy::new(2, Duration::ZERO),
14326        );
14327        let runtime = supervisor.runtime_config();
14328        let spec = windowed_crash_spec("operator-cleared-budget");
14329        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14330
14331        for attempt in 1..=2 {
14332            assert!(matches!(
14333                on_child_exit(
14334                    &spec,
14335                    runtime.restart_policy,
14336                    &supervisor.registry,
14337                    &snapshot,
14338                    &runtime.terminal_ring,
14339                    &runtime.spawn_events,
14340                    &runtime.child_roster,
14341                    crash_exit_report(attempt),
14342                    true,
14343                )
14344                .await,
14345                NextAction::Restart { schedule: _ }
14346            ));
14347        }
14348
14349        reset_restart_count(&snapshot, &spec.module_id).unwrap();
14350        {
14351            let state = lock_snapshot(&snapshot).unwrap();
14352            assert!(
14353                state.crash_restarts.is_empty(),
14354                "an operator restart returns the full budget"
14355            );
14356            assert_eq!(
14357                state.lifetime_restarts, 2,
14358                "clearing the budget must not unmake the crashes"
14359            );
14360        }
14361
14362        assert!(
14363            matches!(
14364                on_child_exit(
14365                    &spec,
14366                    runtime.restart_policy,
14367                    &supervisor.registry,
14368                    &snapshot,
14369                    &runtime.terminal_ring,
14370                    &runtime.spawn_events,
14371                    &runtime.child_roster,
14372                    crash_exit_report(3),
14373                    true,
14374                )
14375                .await,
14376                NextAction::Restart { schedule: _ }
14377            ),
14378            "the cleared budget must be spendable again"
14379        );
14380        let state = lock_snapshot(&snapshot).unwrap();
14381        assert_eq!(state.crash_restarts.len(), 1);
14382        assert_eq!(state.lifetime_restarts, 3);
14383    }
14384
14385    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14386    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14387        let severed = ProcessIdentity {
14388            pid: 41,
14389            start_time: 101,
14390        };
14391        let successor = ProcessIdentity {
14392            pid: 41,
14393            start_time: 202,
14394        };
14395        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14396        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14397            state.pid = Some(successor.pid);
14398            state.process_start_time = Some(successor.start_time);
14399        })
14400        .unwrap();
14401        assert!(!module.record_deliberate_severance(severed).unwrap());
14402
14403        let exit_report = apply_deliberate_severance_marker(
14404            &module.inner.snapshot,
14405            Some(successor),
14406            ExitReport {
14407                kind: ExitKind::Crash,
14408                code: Some(1),
14409                signal: None,
14410                at_ms: 1,
14411            },
14412        );
14413
14414        assert_eq!(exit_report.kind, ExitKind::Crash);
14415    }
14416
14417    #[tokio::test]
14418    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14419        let registry = Registry::default();
14420        let supervisor = Supervisor::new_for_test(
14421            Arc::new(Registry::default()),
14422            RestartPolicy::new(3, Duration::ZERO),
14423        );
14424        let runtime = supervisor.runtime_config();
14425        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14426        let spec = ModuleSpec {
14427            module_id: "drain-deliberate-severance".to_string(),
14428            program: fake_aft_stub_path(),
14429            args: Vec::new(),
14430            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14431            reserved: false,
14432            reserved_prefixes: Vec::new(),
14433            protocol: ModuleProtocol::Subc,
14434            overlap: Default::default(),
14435            carry_containment: false,
14436        };
14437        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14438        let process = ProcessIdentity {
14439            pid: 41,
14440            start_time: 101,
14441        };
14442        child.process_identity = Some(process);
14443        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14444            state.pid = Some(process.pid);
14445            state.process_start_time = Some(process.start_time);
14446        })
14447        .unwrap();
14448        record_deliberate_severance(&snapshot, process).unwrap();
14449
14450        drain_child_to_state(
14451            &spec.module_id,
14452            spec.protocol,
14453            // The child exits on its own; no signal may change the exit this
14454            // test classifies.
14455            StopNotice::SentOverConnection,
14456            &registry,
14457            None,
14458            &snapshot,
14459            &runtime.terminal_ring,
14460            &runtime.spawn_events,
14461            child,
14462            Duration::from_secs(1),
14463            ModuleState::Stopped,
14464            Some(false),
14465            &super::FailureContext::operator(),
14466        )
14467        .await
14468        .unwrap();
14469
14470        let state = lock_snapshot(&snapshot).unwrap();
14471        assert_eq!(
14472            state.last_exit.as_ref().map(|exit| exit.kind),
14473            Some(ExitKind::DeliberateSeverance)
14474        );
14475        assert_eq!(state.lifetime_restarts, 1);
14476        assert_eq!(state.crash_restarts.len(), 0);
14477        drop(state);
14478        let history = runtime.terminal_ring.lock().unwrap().snapshot();
14479        assert_eq!(
14480            history.entries[0].exit_kind,
14481            subc_control::TerminalExitKind::DeliberateSeverance
14482        );
14483    }
14484
14485    #[tokio::test]
14486    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14487        let registry = Registry::default();
14488        let supervisor = Supervisor::new_for_test(
14489            Arc::new(Registry::default()),
14490            RestartPolicy::new(3, Duration::ZERO),
14491        );
14492        let runtime = supervisor.runtime_config();
14493        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14494        let spec = ModuleSpec {
14495            module_id: "ordinary-drain".to_string(),
14496            program: fake_aft_stub_path(),
14497            args: Vec::new(),
14498            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14499            reserved: false,
14500            reserved_prefixes: Vec::new(),
14501            protocol: ModuleProtocol::Subc,
14502            overlap: Default::default(),
14503            carry_containment: false,
14504        };
14505        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14506
14507        drain_child_to_state(
14508            &spec.module_id,
14509            spec.protocol,
14510            // The child exits on its own; no signal may change the exit this
14511            // test classifies.
14512            StopNotice::SentOverConnection,
14513            &registry,
14514            None,
14515            &snapshot,
14516            &runtime.terminal_ring,
14517            &runtime.spawn_events,
14518            child,
14519            Duration::from_secs(1),
14520            ModuleState::Stopped,
14521            Some(false),
14522            &super::FailureContext::operator(),
14523        )
14524        .await
14525        .unwrap();
14526
14527        let state = lock_snapshot(&snapshot).unwrap();
14528        assert_eq!(
14529            state.last_exit.as_ref().map(|exit| exit.kind),
14530            Some(ExitKind::Crash)
14531        );
14532        assert_eq!(state.lifetime_restarts, 0);
14533        assert_eq!(state.crash_restarts.len(), 0);
14534    }
14535
14536    #[test]
14537    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14538        // The server's generic fatal-routing branch only knows that the
14539        // connection failed; it does not know that the daemon deliberately
14540        // initiated a process-killing severance. Keep this seam explicit so a
14541        // future connection error path cannot silently reintroduce the stale
14542        // exemption that mislabels a later genuine crash.
14543        assert!(!include_str!("server.rs")
14544            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14545    }
14546
14547    /// The `route.closed` `drained` value must be the quiescence wait's own
14548    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
14549    /// measurement at all and `false` is the one honest constant. This is the exact
14550    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
14551    /// on every return path, including the one that used to return early via `?`
14552    /// with `route.closing` already sent and no `route.closed` ever following.
14553    #[test]
14554    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14555        assert!(drained_after_quiescence_wait(&Ok(true)));
14556        assert!(!drained_after_quiescence_wait(&Ok(false)));
14557        assert!(!drained_after_quiescence_wait(&Err(
14558            SuperviseError::StatePoisoned { module_id: None }
14559        )));
14560    }
14561
14562    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
14563    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
14564    /// already reaped out-of-band) still leaves a terminal record rather than none
14565    /// at all. Triggering the real `wait()` I/O error from an integration test would
14566    /// need a genuine already-reaped-child race, which is OS-specific and not
14567    /// something this suite attempts elsewhere; this test instead verifies the
14568    /// record produced for that arm end-to-end through the real `TerminalRing`, and
14569    /// the call site itself is verified by inspection to sit in that exact arm.
14570    #[test]
14571    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14572        let ring = Arc::new(Mutex::new(TerminalRing::new(
14573            TerminalRingConfig::default(),
14574            0,
14575        )));
14576        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14577
14578        let snapshot = ring.lock().unwrap().snapshot();
14579        assert_eq!(snapshot.entries.len(), 1);
14580        let entry = &snapshot.entries[0];
14581        assert_eq!(entry.exit_code, None);
14582        assert_eq!(entry.exit_signal, None);
14583        assert_eq!(entry.disposition, TerminalDisposition::Failed);
14584    }
14585
14586    #[test]
14587    fn wait_error_exit_path_preserves_spawn_event_density() {
14588        let feed = super::SpawnEventFeed::default();
14589        feed.configure_incarnation("wait-error-density".to_string());
14590        feed.emit_spawned("wait-error", 41, 1);
14591        let ring = Arc::new(Mutex::new(TerminalRing::new(
14592            TerminalRingConfig::default(),
14593            0,
14594        )));
14595
14596        record_wait_error_terminal("wait-error", &ring, &feed);
14597        feed.emit_spawned("after-wait-error", 42, 2);
14598
14599        let state = feed.0.lock().unwrap();
14600        let sequences = state
14601            .events
14602            .iter()
14603            .map(|event| event.cursor.seq)
14604            .collect::<Vec<_>>();
14605        assert_eq!(sequences, vec![1, 2, 3]);
14606        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14607        assert_eq!(state.events[1].exit_code, None);
14608        assert_eq!(state.events[1].exit_signal, None);
14609    }
14610
14611    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
14612    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
14613    /// not a clean exit it never actually observed.
14614    #[test]
14615    fn wait_error_exit_report_is_classified_as_a_crash() {
14616        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14617    }
14618}
14619
14620#[cfg(all(test, unix))]
14621mod restart_ordering_tests {
14622    use super::*;
14623    use std::io::Write;
14624    use tokio::io::AsyncReadExt;
14625
14626    #[test]
14627    #[ignore = "child process fixture"]
14628    fn serving_child() {
14629        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14630            return;
14631        };
14632        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14633        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14634        for stream in listener.incoming() {
14635            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14636        }
14637    }
14638
14639    async fn restart_before_publication(requests: usize) {
14640        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14641        let address_file = home.path().join("address");
14642        let spec = ModuleSpec {
14643            module_id: "restart-serving-child".into(),
14644            program: std::env::current_exe().unwrap(),
14645            args: vec![
14646                "--exact".into(),
14647                "supervise::restart_ordering_tests::serving_child".into(),
14648                "--ignored".into(),
14649                "--nocapture".into(),
14650            ],
14651            env: vec![(
14652                "RESTART_TEST_ADDRESS_FILE".into(),
14653                address_file.display().to_string(),
14654            )],
14655            reserved: false,
14656            reserved_prefixes: vec![],
14657            protocol: ModuleProtocol::None,
14658            overlap: Default::default(),
14659            carry_containment: false,
14660        };
14661        let mut supervisor =
14662            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14663        let (spawned_tx, spawned_rx) = oneshot::channel();
14664        let spawned_tx = Mutex::new(Some(spawned_tx));
14665        let (release_tx, release_rx) = std::sync::mpsc::channel();
14666        let release_rx = Mutex::new(release_rx);
14667        // Hold publication, not the child: it must answer a real request before
14668        // the restart enters the queue, even though status still has generation 0.
14669        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14670            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14671            release_rx
14672                .lock()
14673                .unwrap()
14674                .recv_timeout(Duration::from_secs(20))
14675                .unwrap();
14676        })));
14677        let registered = supervisor
14678            .register_configured_with_health(
14679                spec.clone(),
14680                true,
14681                HealthConfig::default(),
14682                None,
14683                RestartPolicy::new(3, Duration::ZERO),
14684                DEFAULT_REGISTER_TIMEOUT,
14685            )
14686            .unwrap();
14687        let module = registered.module.clone();
14688        let startup = tokio::task::spawn_blocking(move || {
14689            supervisor.start_registered_configured(registered, None, true)
14690        });
14691        spawned_rx.await.unwrap();
14692        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14693            let address = loop {
14694                if let Ok(address) = std::fs::read_to_string(&address_file) {
14695                    if !address.is_empty() {
14696                        break address;
14697                    }
14698                }
14699                tokio::time::sleep(Duration::from_millis(1)).await;
14700            };
14701            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14702            let mut reply = String::new();
14703            stream.read_to_string(&mut reply).await.unwrap();
14704            reply.trim().parse::<u32>().unwrap()
14705        })
14706        .await
14707        .expect("child must serve before generation publication");
14708        assert_eq!(
14709            lock_snapshot(&module.inner.snapshot)
14710                .unwrap()
14711                .spawn_generation,
14712            0
14713        );
14714        let mut replies = Vec::new();
14715        for _ in 0..requests {
14716            let mut restart = Box::pin(module.restart(Some(0)));
14717            // Poll the public API through queue admission, leaving only its ACK
14718            // pending. The monitor cannot consume the command until released.
14719            std::future::poll_fn(|cx| {
14720                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14721                std::task::Poll::Ready(())
14722            })
14723            .await;
14724            replies.push(restart);
14725        }
14726        release_tx.send(()).unwrap();
14727        startup.await.unwrap().unwrap();
14728        for reply in replies {
14729            reply.await.unwrap();
14730        }
14731        // Restart schedules its replacement after completing the handler. Wait
14732        // for that separate actor turn before placing the command barrier.
14733        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14734            loop {
14735                if lock_snapshot(&module.inner.snapshot)
14736                    .unwrap()
14737                    .spawn_generation
14738                    > 1
14739                {
14740                    break;
14741                }
14742                tokio::time::sleep(Duration::from_millis(1)).await;
14743            }
14744        })
14745        .await;
14746        // Commands are handled serially. The update-spec reply proves every
14747        // restart handler has finished before we inspect the snapshot; restart's
14748        // own ACK only acknowledges initiation, not replacement completion.
14749        module.update_spec_for_test(spec).await.unwrap();
14750        let (generation, pid) = {
14751            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14752            (state.spawn_generation, state.pid)
14753        };
14754        module.stop().await.unwrap();
14755        replacement.expect("restart must publish a replacement for the already-serving child");
14756        assert_ne!(
14757            pid,
14758            Some(old_pid),
14759            "a serving child predating the restart must be replaced"
14760        );
14761        assert_eq!(generation, 2, "queued restarts must share one replacement");
14762    }
14763
14764    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14765    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14766        restart_before_publication(1).await;
14767    }
14768
14769    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14770    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14771        restart_before_publication(2).await;
14772    }
14773
14774    #[tokio::test]
14775    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14776        let supervisor =
14777            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14778        let registered = supervisor
14779            .register_configured_with_health(
14780                ModuleSpec {
14781                    module_id: "queued-before-replacement".into(),
14782                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14783                    // macOS a module whose running image is not the configured
14784                    // program is refused once its exec is confirmed (see
14785                    // `record_write_stall_tests::sleeping_spec`).
14786                    program: "/bin/sleep".into(),
14787                    args: vec!["600".into()],
14788                    env: vec![],
14789                    reserved: false,
14790                    reserved_prefixes: vec![],
14791                    protocol: ModuleProtocol::None,
14792                    overlap: Default::default(),
14793                    carry_containment: false,
14794                },
14795                true,
14796                HealthConfig::default(),
14797                None,
14798                RestartPolicy::default(),
14799                DEFAULT_REGISTER_TIMEOUT,
14800            )
14801            .unwrap();
14802        let RegisteredConfiguredModule {
14803            mut spec,
14804            mut runtime,
14805            module,
14806            mut commands,
14807        } = registered;
14808        let snapshot = &module.inner.snapshot;
14809        let mut restart = Box::pin(module.restart(Some(0)));
14810        std::future::poll_fn(|cx| {
14811            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14812            std::task::Poll::Ready(())
14813        })
14814        .await;
14815        // Drive the actor explicitly so a replacement is definitely spawned
14816        // after receipt, but before this queued command is dequeued.
14817        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14818        let replacement_pid = child.as_ref().unwrap().id();
14819        let command = commands.recv().await.unwrap();
14820        assert!(
14821            handle_supervisor_command(
14822                command,
14823                &mut spec,
14824                &mut runtime,
14825                &supervisor.registry,
14826                &supervisor.process_liveness,
14827                snapshot,
14828                &mut child,
14829                &mut commands,
14830                &mut VecDeque::new(),
14831            )
14832            .await
14833        );
14834        restart.await.unwrap();
14835        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14836        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14837        if let Some(mut child) = child {
14838            child.start_kill().unwrap();
14839            child.wait().await.unwrap();
14840        }
14841        assert_eq!(
14842            kept_pid, replacement_pid,
14843            "a newer replacement must survive"
14844        );
14845        assert!(
14846            !pending,
14847            "a satisfied restart must not schedule another spawn"
14848        );
14849    }
14850}
14851
14852#[cfg(all(test, unix))]
14853mod health_event_tests {
14854    use super::*;
14855    use cortexkit_test_support::ScratchDir;
14856    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14857
14858    struct Actor {
14859        module: SupervisedModule,
14860        registry: Arc<Registry>,
14861        forwarding: Arc<ForwardingTable>,
14862        spec: ModuleSpec,
14863        health: HealthConfig,
14864        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14865        roster: ChildRoster,
14866        _home: ScratchDir,
14867    }
14868
14869    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14870        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14871        // for an observed condition, rather than letting paused time auto-advance
14872        // deadlines or assuming a fixed number of yields finishes the work.
14873        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14874        let tick = Instant::now();
14875        loop {
14876            if let Some(value) = observe() {
14877                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14878                return value;
14879            }
14880            assert!(
14881                std::time::Instant::now() < deadline,
14882                "timed out waiting for {reason}"
14883            );
14884            tokio::task::yield_now().await;
14885        }
14886    }
14887
14888    impl Actor {
14889        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14890            Self::start_with_policy(
14891                protocol,
14892                cadence,
14893                DEFAULT_REGISTER_TIMEOUT,
14894                RestartPolicy::new(3, Duration::ZERO),
14895            )
14896            .await
14897        }
14898
14899        async fn start_with_policy(
14900            protocol: ModuleProtocol,
14901            cadence: Duration,
14902            register_timeout: Duration,
14903            policy: RestartPolicy,
14904        ) -> Self {
14905            let home = ScratchDir::new("health-events");
14906            let registry = Arc::new(Registry::default());
14907            let forwarding = Arc::new(ForwardingTable::default());
14908            let health = HealthConfig {
14909                cadence,
14910                deadline: Duration::from_secs(3600),
14911                ..HealthConfig::default()
14912            };
14913            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14914                .with_forwarding(forwarding.clone())
14915                .with_health_config(health.clone());
14916            let spec = ModuleSpec {
14917                module_id: "event-health-child".into(),
14918                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14919                // the privacy trampoline refuses a module whose running image
14920                // is not the configured program, and `start` below waits for
14921                // the process to be confirmed (`reported_pid`). The shell form
14922                // passed only while the image check ran before the shell had
14923                // replaced itself with `sleep` (see
14924                // `record_write_stall_tests::sleeping_spec`).
14925                program: PathBuf::from("/bin/sleep"),
14926                // The process stays alive but has no real bus peer. Registrations
14927                // below exercise the same registry writes as accepted HELLOs.
14928                args: vec!["600".into()],
14929                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14930                    .into_iter()
14931                    .map(|key| (key.into(), home.path().display().to_string()))
14932                    .collect(),
14933                reserved: false,
14934                reserved_prefixes: vec![],
14935                protocol,
14936                overlap: Default::default(),
14937                carry_containment: false,
14938            };
14939            let module = supervisor
14940                .supervise_configured_requested(
14941                    spec.clone(),
14942                    true,
14943                    health.clone(),
14944                    Some(0),
14945                    policy,
14946                    register_timeout,
14947                    None,
14948                )
14949                .unwrap();
14950            let (_, rx) = mpsc::channel(8);
14951            let actor = Self {
14952                module,
14953                registry,
14954                forwarding,
14955                spec,
14956                health,
14957                rx,
14958                roster: supervisor.child_roster.clone(),
14959                _home: home,
14960            };
14961            actor
14962                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14963                .await;
14964            actor
14965        }
14966
14967        async fn expire_registration_wait(&self, remaining: Duration) {
14968            tokio::time::advance(remaining).await;
14969            wait_for("registration timeout TERM grace", || {
14970                lock_snapshot(&self.module.inner.snapshot)
14971                    .unwrap()
14972                    .actor_select
14973                    .is_none()
14974                    .then_some(())
14975            })
14976            .await;
14977            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14978        }
14979
14980        fn turns(&self) -> u64 {
14981            lock_snapshot(&self.module.inner.snapshot)
14982                .unwrap()
14983                .actor_turns
14984        }
14985
14986        async fn wait_for_select(
14987            &self,
14988            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14989        ) -> ActorSelectCheckpoint {
14990            wait_for("actor select checkpoint", || {
14991                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14992                state.actor_select.filter(|checkpoint| {
14993                    checkpoint.generation == state.spawn_generation
14994                        && state.state == ModuleState::Running
14995                        && state.process_alive
14996                        && state.reported_pid().is_some()
14997                        && expected(checkpoint)
14998                })
14999            })
15000            .await
15001        }
15002
15003        fn hello(&mut self, connection: u64, health: bool) {
15004            let manifest =
15005                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
15006                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
15007                    .build();
15008            let connection = ConnectionId::new(connection);
15009            let (tx, rx) = mpsc::channel(8);
15010            self.rx = rx;
15011            self.forwarding
15012                .register_module_connection(
15013                    connection,
15014                    self.spec.module_id.clone(),
15015                    subc_protocol::PROTOCOL_VERSION,
15016                    Concurrency::ModuleManaged,
15017                    FrameSink::new(tx),
15018                )
15019                .unwrap();
15020            self.registry
15021                .register_with_control_ops(
15022                    manifest,
15023                    subc_protocol::PROTOCOL_VERSION,
15024                    connection,
15025                    if health {
15026                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15027                    } else {
15028                        vec![]
15029                    },
15030                )
15031                .unwrap();
15032        }
15033
15034        async fn probe(&mut self) -> crate::router::OutboundFrame {
15035            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15036                Ok(frame) => Some(frame),
15037                Err(mpsc::error::TryRecvError::Empty) => None,
15038                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15039            })
15040            .await;
15041            let body: Value = serde_json::from_slice(&frame.body).unwrap();
15042            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15043            frame
15044        }
15045
15046        fn no_probe(&mut self) {
15047            assert!(matches!(
15048                self.rx.try_recv(),
15049                Err(mpsc::error::TryRecvError::Empty)
15050            ));
15051        }
15052
15053        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15054            self.forwarding
15055                .complete_module_control_rpc(
15056                    ConnectionId::new(connection),
15057                    frame.header.corr,
15058                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15059                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15060                        status: HealthStatus::Ok,
15061                        detail: None,
15062                        metrics: None,
15063                    }),
15064                )
15065                .unwrap();
15066        }
15067    }
15068
15069    #[tokio::test(start_paused = true)]
15070    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15071        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15072        let initial = actor.module.status().unwrap();
15073        tokio::time::advance(Duration::from_secs(119)).await;
15074        tokio::task::yield_now().await;
15075        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15076        assert!(actor.module.terminal_history().entries.is_empty());
15077        actor.expire_registration_wait(Duration::from_secs(1)).await;
15078        actor
15079            .wait_for_select(|at| at.generation > initial.spawn_generation)
15080            .await;
15081        let status = actor.module.status().unwrap();
15082        assert_ne!(status.pid, initial.pid);
15083        assert!(!cortexkit_test_support::process_alive(
15084            initial.pid.unwrap() as i32
15085        ));
15086        assert_eq!(status.restart_count, 1);
15087        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15088        let entries = actor.module.terminal_history().entries;
15089        assert_eq!(entries.len(), 1);
15090        assert_eq!(
15091            entries[0].disposition_detail.as_deref(),
15092            Some("registration_timeout")
15093        );
15094        actor.module.stop().await.unwrap();
15095    }
15096
15097    #[tokio::test(start_paused = true)]
15098    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15099        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15100        let pid = actor.module.status().unwrap().pid;
15101        tokio::time::advance(Duration::from_secs(119)).await;
15102        actor.hello(1, false);
15103        let registered = actor
15104            .wait_for_select(|at| at.registered_connection.is_some())
15105            .await;
15106        assert_eq!(
15107            registered.wake_after, None,
15108            "HELLO must cancel the deadline"
15109        );
15110        tokio::time::advance(Duration::from_secs(300)).await;
15111        tokio::task::yield_now().await;
15112        assert_eq!(actor.module.status().unwrap().pid, pid);
15113        assert!(actor.module.terminal_history().entries.is_empty());
15114        // A disconnect after the first HELLO must not re-arm a startup timer.
15115        actor
15116            .registry
15117            .deregister_connection(ConnectionId::new(1))
15118            .unwrap();
15119        actor
15120            .wait_for_select(|at| at.registered_connection.is_none())
15121            .await;
15122        tokio::time::advance(Duration::from_secs(300)).await;
15123        tokio::task::yield_now().await;
15124        assert_eq!(actor.module.status().unwrap().pid, pid);
15125        assert!(actor.module.terminal_history().entries.is_empty());
15126        actor.module.stop().await.unwrap();
15127
15128        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15129        let pid = transient.module.status().unwrap().pid;
15130        tokio::time::advance(Duration::from_secs(119)).await;
15131        let turns = transient.turns();
15132        // Both writes happen before the supervisor can refresh the registry.
15133        // An accepted HELLO must remain evidence after its connection is gone.
15134        transient.hello(1, false);
15135        transient
15136            .registry
15137            .deregister_connection(ConnectionId::new(1))
15138            .unwrap();
15139        let parked = transient.wait_for_select(|at| at.turn > turns).await;
15140        assert_eq!(
15141            parked.wake_after, None,
15142            "transient HELLO must cancel the deadline"
15143        );
15144        tokio::time::advance(Duration::from_secs(300)).await;
15145        tokio::task::yield_now().await;
15146        assert_eq!(transient.module.status().unwrap().pid, pid);
15147        assert!(transient.module.terminal_history().entries.is_empty());
15148        transient.module.stop().await.unwrap();
15149    }
15150
15151    #[tokio::test(start_paused = true)]
15152    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15153        let actor = Actor::start_with_policy(
15154            ModuleProtocol::Subc,
15155            Duration::from_secs(30),
15156            Duration::from_secs(10),
15157            RestartPolicy::new(1, Duration::ZERO),
15158        )
15159        .await;
15160        let generation = actor.module.status().unwrap().spawn_generation;
15161        actor
15162            .expire_registration_wait(Duration::from_secs(10))
15163            .await;
15164        actor.wait_for_select(|at| at.generation > generation).await;
15165        actor
15166            .expire_registration_wait(Duration::from_secs(10))
15167            .await;
15168        let retry = wait_for("failed retry schedule", || {
15169            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15170            (state.state == ModuleState::Failed)
15171                .then_some(state.failed_retry.pending)
15172                .flatten()
15173        })
15174        .await;
15175        assert_eq!(retry.delay, Duration::from_secs(600));
15176        assert_eq!(retry.attempt, 1);
15177        let entries = actor.module.terminal_history().entries;
15178        assert_eq!(entries.len(), 2);
15179        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15180        assert!(entries[1]
15181            .disposition_detail
15182            .as_deref()
15183            .unwrap()
15184            .contains("registration_timeout"));
15185        tokio::time::advance(Duration::from_secs(599)).await;
15186        tokio::task::yield_now().await;
15187        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15188        tokio::time::advance(Duration::from_secs(1)).await;
15189        actor
15190            .wait_for_select(|at| at.generation > generation + 1)
15191            .await;
15192        // The slow retry is another ordinary spawn, with its own HELLO budget.
15193        actor
15194            .expire_registration_wait(Duration::from_secs(10))
15195            .await;
15196        wait_for("slow retry registration timeout", || {
15197            (actor.module.terminal_history().entries.len() == 3).then_some(())
15198        })
15199        .await;
15200        actor.module.stop().await.unwrap();
15201    }
15202
15203    #[tokio::test(start_paused = true)]
15204    async fn registration_timeout_honours_ten_second_override() {
15205        let actor = Actor::start_with_policy(
15206            ModuleProtocol::Subc,
15207            Duration::from_secs(30),
15208            Duration::from_secs(10),
15209            RestartPolicy::new(3, Duration::ZERO),
15210        )
15211        .await;
15212        let generation = actor.module.status().unwrap().spawn_generation;
15213        tokio::time::advance(Duration::from_secs(9)).await;
15214        tokio::task::yield_now().await;
15215        assert!(actor.module.terminal_history().entries.is_empty());
15216        actor.expire_registration_wait(Duration::from_secs(1)).await;
15217        actor.wait_for_select(|at| at.generation > generation).await;
15218        assert_eq!(actor.module.terminal_history().entries.len(), 1);
15219        actor.module.stop().await.unwrap();
15220    }
15221
15222    #[tokio::test]
15223    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15224        use std::os::unix::process::{CommandExt, ExitStatusExt};
15225
15226        struct Helper(std::process::Child);
15227        impl Drop for Helper {
15228            fn drop(&mut self) {
15229                // This handle still owns the helper until wait returns. Never
15230                // clean up its old process group after the leader is reaped.
15231                if matches!(self.0.try_wait(), Ok(None)) {
15232                    let _ = self.0.kill();
15233                    let _ = self.0.wait();
15234                }
15235            }
15236        }
15237
15238        let home = ScratchDir::new("registration-timeout-helper");
15239        let supervisor = Supervisor::new_for_test(
15240            Arc::new(Registry::default()),
15241            RestartPolicy::new(0, Duration::ZERO),
15242        );
15243        let spec = ModuleSpec {
15244            module_id: "registration-timeout-group".into(),
15245            program: "/bin/sleep".into(),
15246            args: vec!["120".into()],
15247            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15248                .into_iter()
15249                .map(|key| (key.into(), home.path().display().to_string()))
15250                .collect(),
15251            reserved: false,
15252            reserved_prefixes: vec![],
15253            protocol: ModuleProtocol::Subc,
15254            overlap: Default::default(),
15255            carry_containment: false,
15256        };
15257        let module = supervisor
15258            .supervise_configured_requested(
15259                spec,
15260                true,
15261                HealthConfig::default(),
15262                None,
15263                RestartPolicy::new(0, Duration::ZERO),
15264                Duration::from_secs(10),
15265                None,
15266            )
15267            .unwrap();
15268        let leader = timeout(Duration::from_secs(5), async {
15269            loop {
15270                if let Some(pid) = module.status().unwrap().pid {
15271                    break pid;
15272                }
15273                sleep(Duration::from_millis(5)).await;
15274            }
15275        })
15276        .await
15277        .expect("supervised group leader must be reported before the deadline");
15278        assert!(leader > 1);
15279        let ready = home.join("helper.ready");
15280        // Own the helper separately so even a broken group kill can be cleaned
15281        // up by its own unreaped child handle, without signalling a stale pgid.
15282        let mut helper = Helper(
15283            std::process::Command::new("/bin/sh")
15284                .args([
15285                    "-c",
15286                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15287                    "helper",
15288                ])
15289                .arg(&ready)
15290                .process_group(i32::try_from(leader).unwrap())
15291                .stdout(std::process::Stdio::null())
15292                .stderr(std::process::Stdio::null())
15293                .spawn()
15294                .unwrap(),
15295        );
15296        timeout(Duration::from_secs(5), async {
15297            while !ready.exists() {
15298                assert!(helper.0.try_wait().unwrap().is_none());
15299                sleep(Duration::from_millis(5)).await;
15300            }
15301        })
15302        .await
15303        .expect("helper must install TERM ignore before the registration deadline");
15304        let helper_pid =
15305            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15306        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15307        sleep(Duration::from_millis(20)).await;
15308        assert!(
15309            helper.0.try_wait().unwrap().is_none(),
15310            "helper must ignore TERM"
15311        );
15312
15313        timeout(Duration::from_secs(20), async {
15314            while module.state().unwrap() != ModuleState::Failed {
15315                sleep(Duration::from_millis(5)).await;
15316            }
15317        })
15318        .await
15319        .expect("registration timeout must stop and reap the group leader");
15320        let helper_exit = timeout(Duration::from_secs(3), async {
15321            loop {
15322                if let Some(status) = helper.0.try_wait().unwrap() {
15323                    break status;
15324                }
15325                sleep(Duration::from_millis(5)).await;
15326            }
15327        })
15328        .await
15329        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15330        assert_eq!(helper_exit.signal(), Some(9));
15331        assert!(!cortexkit_test_support::process_alive(
15332            i32::try_from(leader).unwrap()
15333        ));
15334        assert_eq!(
15335            module.status().unwrap().last_exit.unwrap().kind,
15336            ExitKind::Crash
15337        );
15338        assert!(module.terminal_history().entries[0]
15339            .disposition_detail
15340            .as_deref()
15341            .unwrap()
15342            .contains("registration_timeout"));
15343        module.stop().await.unwrap();
15344    }
15345
15346    #[tokio::test(start_paused = true)]
15347    async fn registration_timeout_does_not_count_an_earlier_exit() {
15348        let actor = Actor::start_with_policy(
15349            ModuleProtocol::Subc,
15350            Duration::from_secs(30),
15351            DEFAULT_REGISTER_TIMEOUT,
15352            RestartPolicy::new(0, Duration::ZERO),
15353        )
15354        .await;
15355        let pid = actor.module.status().unwrap().pid.unwrap();
15356        rustix::process::kill_process(
15357            rustix::process::Pid::from_raw(pid as i32).unwrap(),
15358            rustix::process::Signal::TERM,
15359        )
15360        .unwrap();
15361        wait_for("early exit", || {
15362            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15363        })
15364        .await;
15365        tokio::time::advance(Duration::from_secs(121)).await;
15366        tokio::task::yield_now().await;
15367        let entries = actor.module.terminal_history().entries;
15368        assert_eq!(entries.len(), 1);
15369        assert!(!entries[0]
15370            .disposition_detail
15371            .as_deref()
15372            .unwrap()
15373            .contains("registration_timeout"));
15374        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15375        actor.module.stop().await.unwrap();
15376    }
15377
15378    #[tokio::test(start_paused = true)]
15379    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15380        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15381        let pid = actor.module.status().unwrap().pid;
15382        tokio::time::advance(Duration::from_secs(119)).await;
15383        actor.roster.close();
15384        let turns = actor.turns();
15385        tokio::time::advance(Duration::from_secs(300)).await;
15386        let parked = actor.wait_for_select(|at| at.turn > turns).await;
15387        assert_eq!(parked.wake_after, None);
15388        assert_eq!(actor.module.status().unwrap().pid, pid);
15389        assert!(actor.module.terminal_history().entries.is_empty());
15390        actor.module.stop().await.unwrap();
15391        let status = actor.module.status().unwrap();
15392        assert_eq!(status.restart_count, 0);
15393        assert_eq!(
15394            actor.module.terminal_history().entries[0].disposition,
15395            TerminalDisposition::DaemonShutdown
15396        );
15397    }
15398
15399    #[tokio::test(start_paused = true)]
15400    async fn no_health_children_park_for_sixty_seconds() {
15401        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15402        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15403        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15404        unadvertised.hello(1, false);
15405        // Include any one-off registration wake in the budget; receiving a
15406        // no-health HELLO must not turn a parked child into a polling child.
15407        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
15408        // a single 60 s jump would only observe one expired timer.
15409        for _ in 0..6000 {
15410            tokio::time::advance(Duration::from_millis(10)).await;
15411            tokio::task::yield_now().await;
15412        }
15413        for actor in [&none, &unregistered, &unadvertised] {
15414            assert!(
15415                actor.turns() <= 5,
15416                "no-health actor woke {} times",
15417                actor.turns()
15418            );
15419            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15420        }
15421    }
15422
15423    #[tokio::test(start_paused = true)]
15424    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15425        // With a zero cadence the first probe is due at once, so this measures
15426        // only how fast the registration notification starts health probing,
15427        // not the normal 30-33 s wait before a first probe (checked below).
15428        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15429        let tick = Instant::now();
15430        actor.hello(1, true);
15431        actor.probe().await;
15432        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15433    }
15434
15435    #[tokio::test(start_paused = true)]
15436    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15437        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15438        let hello_tick = Instant::now();
15439        actor.hello(1, true);
15440        let armed = actor
15441            .wait_for_select(|checkpoint| {
15442                checkpoint.registered_connection == Some(ConnectionId::new(1))
15443                    && checkpoint.next_probe_at.is_some()
15444            })
15445            .await;
15446        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15447        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15448        let first_deadline = hello_tick + delay;
15449        assert_eq!(armed.next_probe_at, Some(first_deadline));
15450        tokio::time::advance(Duration::from_secs(29)).await;
15451        let turns = actor.turns();
15452        actor
15453            .registry
15454            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15455            .unwrap();
15456        let updated = actor
15457            .wait_for_select(|checkpoint| checkpoint.turn > turns)
15458            .await;
15459        assert_eq!(
15460            updated.next_probe_at,
15461            Some(first_deadline),
15462            "catalog update must not reset cadence"
15463        );
15464        actor.no_probe();
15465        tokio::time::advance(first_deadline - Instant::now()).await;
15466        let frame = actor.probe().await;
15467        actor.answer(1, frame);
15468        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15469        let next_deadline = Instant::now() + next;
15470        let rearmed = actor
15471            .wait_for_select(|checkpoint| {
15472                checkpoint
15473                    .next_probe_at
15474                    .is_some_and(|deadline| deadline > first_deadline)
15475            })
15476            .await;
15477        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15478        tokio::time::advance(next - Duration::from_nanos(1)).await;
15479        actor.no_probe();
15480        tokio::time::advance(Duration::from_nanos(1)).await;
15481        actor.probe().await;
15482    }
15483
15484    #[tokio::test(start_paused = true)]
15485    async fn disconnect_disarms_and_reconnect_rearms_health() {
15486        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15487        actor.hello(1, true);
15488        actor
15489            .wait_for_select(|checkpoint| {
15490                checkpoint.registered_connection == Some(ConnectionId::new(1))
15491                    && checkpoint.next_probe_at.is_some()
15492            })
15493            .await;
15494        let turns = actor.turns();
15495        let tick = Instant::now();
15496        actor
15497            .registry
15498            .deregister_connection(ConnectionId::new(1))
15499            .unwrap();
15500        let disarmed = actor
15501            .wait_for_select(|checkpoint| {
15502                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15503            })
15504            .await;
15505        assert_eq!(disarmed.next_probe_at, None);
15506        assert_eq!(disarmed.wake_after, None);
15507        assert_eq!(
15508            Instant::now(),
15509            tick,
15510            "disconnect must disarm in the notification tick"
15511        );
15512        tokio::time::advance(Duration::from_secs(60)).await;
15513        actor.no_probe();
15514        let reconnect_tick = Instant::now();
15515        actor.hello(2, true);
15516        let rearmed = actor
15517            .wait_for_select(|checkpoint| {
15518                checkpoint.registered_connection == Some(ConnectionId::new(2))
15519                    && checkpoint.next_probe_at.is_some()
15520            })
15521            .await;
15522        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15523        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15524        tokio::time::advance(delay).await;
15525        actor.probe().await;
15526    }
15527
15528    #[tokio::test(start_paused = true)]
15529    async fn reregistration_without_health_disarms_the_old_schedule() {
15530        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15531        actor.hello(1, true);
15532        actor
15533            .wait_for_select(|checkpoint| {
15534                checkpoint.registered_connection == Some(ConnectionId::new(1))
15535                    && checkpoint.next_probe_at.is_some()
15536            })
15537            .await;
15538        actor
15539            .registry
15540            .deregister_connection(ConnectionId::new(1))
15541            .unwrap();
15542        actor.hello(2, false);
15543        let disarmed = actor
15544            .wait_for_select(|checkpoint| {
15545                checkpoint.registered_connection == Some(ConnectionId::new(2))
15546            })
15547            .await;
15548        assert_eq!(disarmed.next_probe_at, None);
15549        assert_eq!(disarmed.wake_after, None);
15550        tokio::time::advance(Duration::from_secs(60)).await;
15551        actor.no_probe();
15552    }
15553
15554    #[tokio::test(start_paused = true)]
15555    async fn restart_waits_for_its_own_hello_before_rearming_health() {
15556        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15557        actor.hello(1, true);
15558        let armed = actor
15559            .wait_for_select(|checkpoint| {
15560                checkpoint.registered_connection == Some(ConnectionId::new(1))
15561                    && checkpoint.next_probe_at.is_some()
15562            })
15563            .await;
15564        // Simulate the old process's connection closing before the restart
15565        // tears the process down. The replacement process has started but has
15566        // not sent its HELLO, so it is not registered.
15567        actor
15568            .registry
15569            .deregister_connection(ConnectionId::new(1))
15570            .unwrap();
15571        actor.module.restart(Some(0)).await.unwrap();
15572        let parked = actor
15573            .wait_for_select(|checkpoint| {
15574                checkpoint.generation > armed.generation
15575                    && checkpoint.registered_connection.is_none()
15576                    && checkpoint.next_probe_at.is_none()
15577            })
15578            .await;
15579        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15580        actor.rx.close();
15581        // Drain the stop notice the restart queued for the old process's connection,
15582        // so it isn't mistaken for traffic to the replacement.
15583        while actor.rx.try_recv().is_ok() {}
15584        let turns = actor.turns();
15585        tokio::time::advance(Duration::from_secs(60)).await;
15586        let after = actor.turns();
15587        eprintln!(
15588            "restart park turns: before={turns} after={after} delta={}",
15589            after - turns
15590        );
15591        assert_eq!(after, turns, "replacement must park before HELLO");
15592        let hello_tick = Instant::now();
15593        actor.hello(2, true);
15594        let rearmed = actor
15595            .wait_for_select(|checkpoint| {
15596                checkpoint.registered_connection == Some(ConnectionId::new(2))
15597                    && checkpoint.next_probe_at.is_some()
15598            })
15599            .await;
15600        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15601        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15602        tokio::time::advance(delay).await;
15603        actor.probe().await;
15604    }
15605
15606    #[tokio::test(start_paused = true)]
15607    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15608        use tokio::{
15609            io::{AsyncReadExt, AsyncWriteExt},
15610            net::TcpListener,
15611        };
15612        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15613        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15614        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15615        let (started_tx, mut started_rx) = mpsc::channel(8);
15616        let peer = tokio::spawn(async move {
15617            loop {
15618                let (mut socket, _) = listener.accept().await.unwrap();
15619                let mut request = [0; 1024];
15620                assert!(socket.read(&mut request).await.unwrap() > 0);
15621                started_tx.send(Instant::now()).await.unwrap();
15622                socket
15623                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15624                    .await
15625                    .unwrap();
15626            }
15627        });
15628        actor
15629            .module
15630            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15631            .await
15632            .unwrap();
15633        let first_tick = Instant::now();
15634        let first = actor
15635            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15636            .await;
15637        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15638        assert!(started_rx.try_recv().is_err());
15639
15640        // Restart just before the old deadline: reusing it would give the new
15641        // process only one second to start, rather than a full cadence.
15642        tokio::time::advance(
15643            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15644        )
15645        .await;
15646        actor.module.restart(Some(0)).await.unwrap();
15647        let spawn_tick = Instant::now();
15648        let replacement = actor
15649            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15650            .await;
15651        let deadline = replacement.next_probe_at.unwrap();
15652        assert!(
15653            deadline >= spawn_tick + actor.health.cadence,
15654            "a replacement must not inherit its predecessor's probe deadline"
15655        );
15656        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15657        assert!(
15658            started_rx.try_recv().is_err(),
15659            "no HTTP probe during startup grace"
15660        );
15661        tokio::time::advance(deadline - Instant::now()).await;
15662        let landed = wait_for("first replacement HTTP probe", || {
15663            started_rx.try_recv().ok()
15664        })
15665        .await;
15666        assert!(landed >= spawn_tick + actor.health.cadence);
15667        wait_for("healthy replacement", || {
15668            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15669                .then_some(())
15670        })
15671        .await;
15672        peer.abort();
15673    }
15674
15675    #[tokio::test(start_paused = true)]
15676    async fn rescan_adds_and_removes_http_health_without_polling() {
15677        use tokio::{
15678            io::{AsyncReadExt, AsyncWriteExt},
15679            net::TcpListener,
15680        };
15681        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15682        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15683        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15684        // One millisecond separates successive probes without a polling delay.
15685        actor.health.cadence = Duration::from_millis(1);
15686        let (started_tx, mut started_rx) = mpsc::channel(8);
15687        let peer = tokio::spawn(async move {
15688            loop {
15689                let (mut socket, _) = listener.accept().await.unwrap();
15690                let mut request = [0; 1024];
15691                assert!(socket.read(&mut request).await.unwrap() > 0);
15692                started_tx.send(()).await.unwrap();
15693                socket
15694                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15695                    .await
15696                    .unwrap();
15697            }
15698        });
15699        let tick = Instant::now();
15700        let turns = actor.turns();
15701        actor
15702            .module
15703            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15704            .await
15705            .unwrap();
15706        let armed = actor
15707            .wait_for_select(|checkpoint| {
15708                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15709            })
15710            .await;
15711        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15712        assert_eq!(Instant::now(), tick);
15713        tokio::time::advance(Duration::from_millis(1)).await;
15714        let mut started = false;
15715        wait_for("successful HTTP probe", || {
15716            started |= started_rx.try_recv().is_ok();
15717            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15718                .then_some(())
15719        })
15720        .await;
15721        assert!(started, "added HTTP check must start at its first deadline");
15722        assert_eq!(
15723            actor.module.status().unwrap().health.status,
15724            SupervisorHealthStatus::Ok
15725        );
15726        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15727        actor.health.http = None;
15728        let turns = actor.turns();
15729        actor
15730            .module
15731            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15732            .await
15733            .unwrap();
15734        let parked = actor
15735            .wait_for_select(|checkpoint| {
15736                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15737            })
15738            .await;
15739        assert_eq!(parked.wake_after, None);
15740        let turns = actor.turns();
15741        tokio::time::advance(Duration::from_secs(60)).await;
15742        assert_eq!(
15743            actor.turns(),
15744            turns,
15745            "removed HTTP check must park the actor"
15746        );
15747        assert!(
15748            started_rx.try_recv().is_err(),
15749            "removed HTTP check must stay disarmed"
15750        );
15751        assert_eq!(
15752            actor.module.status().unwrap().health,
15753            ModuleHealthStatus::default()
15754        );
15755        peer.abort();
15756    }
15757}
15758
15759#[cfg(test)]
15760mod drain_busy_reprobe_tests {
15761    //! The drain's busy-gauge wait, driven against an in-process module
15762    //! connection whose health answers the test scripts. Time is paused, so
15763    //! every elapsed duration below is exact virtual time.
15764    use super::*;
15765    use std::collections::BTreeMap;
15766    use std::sync::Mutex as StdMutex;
15767    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15768
15769    const MODULE_ID: &str = "drain-reprobe-module";
15770    const CONNECTION: u64 = 7;
15771    const GAUGE: &str = "runs_in_flight";
15772    /// The production default for both the health cadence and the drain
15773    /// ceiling, which is the pairing that used to cost a full ceiling.
15774    const CADENCE: Duration = Duration::from_secs(30);
15775    const CEILING: Duration = Duration::from_secs(30);
15776
15777    /// How the scripted module answers each `health.check`.
15778    #[derive(Clone, Copy)]
15779    enum Answer {
15780        /// Report the gauge busy until this instant, quiet from then on, after
15781        /// taking `latency` to answer.
15782        BusyUntil { until: Instant, latency: Duration },
15783        /// Answer at once, with metrics that leave the declared gauge out.
15784        OmitsGauge,
15785        /// Never answer.
15786        Silent,
15787    }
15788
15789    struct Drain {
15790        forwarding: Arc<ForwardingTable>,
15791        runtime: SupervisorRuntimeConfig,
15792        endpoint: crate::ModuleEndpointId,
15793        /// When each probe reached the module.
15794        probes: Arc<StdMutex<Vec<Instant>>>,
15795        responder: tokio::task::JoinHandle<()>,
15796    }
15797
15798    impl Drain {
15799        fn start(answer: Answer) -> Self {
15800            let registry = Arc::new(Registry::default());
15801            let forwarding = Arc::new(ForwardingTable::default());
15802            let health = HealthConfig {
15803                cadence: CADENCE,
15804                ..HealthConfig::default()
15805            };
15806            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15807                .with_forwarding(forwarding.clone())
15808                .with_health_config(health)
15809                .runtime_config();
15810            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15811            let endpoint = forwarding
15812                .register_module_connection(
15813                    ConnectionId::new(CONNECTION),
15814                    MODULE_ID.to_string(),
15815                    subc_protocol::PROTOCOL_VERSION,
15816                    Concurrency::ModuleManaged,
15817                    FrameSink::new(tx),
15818                )
15819                .unwrap();
15820            let probes = Arc::new(StdMutex::new(Vec::new()));
15821            let responder = tokio::spawn({
15822                let forwarding = forwarding.clone();
15823                let probes = probes.clone();
15824                async move {
15825                    while let Some(frame) = rx.recv().await {
15826                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15827                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15828                        probes.lock().unwrap().push(Instant::now());
15829                        let metrics = match answer {
15830                            Answer::BusyUntil { until, latency } => {
15831                                sleep(latency).await;
15832                                let busy = u64::from(Instant::now() < until);
15833                                serde_json::json!({ GAUGE: busy })
15834                            }
15835                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15836                            Answer::Silent => continue,
15837                        };
15838                        // A probe abandoned at its deadline has no RPC left to
15839                        // complete, so a late answer may be refused.
15840                        let _ = forwarding.complete_module_control_rpc(
15841                            ConnectionId::new(CONNECTION),
15842                            frame.header.corr,
15843                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15844                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15845                                status: HealthStatus::Ok,
15846                                detail: None,
15847                                metrics: Some(metrics),
15848                            }),
15849                        );
15850                    }
15851                }
15852            });
15853            Self {
15854                forwarding,
15855                runtime,
15856                endpoint,
15857                probes,
15858                responder,
15859            }
15860        }
15861
15862        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15863        /// how long it took.
15864        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15865            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15866            (drained, elapsed)
15867        }
15868
15869        /// [`Self::run`], also returning what the drain's probes read.
15870        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15871            let started = Instant::now();
15872            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15873            let drained = wait_for_forwarding_quiescence(
15874                &self.forwarding,
15875                MODULE_ID,
15876                &self.runtime,
15877                self.endpoint,
15878                started + ceiling,
15879                &[GAUGE.to_string()],
15880                DrainScope::Active,
15881                &mut trace,
15882            )
15883            .await
15884            .unwrap();
15885            (drained, started.elapsed(), trace)
15886        }
15887
15888        fn probes(&self) -> Vec<Instant> {
15889            self.probes.lock().unwrap().clone()
15890        }
15891    }
15892
15893    impl Drop for Drain {
15894        fn drop(&mut self) {
15895            self.responder.abort();
15896        }
15897    }
15898
15899    #[tokio::test(start_paused = true)]
15900    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15901        let drain = Drain::start(Answer::BusyUntil {
15902            until: Instant::now() + Duration::from_secs(1),
15903            latency: Duration::ZERO,
15904        });
15905        let (drained, elapsed) = drain.run(CEILING).await;
15906        assert!(
15907            drained,
15908            "gauges that read zero during the drain must let it finish drained, \
15909             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15910        );
15911        assert!(
15912            elapsed < Duration::from_secs(3),
15913            "drain took {elapsed:?} after the gauges went quiet at 1s"
15914        );
15915        let probes = drain.probes();
15916        assert!(
15917            probes.len() >= 2,
15918            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15919        );
15920    }
15921
15922    /// The same shape in real time on a multi-threaded runtime: the first
15923    /// probe is answered busy at once, within a millisecond of the drain
15924    /// starting, and later probes read idle. The drain must keep probing
15925    /// after that first instant answer and end soon after the gauge falls.
15926    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15927    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15928        let drain = Drain::start(Answer::BusyUntil {
15929            until: Instant::now() + Duration::from_millis(600),
15930            latency: Duration::ZERO,
15931        });
15932        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15933        assert!(drained, "the gauge fell at 600ms: {trace}");
15934        assert!(
15935            elapsed < Duration::from_secs(3),
15936            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15937        );
15938        let probes = drain.probes();
15939        assert!(
15940            probes.len() >= 2 && trace.probes as usize == probes.len(),
15941            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15942        );
15943    }
15944
15945    #[tokio::test(start_paused = true)]
15946    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15947        let started = Instant::now();
15948        let drain = Drain::start(Answer::BusyUntil {
15949            until: started + Duration::from_secs(3600),
15950            latency: Duration::ZERO,
15951        });
15952        let (drained, elapsed) = drain.run(CEILING).await;
15953        assert!(!drained, "a module busy throughout must not drain");
15954        assert!(
15955            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15956            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15957        );
15958        // Re-probing is paced by the interval, not a busy loop: at most one
15959        // probe per interval across the ceiling, plus the first at time zero.
15960        let probes = drain.probes();
15961        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15962        assert!(
15963            probes.len() as u128 <= max_probes,
15964            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15965            probes.len()
15966        );
15967        for pair in probes.windows(2) {
15968            assert!(
15969                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15970                "consecutive probes {:?} apart",
15971                pair[1] - pair[0]
15972            );
15973        }
15974    }
15975
15976    #[tokio::test(start_paused = true)]
15977    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15978        // Each answer takes longer than the re-probe interval; the next probe
15979        // must wait for the previous answer and then the interval.
15980        let latency = Duration::from_secs(1);
15981        let drain = Drain::start(Answer::BusyUntil {
15982            until: Instant::now() + Duration::from_secs(3600),
15983            latency,
15984        });
15985        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15986        assert!(!drained);
15987        let probes = drain.probes();
15988        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15989        for pair in probes.windows(2) {
15990            assert!(
15991                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15992                "a probe started {:?} after the previous one, before it could finish",
15993                pair[1] - pair[0]
15994            );
15995        }
15996    }
15997
15998    #[tokio::test(start_paused = true)]
15999    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
16000        let drain = Drain::start(Answer::Silent);
16001        let ceiling = Duration::from_secs(2);
16002        let (drained, elapsed) = drain.run(ceiling).await;
16003        assert!(!drained, "an unanswered probe counts as busy");
16004        assert!(
16005            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
16006            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
16007        );
16008        assert_eq!(
16009            drain.probes().len(),
16010            1,
16011            "the unanswered probe holds the only slot until the ceiling"
16012        );
16013    }
16014
16015    /// Every event logged while it is installed, as `message` plus fields.
16016    #[derive(Clone, Default)]
16017    struct LogCapture {
16018        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16019    }
16020
16021    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16022        fn on_event(
16023            &self,
16024            event: &tracing::Event<'_>,
16025            _context: tracing_subscriber::layer::Context<'_, S>,
16026        ) {
16027            struct Fields(BTreeMap<String, String>);
16028            impl tracing::field::Visit for Fields {
16029                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16030                    self.0
16031                        .insert(field.name().to_string(), format!("{value:?}"));
16032                }
16033            }
16034            let mut fields = Fields(BTreeMap::new());
16035            event.record(&mut fields);
16036            self.events.lock().unwrap().push(fields.0);
16037        }
16038    }
16039
16040    impl LogCapture {
16041        fn install(&self) -> tracing::subscriber::DefaultGuard {
16042            use tracing_subscriber::layer::SubscriberExt;
16043            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16044        }
16045
16046        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16047            self.events
16048                .lock()
16049                .unwrap()
16050                .iter()
16051                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16052                .cloned()
16053                .collect()
16054        }
16055    }
16056
16057    /// A module that declares a busy gauge but leaves it out of its health
16058    /// answers holds a drain to its ceiling. The timeout line must say so by
16059    /// name, and the drain must warn about the missing gauge once, not on
16060    /// every probe.
16061    #[tokio::test(start_paused = true)]
16062    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16063        let capture = LogCapture::default();
16064        let _guard = capture.install();
16065        let drain = Drain::start(Answer::OmitsGauge);
16066        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16067        assert!(!drained, "a gauge never reported cannot read quiet");
16068        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16069
16070        let missing = capture.with_message("lacks a declared busy gauge");
16071        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16072        assert_eq!(
16073            missing[0].get("missing_gauges").map(String::as_str),
16074            Some(GAUGE)
16075        );
16076
16077        warn_drain_timed_out(
16078            MODULE_ID,
16079            Duration::from_secs(2),
16080            RouteCloseReason::Restart,
16081            &Default::default(),
16082            &trace,
16083        );
16084        let timeouts = capture.with_message("route drain timed out");
16085        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16086        let gauges = &timeouts[0]["busy_gauges"];
16087        assert!(
16088            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16089            "the timeout line must name the gauge and why it read busy: {gauges}"
16090        );
16091        assert!(
16092            gauges.ends_with("; no value ever read"),
16093            "no value was ever read for the gauge: {gauges}"
16094        );
16095        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16096    }
16097
16098    /// The trace names a busy value and when it was read. For a probe that
16099    /// never got an answer, it names the probe's error kind and says that no
16100    /// value was ever read.
16101    #[tokio::test(start_paused = true)]
16102    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16103        let busy = Drain::start(Answer::BusyUntil {
16104            until: Instant::now() + Duration::from_secs(3600),
16105            latency: Duration::ZERO,
16106        });
16107        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16108        let rendered = trace.to_string();
16109        assert!(
16110            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16111                && rendered.contains("; last value non_zero(1) @"),
16112            "{rendered}"
16113        );
16114
16115        let silent = Drain::start(Answer::Silent);
16116        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16117        // The probe is capped at the 1 s ceiling, and its reading is stamped
16118        // when it gave up.
16119        assert_eq!(
16120            trace.to_string(),
16121            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16122        );
16123        assert_eq!(trace.probes, 1);
16124    }
16125
16126    /// A gauge value that is not a count reads busy and is shown as sent.
16127    #[test]
16128    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16129        let gauges = [GAUGE.to_string()];
16130        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16131        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16132        assert_eq!(
16133            busy_gauge_observation(&verdicts),
16134            BusyGaugeObservation::Busy
16135        );
16136        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16137        assert_eq!(
16138            busy_gauge_observation(&zero),
16139            BusyGaugeObservation::Quiescent
16140        );
16141        let none = gauge_verdicts(None, &gauges);
16142        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16143    }
16144}
16145
16146#[cfg(test)]
16147mod health_evidence_tests {
16148    use super::{HealthProbeError, HealthProbeEvidence};
16149    use std::collections::HashSet;
16150
16151    /// The evidential asymmetry, asserted rather than described.
16152    ///
16153    /// Exactly ONE observation is proof a module cannot serve, and the one that
16154    /// fires under CPU starvation is not it. Before the split, all fifteen
16155    /// construction sites collapsed into a single String, so a timeout carried the
16156    /// same weight as a dead lane -- which is how a healthy module was restarted
16157    /// three times in one day.
16158    #[test]
16159    fn only_a_dead_lane_is_proof_of_death() {
16160        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16161        // Three non-proof classes, each for a different reason: silence is
16162        // consistent with health, a bad answer proves the module ALIVE, and a
16163        // daemon-side fault never reached the module at all.
16164        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16165        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16166        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16167    }
16168
16169    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
16170    ///
16171    /// A shared label renders two different observations identically in the line an
16172    /// operator reads after an unexplained restart -- the exact confusion this
16173    /// change removes.
16174    #[test]
16175    fn every_evidence_class_has_a_distinct_label() {
16176        let labels = [
16177            HealthProbeError::lane_dead("").label(),
16178            HealthProbeError::no_answer("").label(),
16179            HealthProbeError::bad_answer("").label(),
16180            HealthProbeError::misconfigured("").label(),
16181        ];
16182        let unique: HashSet<_> = labels.iter().collect();
16183        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16184    }
16185
16186    /// The class is additional information, not a replacement.
16187    ///
16188    /// An operator needs both "this was silence" and the specific text saying how
16189    /// long we waited; a classification that swallowed the message would trade one
16190    /// missing distinction for another.
16191    #[test]
16192    fn classification_preserves_the_original_message() {
16193        let err = HealthProbeError::no_answer("module did not answer within 5s");
16194        assert_eq!(err.to_string(), "module did not answer within 5s");
16195        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16196    }
16197}
16198
16199#[cfg(test)]
16200mod health_tombstone_tests {
16201    use std::{path::PathBuf, sync::Arc, time::Duration};
16202
16203    use subc_protocol::{
16204        manifest::Concurrency,
16205        session::{HealthStatus, ModuleControlResponse},
16206    };
16207    use tokio::sync::mpsc;
16208
16209    use super::{
16210        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16211        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16212    };
16213    use crate::{
16214        control::ControlHandler,
16215        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16216        registry::{ConnectionId, Registry},
16217        router::FrameSink,
16218    };
16219
16220    struct ProbeHarness {
16221        spec: ModuleSpec,
16222        runtime: SupervisorRuntimeConfig,
16223        forwarding: Arc<ForwardingTable>,
16224        module_connection: ConnectionId,
16225        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16226        handler: ControlHandler,
16227        module: super::SupervisedModule,
16228    }
16229
16230    fn probe_harness() -> ProbeHarness {
16231        let registry = Arc::new(Registry::default());
16232        let forwarding = Arc::new(ForwardingTable::default());
16233        let supervisor_handle = super::SupervisorHandle::new();
16234        let health = HealthConfig {
16235            http: None,
16236            cadence: Duration::from_secs(30),
16237            deadline: Duration::from_secs(5),
16238            failure_threshold: 3,
16239            on_degraded: HealthAction::Report,
16240            on_failing: HealthAction::Report,
16241            critical: false,
16242        };
16243        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16244            .with_forwarding(Arc::clone(&forwarding))
16245            .with_handle(supervisor_handle.clone())
16246            .with_health_config(health);
16247        let spec = ModuleSpec {
16248            module_id: "late-health-module".to_string(),
16249            program: PathBuf::from("disabled-module"),
16250            args: Vec::new(),
16251            env: Vec::new(),
16252            reserved: false,
16253            reserved_prefixes: Vec::new(),
16254            protocol: ModuleProtocol::Subc,
16255            overlap: Default::default(),
16256            carry_containment: false,
16257        };
16258        let module = supervisor
16259            .supervise_configured(spec.clone(), false)
16260            .unwrap();
16261        let runtime = supervisor.runtime_config();
16262        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16263            .with_supervisor(supervisor_handle);
16264        let module_connection = ConnectionId::new(700);
16265        let (module_tx, module_rx) = mpsc::channel(8);
16266        forwarding
16267            .register_module_connection(
16268                module_connection,
16269                spec.module_id.clone(),
16270                subc_protocol::PROTOCOL_VERSION,
16271                Concurrency::ModuleManaged,
16272                FrameSink::new(module_tx),
16273            )
16274            .unwrap();
16275
16276        ProbeHarness {
16277            spec,
16278            runtime,
16279            forwarding,
16280            module_connection,
16281            module_rx,
16282            handler,
16283            module,
16284        }
16285    }
16286
16287    async fn finish_after(
16288        harness: &mut ProbeHarness,
16289        stall: Duration,
16290    ) -> ModuleControlRpcCompletion {
16291        assert!(stall > harness.runtime.health.deadline);
16292        let deadline = harness.runtime.health.deadline;
16293        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16294        let answer = async {
16295            let frame = harness.module_rx.recv().await.expect("health.check frame");
16296            tokio::time::advance(deadline).await;
16297            tokio::task::yield_now().await;
16298            tokio::time::advance(stall - deadline).await;
16299            harness
16300                .forwarding
16301                .complete_module_control_rpc(
16302                    harness.module_connection,
16303                    frame.header.corr,
16304                    Some("health.check"),
16305                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16306                        status: HealthStatus::Ok,
16307                        detail: None,
16308                        metrics: None,
16309                    }),
16310                )
16311                .unwrap()
16312        };
16313        let (probe_result, completion) = tokio::join!(probe, answer);
16314        let err = probe_result.expect_err("probe must miss its deadline");
16315        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16316        completion
16317    }
16318
16319    async fn time_out_without_answer(harness: &mut ProbeHarness) {
16320        let deadline = harness.runtime.health.deadline;
16321        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16322        let exhaust_deadline = async {
16323            let _frame = harness.module_rx.recv().await.expect("health.check frame");
16324            tokio::time::advance(deadline).await;
16325            tokio::task::yield_now().await;
16326        };
16327        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16328        let err = probe_result.expect_err("probe must miss its deadline");
16329        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16330    }
16331
16332    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16333        let registry = Arc::clone(&harness.module.inner.registry);
16334        let snapshot = Arc::clone(&harness.module.inner.snapshot);
16335        let process_liveness = super::SupervisorProcessLiveness::default();
16336        let mut child = None;
16337        let cycle = super::run_health_probe_cycle(
16338            &harness.spec,
16339            &harness.runtime,
16340            &registry,
16341            &process_liveness,
16342            &snapshot,
16343            &mut child,
16344        );
16345        let peer = async {
16346            let frame = harness.module_rx.recv().await.expect("health.check frame");
16347            if answer {
16348                harness
16349                    .forwarding
16350                    .complete_module_control_rpc(
16351                        harness.module_connection,
16352                        frame.header.corr,
16353                        Some("health.check"),
16354                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16355                            status: HealthStatus::Ok,
16356                            detail: None,
16357                            metrics: Some(serde_json::json!({"ready": true})),
16358                        }),
16359                    )
16360                    .unwrap();
16361            } else {
16362                tokio::time::advance(harness.runtime.health.deadline).await;
16363                tokio::task::yield_now().await;
16364            }
16365        };
16366        tokio::join!(cycle, peer);
16367    }
16368
16369    #[tokio::test(start_paused = true)]
16370    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16371        let mut harness = probe_harness();
16372        // Drive the probe cycle directly with an in-memory wire peer. Stop the
16373        // disabled module's monitor so only this test owns lifecycle transitions;
16374        // no OS process is launched, and a restart is observed at scheduling.
16375        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16376        monitor.abort();
16377        let _ = monitor.await;
16378        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16379            state.enabled = true;
16380            state.state = super::ModuleState::Running;
16381            state.process_alive = true;
16382        })
16383        .unwrap();
16384
16385        run_probe_cycle(&mut harness, true).await;
16386        assert_eq!(
16387            harness.module.status().unwrap().health.status,
16388            super::SupervisorHealthStatus::Ok
16389        );
16390
16391        for failures in 1..harness.runtime.health.failure_threshold {
16392            run_probe_cycle(&mut harness, false).await;
16393            let status = harness.module.status().unwrap();
16394            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16395            assert_eq!(status.health.consecutive_failures, failures);
16396            assert!(status.health.last_probe_ms.is_some());
16397            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16398            assert!(status.health.metrics.is_none());
16399            assert_eq!(status.state, super::ModuleState::Running);
16400            assert!(status.process_alive);
16401            assert_eq!(status.restart_count, 0);
16402            assert_eq!(status.lifetime_restarts, 0);
16403            assert!(status.health.last_action.is_none());
16404            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16405        }
16406
16407        run_probe_cycle(&mut harness, true).await;
16408        let recovered = harness.module.status().unwrap();
16409        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16410        assert_eq!(recovered.health.consecutive_failures, 0);
16411        assert!(recovered.health.detail.is_none());
16412        assert_eq!(
16413            recovered.health.metrics,
16414            Some(serde_json::json!({"ready": true}))
16415        );
16416        assert_eq!(recovered.lifetime_restarts, 0);
16417
16418        for failures in 1..=harness.runtime.health.failure_threshold {
16419            run_probe_cycle(&mut harness, false).await;
16420            let status = harness.module.status().unwrap();
16421            assert_eq!(status.health.consecutive_failures, failures);
16422            if failures < harness.runtime.health.failure_threshold {
16423                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16424                assert_eq!(status.state, super::ModuleState::Running);
16425                assert_eq!(status.lifetime_restarts, 0);
16426                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16427            } else {
16428                assert_eq!(
16429                    status.health.status,
16430                    super::SupervisorHealthStatus::Unresponsive
16431                );
16432                assert_eq!(status.state, super::ModuleState::Restarting);
16433                assert_eq!(status.restart_count, 1);
16434                assert_eq!(status.lifetime_restarts, 1);
16435                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16436                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16437            }
16438        }
16439    }
16440
16441    #[tokio::test(start_paused = true)]
16442    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16443        let mut harness = probe_harness();
16444
16445        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16446        let first_latency = match &first {
16447            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16448            other => panic!("late answer was not retained: {other:?}"),
16449        };
16450        assert!(harness.handler.observe_module_control_completion(first));
16451
16452        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16453        let second_latency = match &second {
16454            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16455            other => panic!("late answer was not retained: {other:?}"),
16456        };
16457        assert!(harness.handler.observe_module_control_completion(second));
16458
16459        assert_eq!(first_latency, Duration::from_secs(8));
16460        assert_eq!(
16461            second_latency - first_latency,
16462            Duration::from_secs(3),
16463            "latency must grow linearly with the additional stall"
16464        );
16465        let health = harness.module.status().unwrap().health;
16466        assert_eq!(health.late_answer_count, 2);
16467        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16468    }
16469
16470    /// A module that answers every probe late must never march to the kill
16471    /// threshold: the late answer proves it is alive, so it must clear the miss
16472    /// streak the timeout recorded. Without the reset, a CPU-starved module
16473    /// that serves every probe seconds past the deadline accumulates
16474    /// `consecutive_failures` to the threshold and is killed — the exact
16475    /// sequence from the 2026-08-14 aft disable, where the daemon logged
16476    /// "proves the module is alive" five times while counting five misses.
16477    #[tokio::test(start_paused = true)]
16478    async fn late_answer_clears_the_consecutive_failure_streak() {
16479        let mut harness = probe_harness();
16480
16481        // Timeout recorded first: the probe path saw no answer in time.
16482        time_out_without_answer(&mut harness).await;
16483        harness
16484            .module
16485            .record_health_probe_failure_for_test("[no-answer] test miss")
16486            .unwrap();
16487        assert_eq!(
16488            harness.module.status().unwrap().health.consecutive_failures,
16489            1,
16490            "precondition: the miss must be on the streak before the late answer"
16491        );
16492
16493        // The stalled reply then lands: proof of life.
16494        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16495        assert!(matches!(
16496            late,
16497            ModuleControlRpcCompletion::LateHealthAnswer { .. }
16498        ));
16499        assert!(harness.handler.observe_module_control_completion(late));
16500
16501        let health = harness.module.status().unwrap().health;
16502        assert_eq!(
16503            health.consecutive_failures, 0,
16504            "a late answer is an answer: the streak must reset"
16505        );
16506        assert_eq!(health.late_answer_count, 1);
16507    }
16508
16509    #[tokio::test(start_paused = true)]
16510    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16511        let mut harness = probe_harness();
16512
16513        for _ in 0..20 {
16514            time_out_without_answer(&mut harness).await;
16515            assert_eq!(
16516                harness.forwarding.health_probe_tombstone_count().unwrap(),
16517                1
16518            );
16519        }
16520    }
16521}
16522
16523#[cfg(test)]
16524mod child_env_tests {
16525    #[cfg(windows)]
16526    use super::{
16527        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16528    };
16529    #[cfg(windows)]
16530    use std::sync::{Arc, OnceLock};
16531    #[cfg(windows)]
16532    #[test]
16533    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16534        let spec = spec(Vec::new());
16535        let mut command = Command::new("unused");
16536        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16537            .unwrap()
16538            .unwrap();
16539        let name = command
16540            .as_std()
16541            .get_envs()
16542            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16543            .and_then(|(_, value)| value)
16544            .unwrap();
16545        assert_eq!(name, OsStr::new(handoff.name()));
16546        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16547        assert!(command.as_std().get_envs().any(|(key, value)| key
16548            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16549            && value == Some(OsStr::new("env"))));
16550        assert!(command
16551            .as_std()
16552            .get_envs()
16553            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16554    }
16555
16556    #[cfg(windows)]
16557    #[test]
16558    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16559        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16560        snapshot.process_alive = true;
16561        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16562        assert_eq!(
16563            observed_launch_nonce_source(&snapshot).as_deref(),
16564            Some("env")
16565        );
16566        let consumed = Arc::new(OnceLock::new());
16567        snapshot.pipe_consumed = Some(consumed.clone());
16568        assert_eq!(
16569            observed_launch_nonce_source(&snapshot).as_deref(),
16570            Some("env")
16571        );
16572        consumed.set(()).unwrap();
16573        assert_eq!(
16574            observed_launch_nonce_source(&snapshot).as_deref(),
16575            Some("pipe")
16576        );
16577        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16578        assert_eq!(
16579            observed_launch_nonce_source(&snapshot).as_deref(),
16580            Some("env")
16581        );
16582        clear_current_process_facts(&mut snapshot);
16583        assert_eq!(observed_launch_nonce_source(&snapshot), None);
16584    }
16585    use super::{
16586        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16587        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16588        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16589    };
16590    use std::{ffi::OsStr, path::PathBuf};
16591    use tokio::process::Command;
16592
16593    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16594        ModuleSpec {
16595            module_id: "env-plan".to_string(),
16596            program: PathBuf::from("/nonexistent"),
16597            args: Vec::new(),
16598            env,
16599            reserved: false,
16600            reserved_prefixes: Vec::new(),
16601            protocol: ModuleProtocol::Subc,
16602            overlap: Default::default(),
16603            carry_containment: false,
16604        }
16605    }
16606
16607    #[test]
16608    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16609        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16610        let digest = LaunchSpecDigest::from_spec(&base);
16611        let mut reordered = base.clone();
16612        reordered.env.reverse();
16613        reordered.module_id = "different-id".into();
16614        reordered.reserved = true;
16615        reordered.overlap = Default::default();
16616        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16617        let mut delimiter_collision = base.clone();
16618        delimiter_collision.env[0] = ("A=B".into(), "C".into());
16619        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16620        let mut changed = base.clone();
16621        changed.protocol = ModuleProtocol::None;
16622        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16623        changed = base.clone();
16624        changed.args.push("--flag".into());
16625        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16626        changed = base.clone();
16627        changed.program = PathBuf::from("/other");
16628        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16629        changed = base.clone();
16630        changed.env[1].1 = "other".into();
16631        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16632        assert!(!format!("{digest:?}").contains("B=C"));
16633        assert!(!format!("{digest:?}").contains("[u8"));
16634    }
16635
16636    #[cfg(unix)]
16637    #[test]
16638    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16639        use std::os::unix::ffi::OsStringExt;
16640        let mut left = spec(Vec::new());
16641        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16642        let mut right = left.clone();
16643        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16644        assert_eq!(
16645            left.program.to_string_lossy(),
16646            right.program.to_string_lossy()
16647        );
16648        assert_ne!(
16649            LaunchSpecDigest::from_spec(&left),
16650            LaunchSpecDigest::from_spec(&right)
16651        );
16652    }
16653
16654    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
16655    /// one still gets its own.
16656    ///
16657    /// This is the narrow goal `env_clear()` was reached for, and the reason the
16658    /// fix is `env_remove` rather than deleting the line: an operator's ambient
16659    /// filter silently becoming an unconfigured module's log level is a real
16660    /// defect, just a much smaller one than clearing the environment.
16661    ///
16662    /// Asserted on the command plan rather than a spawned child because proving
16663    /// the ABSENCE of an inherited variable needs the parent's environment
16664    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
16665    /// removal as `(key, None)`, which is exactly the distinction wanted: not
16666    /// "absent because nobody set it" but "explicitly unset for the child".
16667    #[test]
16668    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16669        let mut command = Command::new("/nonexistent");
16670        apply_child_env(&mut command, &spec(Vec::new()));
16671        let removed = command
16672            .as_std()
16673            .get_envs()
16674            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16675        assert!(
16676            removed,
16677            "ambient CK_LOG must be explicitly removed for an unconfigured module"
16678        );
16679
16680        let mut configured = Command::new("/nonexistent");
16681        apply_child_env(
16682            &mut configured,
16683            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16684        );
16685        let effective = configured
16686            .as_std()
16687            .get_envs()
16688            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16689            .last()
16690            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16691        assert_eq!(
16692            effective,
16693            Some(Some("debug".to_string())),
16694            "a module's configured CK_LOG must survive the ambient removal"
16695        );
16696    }
16697
16698    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
16699    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
16700    /// the same reason as the CK_LOG test above.
16701    ///
16702    /// The argument is the load-bearing half: a stock binary exits on an
16703    /// unknown flag before it listens, so with `--subc` appended the mode
16704    /// could not supervise the one process it exists for. Found by the first
16705    /// conformance run (nats-server: `flag provided but not defined: -subc`).
16706    #[test]
16707    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16708        let connection_file = std::path::Path::new("/run/subc-connection.json");
16709        let handle = SupervisorHandle::new();
16710
16711        let mut none_spec = spec(Vec::new());
16712        none_spec.protocol = ModuleProtocol::None;
16713        let mut none = Command::new("/nonexistent");
16714        let none_handoff =
16715            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16716                .expect("protocol-none spawn args apply");
16717        assert!(
16718            none_handoff.is_none(),
16719            "protocol:none spawn must not receive a nonce descriptor"
16720        );
16721        #[cfg(windows)]
16722        assert!(!none.as_std().get_envs().any(|(key, value)| key
16723            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16724            && value.is_some()));
16725        assert!(
16726            !none.as_std().get_envs().any(|(key, value)| key
16727                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16728                && value.is_some()),
16729            "protocol:none spawn must not name a nonce descriptor"
16730        );
16731        let none_args: Vec<String> = none
16732            .as_std()
16733            .get_args()
16734            .map(|a| a.to_string_lossy().into_owned())
16735            .collect();
16736        assert!(
16737            !none_args.iter().any(|a| a == SUBC_ARG),
16738            "protocol:none argv must not carry --subc; got {none_args:?}"
16739        );
16740        let none_has_nonce = none
16741            .as_std()
16742            .get_envs()
16743            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16744        assert!(
16745            !none_has_nonce,
16746            "protocol:none spawn must not receive a launch nonce"
16747        );
16748        let none_has_module_id = none
16749            .as_std()
16750            .get_envs()
16751            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16752        assert!(
16753            none_has_module_id,
16754            "SUBC_MODULE_ID is inert and stays on every path"
16755        );
16756        assert!(
16757            handle.spawn_nonce(&none_spec.module_id).is_none(),
16758            "no nonce record for a process that will never present one"
16759        );
16760
16761        // Control: the subc-wire path is unchanged by the branch above.
16762        let wire_spec = spec(Vec::new());
16763        let mut wire = Command::new("/nonexistent");
16764        let wire_handoff =
16765            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16766                .expect("subc-wire spawn args apply");
16767        let wire_fd_env = wire
16768            .as_std()
16769            .get_envs()
16770            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16771            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16772        #[cfg(unix)]
16773        assert_eq!(
16774            wire_fd_env,
16775            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16776            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16777        );
16778        #[cfg(windows)]
16779        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16780        #[cfg(not(any(unix, windows)))]
16781        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16782        let wire_args: Vec<String> = wire
16783            .as_std()
16784            .get_args()
16785            .map(|a| a.to_string_lossy().into_owned())
16786            .collect();
16787        assert_eq!(
16788            wire_args,
16789            vec![
16790                SUBC_ARG.to_string(),
16791                connection_file.to_string_lossy().into_owned()
16792            ],
16793            "a subc-wire spawn still carries --subc <path>"
16794        );
16795        assert_eq!(
16796            wire.as_std()
16797                .get_envs()
16798                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16799            !cfg!(unix),
16800            "only Windows supplies the environment nonce"
16801        );
16802        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16803    }
16804
16805    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16806    /// spec tries to set it; only a swap candidate carries it.
16807    ///
16808    /// "Set it only on candidates" is not enough, because spawn applies the
16809    /// spec's env verbatim and the daemon's own environment is inherited: either
16810    /// could hand a plain restart the swap role, and a module reading it would
16811    /// warm on its long swap budget while callers wait. Asserted as an explicit
16812    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16813    /// test above gives.
16814    #[test]
16815    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16816        let role = |command: &Command| {
16817            command
16818                .as_std()
16819                .get_envs()
16820                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16821                .last()
16822                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16823        };
16824        let forged = spec(vec![(
16825            SUBC_SPAWN_ROLE_ENV.to_string(),
16826            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16827        )]);
16828
16829        let mut plain = Command::new("/nonexistent");
16830        apply_child_env(&mut plain, &forged);
16831        apply_spawn_role(&mut plain, SpawnRole::Plain);
16832        assert_eq!(
16833            role(&plain),
16834            Some(None),
16835            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16836        );
16837
16838        let mut candidate = Command::new("/nonexistent");
16839        apply_child_env(&mut candidate, &spec(Vec::new()));
16840        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16841        assert_eq!(
16842            role(&candidate),
16843            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16844        );
16845    }
16846
16847    /// Daemon-private capture retention keys never reach the child.
16848    ///
16849    /// cortexkit-log exposes retention as a Rust struct with no environment
16850    /// names, so these entries are supervisor metadata. Passing them through
16851    /// would invent a public child-process contract by accident.
16852    #[test]
16853    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16854        let mut command = Command::new("/nonexistent");
16855        apply_child_env(
16856            &mut command,
16857            &spec(vec![
16858                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16859                ("KEPT".to_string(), "yes".to_string()),
16860            ]),
16861        );
16862        let keys: Vec<String> = command
16863            .as_std()
16864            .get_envs()
16865            .filter(|(_, value)| value.is_some())
16866            .map(|(key, _)| key.to_string_lossy().into_owned())
16867            .collect();
16868        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16869        assert!(
16870            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16871            "daemon-private capture key leaked to the child: {keys:?}"
16872        );
16873    }
16874}
16875
16876#[cfg(test)]
16877mod jitter_tests {
16878    use super::jittered_health_delay;
16879    use std::{collections::HashSet, time::Duration};
16880
16881    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16882    /// that actually occur rather than invented ones.
16883    ///
16884    /// This is a SAMPLE, not a registry: the property under test is that distinct
16885    /// ids disperse, which holds for any set of distinct strings. Several entries
16886    /// are already historical (modules get renamed), and that costs nothing here --
16887    /// but it means a reader must not mistake this for the live module set, and a
16888    /// rename sweep will match it without there being anything to change.
16889    const FLEET: [&str; 14] = [
16890        "aft",
16891        "alfonso-core",
16892        "magic-context",
16893        "broca",
16894        "thalamus",
16895        "quota",
16896        "engram",
16897        "plexus",
16898        "cerebellum",
16899        "astrocyte",
16900        "synapse",
16901        "subc-mcp",
16902        "cortexkit-credentials",
16903        "subc-federation",
16904    ];
16905
16906    /// Probes must not converge after a fleet-wide restart.
16907    ///
16908    /// This is the property the jitter exists for: every module reconnects at
16909    /// once, and without dispersal all fourteen would then probe on the same
16910    /// tick forever. Nothing failed visibly when this went untested -- a
16911    /// convergent fleet still probes correctly, just in a burst, so the symptom
16912    /// is a periodic load spike that looks like whatever else is running.
16913    #[test]
16914    fn probe_delays_disperse_across_the_fleet() {
16915        let cadence = Duration::from_secs(30);
16916        let delays: HashSet<Duration> = FLEET
16917            .iter()
16918            .map(|id| jittered_health_delay(id, 0, cadence))
16919            .collect();
16920        assert_eq!(
16921            delays.len(),
16922            FLEET.len(),
16923            "every supervised module must land on its own probe offset"
16924        );
16925    }
16926
16927    /// The offset may only ever DELAY a probe, never bring it forward.
16928    ///
16929    /// A delay below the cadence would probe a module more often than
16930    /// configured, which is the opposite of what an operator asked for and
16931    /// would tighten the failure budget without anyone changing it.
16932    #[test]
16933    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16934        let cadence = Duration::from_secs(30);
16935        let span = cadence / 10;
16936        for id in FLEET {
16937            for probe_index in 0..8 {
16938                let delay = jittered_health_delay(id, probe_index, cadence);
16939                assert!(
16940                    delay >= cadence,
16941                    "{id}#{probe_index}: jitter must not shorten the cadence"
16942                );
16943                assert!(
16944                    delay < cadence + span,
16945                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16946                );
16947            }
16948        }
16949    }
16950
16951    /// A module keeps its offset across daemon restarts.
16952    ///
16953    /// The delay is derived rather than randomised precisely so a restart does
16954    /// not re-roll every module into a fresh chance of collision. A random
16955    /// source would satisfy the dispersal test above and quietly lose this.
16956    #[test]
16957    fn a_module_offset_is_stable_across_restarts() {
16958        let cadence = Duration::from_secs(30);
16959        for id in FLEET {
16960            assert_eq!(
16961                jittered_health_delay(id, 0, cadence),
16962                jittered_health_delay(id, 0, cadence),
16963                "{id}: the same module and probe index must produce the same offset"
16964            );
16965        }
16966    }
16967
16968    /// A zero cadence disables probing rather than producing a busy loop.
16969    #[test]
16970    fn zero_cadence_yields_zero_delay() {
16971        assert_eq!(
16972            jittered_health_delay("aft", 0, Duration::ZERO),
16973            Duration::ZERO
16974        );
16975    }
16976}
16977
16978#[cfg(all(test, target_os = "linux"))]
16979mod cgroup_placement_tests {
16980    use super::{
16981        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16982        SuperviseError, SupervisedChild,
16983    };
16984    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16985    use cortexkit_test_support::ScratchDir;
16986    use std::{
16987        fs, io,
16988        path::{Path, PathBuf},
16989        sync::{Arc, Mutex},
16990    };
16991    use tokio::process::Command;
16992
16993    #[tokio::test]
16994    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16995        use super::*;
16996        let dir = ScratchDir::new("unique-spawn-cgroups");
16997        let root = PathBuf::from(format!(
16998            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16999            std::process::id(),
17000            unix_ms_now()
17001        ));
17002        if let Err(error) = fs::create_dir(&root) {
17003            assert!(
17004                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17005                "required cgroup test cannot execute: {error}"
17006            );
17007            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
17008            return;
17009        }
17010        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
17011        let group_count = || {
17012            fs::read_dir(root.join("subc-modules"))
17013                .unwrap()
17014                .map(|entry| entry.unwrap().file_type().unwrap())
17015                .filter(|kind| kind.is_dir())
17016                .count()
17017        };
17018        let supervisor =
17019            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17020                .with_cgroup_placement(Some(placement.clone()));
17021        let runtime = supervisor.runtime_config();
17022        let mut spec = ModuleSpec {
17023            module_id: "unique-spawn".into(),
17024            program: PathBuf::from("/bin/sleep"),
17025            args: vec!["60".into()],
17026            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17027                .into_iter()
17028                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17029                .collect(),
17030            reserved: false,
17031            reserved_prefixes: vec![],
17032            protocol: ModuleProtocol::None,
17033            overlap: Default::default(),
17034            carry_containment: false,
17035        };
17036        let spawn = |spec: &ModuleSpec| {
17037            spawn_child(
17038                spec,
17039                None,
17040                None,
17041                &runtime.stderr_ring,
17042                None,
17043                &runtime.child_roster,
17044                Some(&placement),
17045            )
17046            .unwrap()
17047        };
17048        let mut live = spawn(&spec);
17049        for _ in 0..3 {
17050            // A new process can enter the old slot while retirement is pending.
17051            let next = spawn(&spec);
17052            assert_ne!(live.module_id, next.module_id);
17053            live.start_kill().unwrap();
17054            live.wait().await.unwrap();
17055            live = next;
17056            assert!(
17057                live.child.try_wait().unwrap().is_none(),
17058                "retiring the old slot must not kill the replacement"
17059            );
17060            assert_eq!(
17061                group_count(),
17062                1,
17063                "only the live spawn's cgroup should remain"
17064            );
17065        }
17066        supervisor.begin_daemon_shutdown();
17067        let reap = tokio::spawn(async move {
17068            live.wait().await.unwrap();
17069        });
17070        supervisor
17071            .end_children_for_daemon_shutdown(false, std::future::pending())
17072            .await;
17073        reap.await.unwrap();
17074        assert_eq!(group_count(), 0);
17075        // A normal exit uses the same tree-cleanup path as a killed spawn.
17076        spec.program = PathBuf::from("/bin/true");
17077        spec.args.clear();
17078        let fresh_roster = ChildRoster::default();
17079        let mut short = spawn_child(
17080            &spec,
17081            None,
17082            None,
17083            &runtime.stderr_ring,
17084            None,
17085            &fresh_roster,
17086            Some(&placement),
17087        )
17088        .unwrap();
17089        short.wait().await.unwrap();
17090        assert_eq!(group_count(), 0);
17091        spec.module_id = "_".repeat(255);
17092        let mut long_id = spawn_child(
17093            &spec,
17094            None,
17095            None,
17096            &runtime.stderr_ring,
17097            None,
17098            &fresh_roster,
17099            Some(&placement),
17100        )
17101        .unwrap();
17102        long_id.wait().await.unwrap();
17103        assert_eq!(
17104            group_count(),
17105            0,
17106            "valid long module IDs must not exceed cgroup NAME_MAX"
17107        );
17108        fs::remove_dir(root.join("subc-modules")).unwrap();
17109        fs::remove_dir(root).unwrap();
17110        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17111    }
17112
17113    #[test]
17114    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17115        let path = Path::new("/definitely-missing-subc-cgroup");
17116        let mut command = Command::new("true");
17117        let error = apply_cgroup_placement(
17118            &mut command,
17119            &ModuleSpec {
17120                module_id: "broken-cgroup".to_string(),
17121                program: PathBuf::from("true"),
17122                args: Vec::new(),
17123                env: Vec::new(),
17124                reserved: false,
17125                reserved_prefixes: Vec::new(),
17126                protocol: ModuleProtocol::Subc,
17127                overlap: Default::default(),
17128                carry_containment: false,
17129            },
17130            path,
17131        )
17132        .expect_err("a parent cgroup open failure must reject the supervised spawn");
17133        let reason = error.to_string();
17134
17135        assert!(
17136            matches!(error, SuperviseError::Cgroup { .. }),
17137            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17138        );
17139        assert!(
17140            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17141            "parent cgroup open failure must name cgroup.procs: {reason}"
17142        );
17143    }
17144
17145    #[tokio::test]
17146    async fn reaping_a_child_removes_its_empty_module_cgroup() {
17147        let root = ScratchDir::new("supervisor-reap-cgroup");
17148        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17149        let placement = subc_cgroup::prepare_at(&root)
17150            .expect("prepare scratch cgroup root")
17151            .expect("scratch root has a cgroup.procs marker");
17152        let module_id = "reaped-module";
17153        let module = placement
17154            .module_path(module_id)
17155            .expect("create scratch module cgroup");
17156        let child = Command::new("true")
17157            .env("XDG_DATA_HOME", root.path())
17158            .env("XDG_RUNTIME_DIR", root.path())
17159            .env("XDG_CONFIG_HOME", root.path())
17160            .spawn()
17161            .expect("spawn short-lived child");
17162        let pid = child.id().expect("spawned child has pid");
17163        let mut child = SupervisedChild {
17164            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17165            child: crate::supervise::containment::ProcessChild::plain(child, pid),
17166            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17167            protocol: ModuleProtocol::Subc,
17168            module_id: module_id.to_string(),
17169            cgroup_placement: Some(placement),
17170            stdout_pump: None,
17171            stderr_pump: None,
17172            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17173            spawned_at: tokio::time::Instant::now(),
17174            registration_deadline: None,
17175            registration_watch: None,
17176            spawned_at_ms: 0,
17177            spawned_from: PathBuf::from("true"),
17178            launch_spec_digest: LaunchSpecDigest([0; 32]),
17179            spawned_file_identity: None,
17180            process_start_time: None,
17181            process_identity: None,
17182            pid,
17183            roster_guard: None,
17184            #[cfg(target_os = "macos")]
17185            privacy_exec: None,
17186            spawn_failure: None,
17187        };
17188
17189        child.wait().await.expect("reap short-lived child");
17190
17191        assert!(
17192            !module.exists(),
17193            "waiting for a non-carried child must remove its empty cgroup before returning"
17194        );
17195        child.finish_exit(false).await.unwrap();
17196    }
17197
17198    #[test]
17199    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17200        let root = ScratchDir::new("supervisor-non-empty-cgroup");
17201        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17202        let placement = subc_cgroup::prepare_at(&root)
17203            .expect("prepare scratch cgroup root")
17204            .expect("scratch root has a cgroup.procs marker");
17205        let module = placement
17206            .module_path("surviving-module")
17207            .expect("create scratch module cgroup");
17208        fs::write(module.join("surviving-process"), b"still present")
17209            .expect("make scratch cgroup non-empty");
17210        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17211
17212        remove_module_cgroup(&placement, "surviving-module");
17213
17214        let logs = crate::router::test_log::captured_logs(&logs);
17215        assert!(
17216            module.exists(),
17217            "failed removal must leave the cgroup intact"
17218        );
17219        assert!(
17220            logs.contains("could not remove module cgroup after process exit; continuing teardown")
17221                && logs.contains("surviving-module"),
17222            "best-effort removal must report the failure without returning it: {logs}"
17223        );
17224    }
17225
17226    #[test]
17227    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17228        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17229        let reason = SuperviseError::Spawn {
17230            program: PathBuf::from("/bin/true"),
17231            source: io::Error::from_raw_os_error(13),
17232            cgroup_path: Some(cgroup_path.clone()),
17233        }
17234        .to_string();
17235
17236        assert!(
17237            reason.contains(&cgroup_path.display().to_string()),
17238            "a pre_exec spawn failure must name the cgroup path: {reason}"
17239        );
17240    }
17241}
17242
17243#[cfg(test)]
17244mod spawn_subscriber_lag_tests {
17245    use super::*;
17246
17247    /// A subscriber whose connection stops draining is dropped once its frame
17248    /// channel fills. The client must learn that from a terminal Error frame
17249    /// after the frames already queued for it, not from a stream that simply
17250    /// goes quiet.
17251    #[tokio::test]
17252    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17253        let feed = SpawnEventFeed::default();
17254        feed.configure_incarnation("lag-incarnation".to_string());
17255        // A one-slot connection queue that nobody reads until the emits are
17256        // done: the forwarder parks on it and the subscriber channel fills.
17257        let (tx, mut rx) = mpsc::channel(1);
17258        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17259            .expect("subscribe");
17260        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17261        for index in 0..emitted {
17262            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17263            // Let the forwarder take what it can so the fill point is the
17264            // subscriber channel, not a scheduling accident.
17265            tokio::task::yield_now().await;
17266        }
17267        assert_eq!(
17268            feed.subscriber_count(),
17269            0,
17270            "the lagged subscriber must be removed"
17271        );
17272
17273        let mut data = Vec::new();
17274        let mut last = None;
17275        loop {
17276            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17277                .await
17278                .expect("the forwarder must finish once the subscriber is dropped");
17279            let Some(outbound) = next else { break };
17280            let frame = outbound.frame;
17281            if frame.header.ty == FrameType::StreamData {
17282                assert!(last.is_none(), "no data may follow the terminal frame");
17283                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17284                data.push(event.cursor.seq);
17285            } else {
17286                assert!(last.is_none(), "exactly one terminal frame");
17287                last = Some(frame);
17288            }
17289        }
17290        assert!(!data.is_empty(), "queued frames drain before the terminal");
17291        for pair in data.windows(2) {
17292            assert_eq!(
17293                pair[1],
17294                pair[0] + 1,
17295                "queued frames arrive dense and in order"
17296            );
17297        }
17298        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17299        assert_eq!(terminal.header.ty, FrameType::Error);
17300        assert_eq!(terminal.header.corr, 7);
17301        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17302        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17303        let detail = body.detail.expect("lagged error carries detail");
17304        assert_eq!(
17305            detail["first_undelivered_cursor"]["seq"],
17306            data.last().unwrap() + 1,
17307            "the named cursor is the first event the subscriber did not receive"
17308        );
17309        assert_eq!(
17310            detail["first_undelivered_cursor"]["daemon_incarnation"],
17311            "lag-incarnation"
17312        );
17313    }
17314}
17315
17316#[cfg(test)]
17317mod terminal_history_read_concurrency_tests {
17318    use super::*;
17319    use crate::terminal_journal::read_pause;
17320    use cortexkit_test_support::ScratchDir;
17321    use std::sync::mpsc as std_mpsc;
17322
17323    fn journaled_ring(
17324        journal: &Arc<crate::terminal_journal::TerminalJournal>,
17325    ) -> Arc<Mutex<TerminalRing>> {
17326        Arc::new(Mutex::new(
17327            TerminalRing::new(TerminalRingConfig::default(), 1)
17328                .with_journal(Some(Arc::clone(journal))),
17329        ))
17330    }
17331
17332    fn crash(at_ms: u64) -> ExitReport {
17333        ExitReport {
17334            kind: ExitKind::Crash,
17335            code: Some(1),
17336            signal: None,
17337            at_ms,
17338        }
17339    }
17340
17341    /// Record an exit on another thread and report whether it finished within
17342    /// `bound`. The recorder thread is left running if it did not.
17343    fn record_within(
17344        module_id: &'static str,
17345        ring: &Arc<Mutex<TerminalRing>>,
17346        at_ms: u64,
17347        bound: Duration,
17348    ) -> bool {
17349        let ring = Arc::clone(ring);
17350        let (done, done_rx) = std_mpsc::channel();
17351        std::thread::spawn(move || {
17352            record_terminal(
17353                module_id,
17354                &ring,
17355                &SpawnEventFeed::default(),
17356                &crash(at_ms),
17357                TerminalDisposition::Restarting,
17358            );
17359            let _ = done.send(());
17360        });
17361        done_rx.recv_timeout(bound).is_ok()
17362    }
17363
17364    /// A history read in progress must not hold the journal writer (which every
17365    /// module's exit recording needs) or the module's own ring. Exits recorded
17366    /// while the read is paused complete promptly; the paused read answers as of
17367    /// the moment it started, and the next read has each exit exactly once.
17368    #[test]
17369    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17370        let dir = ScratchDir::new("terminal-history-concurrent-read");
17371        let path = dir.join("terminals.jsonl");
17372        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17373            path.clone(),
17374            "daemon".into(),
17375        ));
17376        let reader_ring = journaled_ring(&journal);
17377        let other_ring = journaled_ring(&journal);
17378        assert!(record_within(
17379            "reader-module",
17380            &reader_ring,
17381            10,
17382            Duration::from_secs(5)
17383        ));
17384
17385        let (started, release) = read_pause::install(&path);
17386        let reading = {
17387            let ring = Arc::clone(&reader_ring);
17388            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17389        };
17390        started
17391            .recv_timeout(Duration::from_secs(5))
17392            .expect("the history read reached its pause");
17393
17394        let bound = Duration::from_secs(1);
17395        assert!(
17396            record_within("other-module", &other_ring, 20, bound),
17397            "another module's exit waited on a history read (journal writer held)"
17398        );
17399        assert!(
17400            record_within("reader-module", &reader_ring, 30, bound),
17401            "the read module's own exit waited on its history read (ring held)"
17402        );
17403
17404        drop(release);
17405        let paused = reading.join().unwrap();
17406        assert_eq!(
17407            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17408            vec![10],
17409            "an exit recorded after the read began lands in neither half of it"
17410        );
17411        assert_eq!(paused.journal_skipped_lines, 0);
17412        assert_eq!(paused.journal_read_errors, 0);
17413
17414        let after = durable_terminal_history_of(&reader_ring, "reader-module");
17415        assert_eq!(
17416            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17417            vec![10, 30],
17418            "the next read merges ring and journal with no duplicate"
17419        );
17420        assert_eq!(after.journal_skipped_lines, 0);
17421    }
17422}
17423
17424/// What a restart does with the exited process's output readers. These drive
17425/// the same settle functions the supervisor calls, with a reader the test
17426/// holds, so a reader that has not been scheduled by the bound is a controlled
17427/// input rather than something only a loaded machine produces.
17428#[cfg(test)]
17429mod stderr_settle_tests {
17430    use std::{
17431        future::Future,
17432        io,
17433        pin::Pin,
17434        sync::{Arc, Mutex},
17435        task::{Context, Poll},
17436        time::Duration,
17437    };
17438
17439    use tokio::{
17440        io::{AsyncRead, ReadBuf},
17441        sync::oneshot,
17442        time::Instant,
17443    };
17444
17445    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17446    use crate::stderr_tail::{
17447        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17448        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17449    };
17450
17451    const BOUND: Duration = Duration::from_millis(250);
17452
17453    /// Yields `before`, then stays pending until the gate is released, then
17454    /// yields `after` and reaches EOF. The bytes after the gate were written
17455    /// by a process that has already exited; only the reader is behind.
17456    struct HeldReader {
17457        before: Option<Vec<u8>>,
17458        gate: Option<oneshot::Receiver<()>>,
17459        after: io::Cursor<Vec<u8>>,
17460    }
17461
17462    impl AsyncRead for HeldReader {
17463        fn poll_read(
17464            mut self: Pin<&mut Self>,
17465            cx: &mut Context<'_>,
17466            buf: &mut ReadBuf<'_>,
17467        ) -> Poll<io::Result<()>> {
17468            if let Some(bytes) = self.before.take() {
17469                buf.put_slice(&bytes);
17470                return Poll::Ready(Ok(()));
17471            }
17472            if let Some(gate) = self.gate.as_mut() {
17473                match Pin::new(gate).poll(cx) {
17474                    Poll::Pending => return Poll::Pending,
17475                    Poll::Ready(_) => self.gate = None,
17476                }
17477            }
17478            Pin::new(&mut self.after).poll_read(cx, buf)
17479        }
17480    }
17481
17482    struct DiscardSink;
17483
17484    impl OutputSink for DiscardSink {
17485        fn write_line(&mut self, _line: &[u8]) {}
17486    }
17487
17488    fn line(text: &str) -> TailEntry {
17489        TailEntry::Line {
17490            text: text.to_string(),
17491            truncated: false,
17492            at_ms: None,
17493        }
17494    }
17495
17496    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17497        ring.lock().unwrap()
17498    }
17499
17500    /// Start a reader for a new process generation that delivers `before`
17501    /// immediately and `after` only once the returned sender fires (or is
17502    /// dropped).
17503    fn held_pump(
17504        ring: &Arc<Mutex<StderrRing>>,
17505        before: &str,
17506        after: &str,
17507    ) -> (StderrPump, oneshot::Sender<()>) {
17508        let generation = lock(ring).begin_process();
17509        let (release, gate) = oneshot::channel();
17510        let reader = HeldReader {
17511            before: Some(before.as_bytes().to_vec()),
17512            gate: Some(gate),
17513            after: io::Cursor::new(after.as_bytes().to_vec()),
17514        };
17515        let task = tokio::spawn(pump_stderr_to(
17516            reader,
17517            Arc::clone(ring),
17518            generation,
17519            DiscardSink,
17520        ));
17521        (StderrPump { task, generation }, release)
17522    }
17523
17524    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17525        for _ in 0..1000 {
17526            if done(&lock(ring)) {
17527                return;
17528            }
17529            tokio::time::sleep(Duration::from_millis(1)).await;
17530        }
17531        panic!(
17532            "ring never reached the expected state: {:?}",
17533            lock(ring).snapshot(None, None)
17534        );
17535    }
17536
17537    #[tokio::test(start_paused = true)]
17538    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17539        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17540        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17541
17542        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17543        let before_release = lock(&ring).snapshot(None, None);
17544        assert!(
17545            matches!(before_release.capture, CaptureState::Incomplete { .. }),
17546            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17547        );
17548
17549        // The restart: the next process starts and writes before the old
17550        // reader catches up.
17551        let next = lock(&ring).begin_process();
17552        lock(&ring).push_line_from(next, "next process booting");
17553        release.send(()).unwrap();
17554        wait_until(&ring, |ring| {
17555            ring.snapshot(None, None).capture == CaptureState::Captured
17556        })
17557        .await;
17558
17559        assert_eq!(
17560            untimed(lock(&ring).snapshot(None, None).entries),
17561            vec![
17562                line("booting"),
17563                line("config error: missing storage"),
17564                TailEntry::ProcessStart,
17565                line("next process booting"),
17566            ],
17567            "the crash's last line must survive a slow reader and stay in the crashed process's section"
17568        );
17569    }
17570
17571    #[tokio::test(start_paused = true)]
17572    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17573    ) {
17574        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17575        // `_held` is never fired: a descendant keeps the pipe open for the
17576        // whole test.
17577        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17578
17579        let started = Instant::now();
17580        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17581        assert_eq!(
17582            started.elapsed(),
17583            BOUND,
17584            "the restart must wait exactly the bound for a pipe that stays open, no longer"
17585        );
17586
17587        let next = lock(&ring).begin_process();
17588        lock(&ring).push_line_from(next, "next process booting");
17589        tokio::time::sleep(Duration::from_secs(60)).await;
17590
17591        let snapshot = lock(&ring).snapshot(None, None);
17592        match &snapshot.capture {
17593            CaptureState::Incomplete { reason } => assert!(
17594                reason.contains("had not reached EOF") && reason.contains("250ms"),
17595                "the reason must say what is missing and after how long: {reason}"
17596            ),
17597            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17598        }
17599        assert_eq!(
17600            untimed(snapshot.entries),
17601            vec![
17602                line("parent exiting"),
17603                TailEntry::ProcessStart,
17604                line("next process booting"),
17605            ]
17606        );
17607    }
17608
17609    #[tokio::test(start_paused = true)]
17610    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17611        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17612        let (pump, release) = held_pump(&ring, "one\n", "two\n");
17613        release.send(()).unwrap();
17614
17615        settle_stderr_pump("clean", &ring, pump, BOUND).await;
17616
17617        let snapshot = lock(&ring).snapshot(None, None);
17618        assert_eq!(snapshot.capture, CaptureState::Captured);
17619        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17620    }
17621
17622    #[tokio::test(start_paused = true)]
17623    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17624        const BURST: usize = 150;
17625        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17626        let path = capture.join("two-pipe.stderr.log");
17627        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17628        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17629        let generation = lock(&ring).begin_process();
17630        let stdout = (0..BURST)
17631            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17632            .collect::<String>();
17633        let stderr = (0..BURST)
17634            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17635            .collect::<String>();
17636        // Model a child that has written both bursts and exited. Hold the stdout
17637        // reader after one line and a partial prefix, as if pipe readiness or
17638        // capture-file writes had delayed it. The final line also needs EOF to
17639        // flush: neither complete nor partial buffered output may be discarded.
17640        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17641        let (release, gate) = oneshot::channel();
17642        let reader = HeldReader {
17643            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17644            gate: Some(gate),
17645            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17646        };
17647        let (finished, completion) = oneshot::channel();
17648        let stdout_sink = sink.clone();
17649        let pump = tokio::spawn(async move {
17650            pump_stdout_to(reader, stdout_sink).await;
17651            finished.send(()).unwrap();
17652        });
17653        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17654
17655        let started = Instant::now();
17656        settle_stdout_pump("two-pipe", pump, BOUND).await;
17657        assert_eq!(
17658            started.elapsed(),
17659            BOUND,
17660            "a late reader must not hold up restart"
17661        );
17662        assert_eq!(
17663            std::fs::read_to_string(&path).unwrap().lines().count(),
17664            BURST + 1
17665        );
17666
17667        release
17668            .send(())
17669            .expect("stdout reader must survive the exit drain bound");
17670        tokio::time::timeout(Duration::from_secs(1), completion)
17671            .await
17672            .expect("released stdout reader must reach EOF")
17673            .expect("stdout capture must finish at EOF, not be cancelled");
17674        let contents = std::fs::read_to_string(&path).unwrap();
17675        let mut lines = contents
17676            .lines()
17677            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17678            .collect::<Vec<_>>();
17679        let mut expected = (0..BURST)
17680            .flat_map(|i| {
17681                [
17682                    format!("out-{i:04}-{}", "o".repeat(64)),
17683                    format!("err-{i:04}-{}", "e".repeat(64)),
17684                ]
17685            })
17686            .collect::<Vec<_>>();
17687        lines.sort_unstable();
17688        expected.sort_unstable();
17689        assert_eq!(
17690            lines, expected,
17691            "both pipes must retain every complete line exactly once"
17692        );
17693    }
17694}
17695
17696/// Containment of a module's process tree (issue #109).
17697///
17698/// The behaviour these defend against is a module helper surviving its module:
17699/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
17700/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
17701/// compounds it.
17702///
17703/// They run against the SUPERVISOR rather than the job-object crate because the
17704/// claim is about teardown: a crate-level test proves a job can reap a tree, not
17705/// that the daemon's drain path reaches it.
17706///
17707/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
17708/// lane there is a separate containment path with its own tests.
17709#[cfg(all(test, windows))]
17710mod job_containment_tests {
17711    use super::*;
17712    use cortexkit_test_support::ScratchDir;
17713    use std::{
17714        path::{Path, PathBuf},
17715        sync::{Arc, Mutex},
17716        time::{Duration, Instant},
17717    };
17718
17719    /// The stub, expected beside this test executable.
17720    ///
17721    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17722    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17723    /// failure then reads as a broken test rather than an unbuilt dependency.
17724    fn stub_path() -> PathBuf {
17725        let mut path = std::env::current_exe().expect("current_exe available in tests");
17726        path.pop();
17727        path.pop();
17728        path.push("fake-aft-stub.exe");
17729        assert!(
17730            path.exists(),
17731            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17732             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17733            path.display()
17734        );
17735        path
17736    }
17737
17738    /// Poll for the grandchild pid the stub records, and parse it.
17739    fn read_grandchild_pid(path: &Path) -> u32 {
17740        let deadline = Instant::now() + Duration::from_secs(10);
17741        loop {
17742            if let Ok(contents) = std::fs::read_to_string(path) {
17743                if let Ok(pid) = contents.trim().parse() {
17744                    return pid;
17745                }
17746            }
17747            assert!(
17748                Instant::now() < deadline,
17749                "the stub never recorded a grandchild pid at {}",
17750                path.display()
17751            );
17752            std::thread::sleep(Duration::from_millis(10));
17753        }
17754    }
17755
17756    /// Everything one fixture run needs, so the two tests below differ in exactly
17757    /// one place: whether the child is contained.
17758    struct Fixture {
17759        _dir: ScratchDir,
17760        module_id: String,
17761        grandchild: u32,
17762        child: Option<SupervisedChild>,
17763        registry: Arc<Registry>,
17764        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17765        terminal_ring: Arc<Mutex<TerminalRing>>,
17766        spawn_events: SpawnEventFeed,
17767    }
17768
17769    fn fixture(label: &str, module_id: &str) -> Fixture {
17770        let dir = ScratchDir::new(label);
17771        let pid_file = dir.join("grandchild.pid");
17772        let supervisor = Supervisor::new_for_test(
17773            Arc::new(Registry::default()),
17774            RestartPolicy::new(3, Duration::ZERO),
17775        );
17776        let runtime = supervisor.runtime_config();
17777        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17778        let spec = ModuleSpec {
17779            module_id: module_id.to_string(),
17780            program: stub_path(),
17781            // Zero args deliberately: a `--subc` argument would make the stub dial
17782            // a daemon that is not there, and the failure would land in the same
17783            // stderr ring this fixture exists to keep quiet.
17784            args: Vec::new(),
17785            env: vec![
17786                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17787                (
17788                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17789                    pid_file.display().to_string(),
17790                ),
17791            ],
17792            reserved: false,
17793            reserved_prefixes: Vec::new(),
17794            protocol: ModuleProtocol::Subc,
17795            overlap: Default::default(),
17796            carry_containment: false,
17797        };
17798        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17799            .expect("spawn the supervised fixture");
17800        let grandchild = read_grandchild_pid(&pid_file);
17801        Fixture {
17802            _dir: dir,
17803            module_id: module_id.to_string(),
17804            grandchild,
17805            child: Some(child),
17806            registry: Arc::new(Registry::default()),
17807            snapshot,
17808            terminal_ring: Arc::clone(&runtime.terminal_ring),
17809            spawn_events: SpawnEventFeed::default(),
17810        }
17811    }
17812
17813    impl Fixture {
17814        /// Drain through the supervisor's own teardown path.
17815        async fn drain(&mut self) {
17816            let child = self
17817                .child
17818                .take()
17819                .expect("the fixture child is still present");
17820            drain_child_to_state(
17821                &self.module_id,
17822                ModuleProtocol::Subc,
17823                // No forwarding table in this fixture, so nothing reaches the
17824                // child over a connection.
17825                StopNotice::NotSent,
17826                &self.registry,
17827                None,
17828                &self.snapshot,
17829                &self.terminal_ring,
17830                &self.spawn_events,
17831                child,
17832                Duration::from_millis(500),
17833                ModuleState::Stopped,
17834                Some(false),
17835                &FailureContext::operator(),
17836            )
17837            .await
17838            .expect("drain the supervised fixture");
17839        }
17840    }
17841
17842    /// Teardown reaps the grandchild, not merely the direct child.
17843    ///
17844    /// This is the assertion the change exists for. Before containment the
17845    /// grandchild survived: it is a separate process, and `start_kill` is
17846    /// `TerminateProcess` scoped to one pid.
17847    #[tokio::test]
17848    async fn teardown_reaps_the_grandchild() {
17849        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17850        let grandchild = fixture.grandchild;
17851
17852        assert!(
17853            subc_jobobject::process_exists(grandchild),
17854            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17855        );
17856
17857        fixture.drain().await;
17858
17859        assert!(
17860            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17861            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17862        );
17863    }
17864
17865    /// The mutation control: with containment withheld, the grandchild survives
17866    /// the same kill.
17867    ///
17868    /// This is the defect reproduction from #109 — a direct-child kill reaches
17869    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17870    /// supervisor because `spawn_and_mark_running` now always contains on
17871    /// Windows, which is the point: there is no longer a path that spawns
17872    /// uncontained, so the control has to construct one.
17873    ///
17874    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17875    /// grandchild ever dies here, that test is passing for a reason unrelated to
17876    /// the job object and the containment claim is unproven.
17877    #[test]
17878    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17879        let dir = ScratchDir::new("teardown-uncontained");
17880        let pid_file = dir.join("grandchild.pid");
17881        let mut child = std::process::Command::new(stub_path())
17882            .env("FAKE_AFT_NEVER_CONNECT", "1")
17883            .env(
17884                "FAKE_AFT_GRANDCHILD_PID_FILE",
17885                pid_file.display().to_string(),
17886            )
17887            .stdin(std::process::Stdio::null())
17888            .stdout(std::process::Stdio::null())
17889            .stderr(std::process::Stdio::null())
17890            .spawn()
17891            .expect("spawn the uncontained fixture");
17892        let grandchild = read_grandchild_pid(&pid_file);
17893
17894        // Exactly what the pre-fix teardown did: kill the direct child.
17895        child.kill().expect("kill the direct child");
17896        let _ = child.wait();
17897
17898        assert!(
17899            subc_jobobject::process_exists(grandchild),
17900            "grandchild {grandchild} died with the direct child, so this control no longer \
17901             distinguishes contained from uncontained teardown and the regression test is \
17902             passing vacuously"
17903        );
17904
17905        // The orphan this control demonstrates is the leak the fix prevents, so
17906        // the control must not leave one behind.
17907        kill_tree(grandchild);
17908    }
17909
17910    /// Crash durability: closing the containment handle reaps the tree with no
17911    /// teardown code running at all.
17912    ///
17913    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17914    /// call anything — and it is why containment is a kernel property of the
17915    /// handle rather than a step in the drain. Discovered by getting the
17916    /// mutation control wrong: clearing `job` to "disable" containment instead
17917    /// killed the tree, which is the guarantee, not a mistake.
17918    #[tokio::test]
17919    async fn dropping_containment_reaps_the_grandchild() {
17920        let mut fixture = fixture("drop-containment", "tree-drop");
17921        let grandchild = fixture.grandchild;
17922
17923        assert!(subc_jobobject::process_exists(grandchild));
17924
17925        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17926        fixture.child.as_mut().expect("child present").job = None;
17927
17928        assert!(
17929            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17930            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17931             crash would leave the tree behind"
17932        );
17933    }
17934
17935    /// Kill a pid and its tree, then confirm it is gone.
17936    fn kill_tree(pid: u32) {
17937        let _ = std::process::Command::new("taskkill.exe")
17938            .args(["/PID", &pid.to_string(), "/T", "/F"])
17939            .stdin(std::process::Stdio::null())
17940            .stdout(std::process::Stdio::null())
17941            .stderr(std::process::Stdio::null())
17942            .status();
17943        assert!(
17944            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17945            "could not clean up grandchild {pid}"
17946        );
17947    }
17948}
17949
17950#[cfg(test)]
17951mod privacy_trampoline_configuration_tests {
17952    #[tokio::test]
17953    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17954    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17955        #[cfg(target_os = "macos")]
17956        {
17957            let supervisor = super::Supervisor::new(
17958                std::sync::Arc::new(crate::Registry::default()),
17959                super::RestartPolicy::default(),
17960            );
17961            let error = supervisor.spawn(spec()).unwrap_err();
17962            assert!(
17963                error
17964                    .to_string()
17965                    .contains("no privacy trampoline configured"),
17966                "{error}"
17967            );
17968        }
17969    }
17970
17971    #[tokio::test]
17972    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17973    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17974        #[cfg(target_os = "macos")]
17975        {
17976            let supervisor = super::Supervisor::new(
17977                std::sync::Arc::new(crate::Registry::default()),
17978                super::RestartPolicy::default(),
17979            )
17980            .with_privacy_trampoline(std::env::current_exe().unwrap());
17981            let error = supervisor.spawn(spec()).unwrap_err();
17982            assert!(
17983                error
17984                    .to_string()
17985                    .contains("binary does not implement the privacy trampoline protocol"),
17986                "{error}"
17987            );
17988        }
17989    }
17990
17991    #[cfg(target_os = "macos")]
17992    fn spec() -> super::ModuleSpec {
17993        super::ModuleSpec {
17994            module_id: "privacy-configuration".into(),
17995            program: "/bin/sleep".into(),
17996            args: vec!["30".into()],
17997            env: vec![],
17998            reserved: false,
17999            reserved_prefixes: vec![],
18000            protocol: subc_control::ModuleProtocol::None,
18001            overlap: super::ModuleOverlap::Exclusive,
18002            carry_containment: false,
18003        }
18004    }
18005}
18006
18007#[cfg(test)]
18008mod privacy_exec_boundary_tests {
18009    #[cfg(target_os = "macos")]
18010    use super::*;
18011    #[cfg(target_os = "macos")]
18012    use std::{
18013        io::{Read, Write},
18014        net::{TcpListener, TcpStream},
18015    };
18016
18017    /// Unit-test-only pause at the actual early image read, not at a later
18018    /// status read. Production supervisors never inspect this environment key.
18019    #[cfg(target_os = "macos")]
18020    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18021        if let Some((_, path)) = spec
18022            .env
18023            .iter()
18024            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18025        {
18026            let mut barrier = TcpStream::connect(path).unwrap();
18027            barrier
18028                .set_read_timeout(Some(Duration::from_secs(30)))
18029                .unwrap();
18030            barrier.write_all(&pid.to_ne_bytes()).unwrap();
18031            let mut release = [0];
18032            barrier.read_exact(&mut release).unwrap();
18033            assert_eq!(&release, b"X");
18034        }
18035    }
18036
18037    #[cfg(target_os = "macos")]
18038    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18039        ModuleSpec {
18040            module_id: "privacy-boundary".into(),
18041            program: program.into(),
18042            args: args.iter().map(|arg| (*arg).into()).collect(),
18043            env: vec![],
18044            reserved: false,
18045            reserved_prefixes: vec![],
18046            protocol: ModuleProtocol::None,
18047            overlap: ModuleOverlap::Exclusive,
18048            carry_containment: false,
18049        }
18050    }
18051
18052    #[cfg(target_os = "macos")]
18053    async fn accept(listener: TcpListener) -> TcpStream {
18054        // Socket readiness, not elapsed time, establishes both pause points.
18055        let listener = tokio::net::TcpListener::from_std({
18056            listener.set_nonblocking(true).unwrap();
18057            listener
18058        })
18059        .unwrap();
18060        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18061            .await
18062            .unwrap()
18063            .unwrap();
18064        let stream = stream.into_std().unwrap();
18065        stream.set_nonblocking(false).unwrap();
18066        stream
18067            .set_read_timeout(Some(Duration::from_secs(30)))
18068            .unwrap();
18069        stream
18070    }
18071
18072    #[tokio::test]
18073    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18074    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18075        #[cfg(target_os = "macos")]
18076        {
18077            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18078            // Loopback sockets also work when the replay adapter's TMPDIR is
18079            // longer than Darwin's Unix-domain socket path limit.
18080            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18081            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18082            let record = root.join("live-children.json");
18083            let supervisor =
18084                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18085                    .with_live_children_record(&record);
18086            let runtime = supervisor.runtime_config();
18087            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18088            let mut spec = spec("/bin/sleep", &["30"]);
18089            spec.env = vec![
18090                (
18091                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18092                    exec_listener.local_addr().unwrap().to_string(),
18093                ),
18094                (
18095                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18096                    sample_listener.local_addr().unwrap().to_string(),
18097                ),
18098            ];
18099            let spawn = tokio::task::spawn_blocking(move || {
18100                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18101            });
18102            let mut sample = accept(sample_listener).await;
18103            let mut pid = [0; 4];
18104            sample.read_exact(&mut pid).unwrap();
18105            let pid = u32::from_ne_bytes(pid);
18106            let mut exec = accept(exec_listener).await;
18107            let mut ready = [0];
18108            exec.read_exact(&mut ready).unwrap();
18109            assert_eq!(&ready, b"R");
18110            // The early read is guaranteed to see a real, non-null trampoline
18111            // image: the fixture has reached its barrier and cannot exec yet.
18112            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18113            assert_eq!(
18114                observe_spawned_image(pid).unwrap().executable,
18115                Some(trampoline)
18116            );
18117            sample.write_all(b"X").unwrap();
18118            let mut child = spawn.await.unwrap();
18119            let early = crate::live_children::read_record(&record).unwrap();
18120            assert_eq!(early.len(), 1);
18121            assert_eq!(early[0].pid, pid);
18122            assert_eq!(
18123                early[0].executable, None,
18124                "unconfirmed trampoline image entered the roster"
18125            );
18126            assert!(child.report_ready.get().is_none());
18127            // The barrier's duration is unrelated to the production five-second
18128            // exec budget. Start the test's confirmation budget upon release.
18129            child.privacy_exec.as_mut().unwrap().deadline =
18130                tokio::time::Instant::now() + Duration::from_secs(30);
18131            exec.write_all(b"X").unwrap();
18132            child.confirm_privacy_exec().await;
18133            assert_eq!(child.spawn_failure, None);
18134            assert!(child.report_ready.get().is_some());
18135            let confirmed = crate::live_children::read_record(&record).unwrap();
18136            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18137            assert_ne!(module, trampoline);
18138            assert_eq!(confirmed[0].executable, Some(module.into()));
18139            child.start_kill().unwrap();
18140            child.wait().await.unwrap();
18141            child.release_roster();
18142        }
18143    }
18144
18145    #[tokio::test]
18146    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18147    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18148        #[cfg(target_os = "macos")]
18149        {
18150            let registry = Arc::new(Registry::default());
18151            let policy = RestartPolicy::new(0, Duration::ZERO);
18152            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
18153            let runtime = supervisor.runtime_config();
18154            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18155            let spec = spec("/bin/sh", &["-c", "exit 121"]);
18156            // Drive spawn and confirmation separately instead of starting the
18157            // monitor. WNOWAIT observes a real exit without consuming its status,
18158            // so confirmation's first try_wait must take the already-exited arm.
18159            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18160            let pid = child.pid;
18161            tokio::task::spawn_blocking(move || {
18162                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18163            })
18164            .await
18165            .unwrap()
18166            .unwrap();
18167            child.privacy_exec.as_mut().unwrap().deadline =
18168                tokio::time::Instant::now() + Duration::from_secs(30);
18169            let status = child.wait().await.unwrap();
18170            assert_eq!(status.code(), Some(121));
18171            assert!(child.privacy_exec.is_none());
18172            assert!(
18173                child.report_ready.get().is_none(),
18174                "an exited module must not publish a live pid"
18175            );
18176            let report = classify_reaped_child_exit(&snapshot, &child, &status);
18177            on_child_exit(
18178                &spec,
18179                policy,
18180                &registry,
18181                &snapshot,
18182                &runtime.terminal_ring,
18183                &runtime.spawn_events,
18184                &runtime.child_roster,
18185                report,
18186                true,
18187            )
18188            .await;
18189            let state = lock_snapshot(&snapshot).unwrap();
18190            assert_eq!(state.state, ModuleState::Failed);
18191            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18192            assert_eq!(state.reported_pid(), None);
18193            drop(state);
18194            let history = runtime.terminal_ring.lock().unwrap().snapshot();
18195            assert_eq!(history.entries.len(), 1);
18196            let terminal = &history.entries[0];
18197            assert_eq!(terminal.exit_code, Some(121));
18198            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18199            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18200            assert_eq!(
18201                terminal.disposition_detail.as_deref(),
18202                Some(policy.budget_exhausted_detail().as_str()),
18203                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18204            );
18205            assert_eq!(child.spawn_failure, None);
18206            child.release_roster();
18207        }
18208    }
18209}
18210
18211/// The daemon's real spawn path hands a subc-wire child its launch nonce on
18212/// descriptor 3, without an environment copy. The shell records the nonce
18213/// and its environment after exec so these tests observe the real handover.
18214#[cfg(all(test, unix))]
18215mod launch_nonce_descriptor_tests {
18216    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18217    use crate::stderr_tail::{StderrRing, StderrTailConfig};
18218    use cortexkit_test_support::ScratchDir;
18219    use std::{
18220        path::PathBuf,
18221        sync::{Arc, Mutex},
18222        time::{Duration, Instant},
18223    };
18224
18225    async fn probe(role: super::SpawnRole) {
18226        let scratch = ScratchDir::new("launch-nonce-descriptor");
18227        let fd_copy = scratch.join("from-descriptor");
18228        let env_copy = scratch.join("environment");
18229        let script = format!(
18230            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18231            fd = fd_copy.display(), env = env_copy.display(),
18232        );
18233        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18234        let spec = ModuleSpec {
18235            module_id: "nonce-descriptor-probe".to_string(),
18236            program: PathBuf::from("/bin/sh"),
18237            args: vec!["-c".to_string(), script],
18238            env: vec![
18239                xdg("XDG_DATA_HOME"),
18240                xdg("XDG_RUNTIME_DIR"),
18241                xdg("XDG_CONFIG_HOME"),
18242                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18243            ],
18244            reserved: true,
18245            reserved_prefixes: Vec::new(),
18246            protocol: ModuleProtocol::Subc,
18247            overlap: Default::default(),
18248            carry_containment: false,
18249        };
18250        let handle = SupervisorHandle::new();
18251        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18252        let roster = ChildRoster::default();
18253        #[cfg(target_os = "macos")]
18254        {
18255            let path = super::test_privacy_trampoline();
18256            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18257        }
18258        let child = super::spawn_child_in_slot(
18259            &spec,
18260            None,
18261            Some(&handle),
18262            &ring,
18263            None,
18264            &roster,
18265            #[cfg(target_os = "linux")]
18266            None,
18267            role,
18268            matches!(role, super::SpawnRole::SwapCandidate),
18269        )
18270        .expect("spawn probe");
18271        let deadline = Instant::now() + Duration::from_secs(10);
18272        while !(fd_copy.exists() && env_copy.exists()) {
18273            assert!(Instant::now() < deadline, "probe never wrote its copies");
18274            tokio::time::sleep(Duration::from_millis(20)).await;
18275        }
18276        let nonce = std::fs::read_to_string(fd_copy).unwrap();
18277        assert!(!nonce.is_empty());
18278        let environment = std::fs::read_to_string(env_copy).unwrap();
18279        assert!(environment
18280            .lines()
18281            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18282        let copy = environment
18283            .lines()
18284            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18285        assert_eq!(
18286            copy, None,
18287            "Unix children must never receive the environment nonce"
18288        );
18289        if matches!(role, super::SpawnRole::Plain) {
18290            assert_eq!(
18291                handle.spawn_nonce(&spec.module_id).as_deref(),
18292                Some(nonce.as_str())
18293            );
18294        }
18295        drop(child);
18296    }
18297
18298    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18299    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18300        probe(super::SpawnRole::Plain).await;
18301    }
18302
18303    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18304    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18305        probe(super::SpawnRole::SwapCandidate).await;
18306    }
18307}
18308
18309#[cfg(all(test, target_os = "linux"))]
18310#[path = "supervise/cgroup_reap_tests.rs"]
18311mod cgroup_reap_tests;
18312
18313#[cfg(all(test, target_os = "linux"))]
18314mod cgroup_containment_tests {
18315    use super::*;
18316    use cortexkit_test_support::ScratchDir;
18317
18318    #[tokio::test]
18319    async fn linux_subtree_memory_covers_module_children() {
18320        let dir = ScratchDir::new("subtree-memory");
18321        let root = PathBuf::from(format!(
18322            "/sys/fs/cgroup/subc-memory-test-{}-{}",
18323            std::process::id(),
18324            unix_ms_now()
18325        ));
18326        if let Err(error) = std::fs::create_dir(&root) {
18327            assert!(
18328                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18329                "required cgroup test cannot execute: {error}"
18330            );
18331            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18332            return;
18333        }
18334        let placement = subc_cgroup::prepare_at(&root)
18335            .unwrap()
18336            .expect("delegated cgroup");
18337        let supervisor = Supervisor::new_for_test(
18338            Arc::new(Registry::default()),
18339            RestartPolicy::new(3, Duration::ZERO),
18340        )
18341        .with_cgroup_placement(Some(placement));
18342        let ready = dir.join("ready");
18343        let module = supervisor
18344            .spawn(ModuleSpec {
18345                module_id: "memory-tree".into(),
18346                program: PathBuf::from("/bin/sh"),
18347                args: vec![
18348                    "-c".into(),
18349                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18350                    "fixture".into(),
18351                    ready.display().to_string(),
18352                ],
18353                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18354                    .into_iter()
18355                    .map(|key| (key.into(), dir.display().to_string()))
18356                    .collect(),
18357                reserved: false,
18358                reserved_prefixes: Vec::new(),
18359                protocol: ModuleProtocol::None,
18360                overlap: Default::default(),
18361                carry_containment: false,
18362            })
18363            .unwrap();
18364        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18365        while !ready.exists() {
18366            assert!(
18367                tokio::time::Instant::now() < deadline,
18368                "children never started"
18369            );
18370            tokio::time::sleep(Duration::from_millis(10)).await;
18371        }
18372        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18373        let reading = &readings[0];
18374        assert!(
18375            reading.process_count.is_some_and(|count| count >= 4),
18376            "{reading:?}"
18377        );
18378        assert!(
18379            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18380            "{reading:?}"
18381        );
18382        module.stop().await.unwrap();
18383        drop(module);
18384        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18385        std::fs::remove_dir(&root).unwrap();
18386        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18387    }
18388
18389    fn running(pid: u32) -> bool {
18390        // An orphan can remain a zombie until the container init reaps it.
18391        std::fs::read_to_string(format!("/proc/{pid}/stat"))
18392            .ok()
18393            .and_then(|stat| {
18394                stat.rsplit_once(") ")
18395                    .map(|(_, rest)| rest.starts_with('Z'))
18396            })
18397            .is_some_and(|zombie| !zombie)
18398    }
18399
18400    #[tokio::test]
18401    async fn linux_teardown_reaps_the_grandchild() {
18402        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18403    }
18404
18405    #[tokio::test]
18406    async fn linux_shutdown_straggler_reaps_the_grandchild() {
18407        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18408    }
18409
18410    async fn teardown_tree(test_name: &str, shutdown: bool) {
18411        let dir = ScratchDir::new(test_name);
18412        let root = PathBuf::from(format!(
18413            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18414            std::process::id(),
18415            unix_ms_now()
18416        ));
18417        if let Err(error) = std::fs::create_dir(&root) {
18418            assert!(
18419                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18420                "required cgroup test cannot execute: {error}"
18421            );
18422            eprintln!(
18423                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18424                root.display()
18425            );
18426            return;
18427        }
18428        let placement = subc_cgroup::prepare_at(&root)
18429            .expect("prepare isolated kernel cgroup")
18430            .expect("isolated cgroup is delegated");
18431        let module_id = "tree-teardown";
18432        let module = placement
18433            .module_path(module_id)
18434            .expect("create isolated module cgroup");
18435        if !module.join("cgroup.kill").exists() {
18436            std::fs::remove_dir(&module).unwrap();
18437            std::fs::remove_dir(root.join("subc-modules")).unwrap();
18438            std::fs::remove_dir(&root).unwrap();
18439            assert!(
18440                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18441                "required cgroup.kill interface unavailable"
18442            );
18443            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18444            return;
18445        }
18446        let supervisor = Supervisor::new_for_test(
18447            Arc::new(Registry::default()),
18448            RestartPolicy::new(3, Duration::ZERO),
18449        )
18450        .with_cgroup_placement(Some(placement));
18451        let mut runtime = supervisor.runtime_config();
18452        runtime.child_roster = runtime
18453            .child_roster
18454            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18455        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18456        let pid_file = dir.join("grandchild.pid");
18457        let spec = ModuleSpec {
18458            module_id: module_id.to_string(),
18459            program: PathBuf::from("/bin/sh"),
18460            args: vec![
18461                "-c".into(),
18462                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18463                "fixture".into(),
18464                pid_file.display().to_string(),
18465            ],
18466            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18467                .into_iter()
18468                .map(|key| (key.to_string(), dir.display().to_string()))
18469                .collect(),
18470            reserved: false,
18471            reserved_prefixes: Vec::new(),
18472            protocol: ModuleProtocol::None,
18473            overlap: Default::default(),
18474            carry_containment: false,
18475        };
18476        let child =
18477            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18478        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18479        let grandchild: u32 = loop {
18480            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18481                if let Ok(pid) = contents.trim().parse() {
18482                    break pid;
18483                }
18484            }
18485            assert!(
18486                tokio::time::Instant::now() < deadline,
18487                "grandchild pid was not recorded"
18488            );
18489            tokio::time::sleep(Duration::from_millis(10)).await;
18490        };
18491        assert!(
18492            running(grandchild),
18493            "grandchild must be alive before teardown"
18494        );
18495        if shutdown {
18496            let mut child = child;
18497            crate::child_roster::end_children_for_daemon_shutdown(
18498                &runtime.child_roster,
18499                false,
18500                std::future::pending(),
18501            )
18502            .await;
18503            child.wait().await.expect("reap shutdown straggler");
18504        } else {
18505            drain_child_to_state(
18506                module_id,
18507                ModuleProtocol::None,
18508                StopNotice::NotSent,
18509                &Registry::default(),
18510                None,
18511                &snapshot,
18512                &runtime.terminal_ring,
18513                &SpawnEventFeed::default(),
18514                child,
18515                Duration::from_millis(100),
18516                ModuleState::Stopped,
18517                Some(false),
18518                &FailureContext::operator(),
18519            )
18520            .await
18521            .expect("real supervisor teardown");
18522        }
18523        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18524        while running(grandchild) && tokio::time::Instant::now() < deadline {
18525            tokio::time::sleep(Duration::from_millis(10)).await;
18526        }
18527        let survived = running(grandchild);
18528        // Kill a surviving grandchild so a failed test does not leave it behind.
18529        if survived {
18530            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18531            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18532            tokio::time::sleep(Duration::from_millis(100)).await;
18533        }
18534        if module.exists() {
18535            std::fs::remove_dir(&module).expect("remove empty module cgroup");
18536        }
18537        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18538        std::fs::remove_dir(&root).unwrap();
18539        assert!(
18540            !survived,
18541            "grandchild {grandchild} outlived module teardown"
18542        );
18543        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18544    }
18545}
18546
18547/// A slow live-children record write on the restart path must not stop the
18548/// daemon from answering the replacement module's authentication handshake.
18549#[cfg(all(test, unix))]
18550mod record_write_stall_tests {
18551    use super::*;
18552    use std::sync::atomic::{AtomicBool, Ordering};
18553    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18554
18555    /// `subc-client-rs` gives the whole handshake this long before the module
18556    /// exits with `Timeout { stage: ServerProof }`.
18557    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18558    /// How long the replacement's admission write is held: longer than the
18559    /// module's deadline, standing in for the delay a busy disk can add
18560    /// across the two syncs in that write.
18561    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18562
18563    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
18564    /// On macOS every module is launched through the privacy trampoline, and
18565    /// a module that is not a `#!` script is admitted only if its running
18566    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
18567    /// the shell soon replaces itself with `/bin/sleep`, which does not match
18568    /// the configured `/bin/sh`. The image is read only after the spawn's
18569    /// live-children record write, and here that write lasted long enough for
18570    /// the shell to have exec'd every time, so each process was refused and
18571    /// killed right after spawn, and the supervisor crash-looped it with
18572    /// backoff until its restart budget ran out. The restart this test
18573    /// measures would then never produce a module at all.
18574    fn sleeping_spec() -> ModuleSpec {
18575        ModuleSpec {
18576            module_id: "record-write-stall".into(),
18577            program: "/bin/sleep".into(),
18578            args: vec!["600".into()],
18579            env: vec![],
18580            reserved: false,
18581            reserved_prefixes: vec![],
18582            protocol: ModuleProtocol::None,
18583            overlap: Default::default(),
18584            carry_containment: false,
18585        }
18586    }
18587
18588    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18589    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18590        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18591        let supervisor =
18592            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18593                .with_live_children_record(home.join("live-children.json"));
18594        let module = supervisor.spawn(sleeping_spec()).unwrap();
18595        // `reported_pid` names a process only once it is confirmed as the
18596        // module (on macOS, once the trampoline's image check has passed), so
18597        // a process the daemon is about to refuse and kill never counts as
18598        // the running module, here or in the final wait below.
18599        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18600            loop {
18601                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18602                    .unwrap()
18603                    .reported_pid()
18604                {
18605                    return pid;
18606                }
18607                tokio::time::sleep(Duration::from_millis(10)).await;
18608            }
18609        })
18610        .await
18611        .expect("the first process was admitted as the module");
18612
18613        // The daemon's listener, served by this runtime's workers like the
18614        // supervisor's own tasks.
18615        let key = vec![0x42; 32];
18616        let daemon_id = [0x24; 16];
18617        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18618        let port = listener.local_addr().unwrap().port();
18619        let server = tokio::spawn(crate::server::serve_listener(
18620            listener,
18621            Arc::new(crate::router::Router::with_default_self_handler()),
18622            crate::server::ServerAuth::with_limits(
18623                key.clone(),
18624                daemon_id,
18625                "test",
18626                crate::server::DEFAULT_AUTH_DEADLINE,
18627                4,
18628            ),
18629        ));
18630
18631        // Hold the write that admits the replacement, once, and say when it
18632        // has started.
18633        let (held_tx, held_rx) = std::sync::mpsc::channel();
18634        let armed = AtomicBool::new(true);
18635        supervisor
18636            .child_roster
18637            .set_record_write_hook_for_test(move |live| {
18638                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18639                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18640                    let _ = held_tx.send(());
18641                    std::thread::sleep(HELD_RECORD_WRITE);
18642                }
18643            });
18644
18645        // The replacement module, as a separate process would be: its own
18646        // thread and runtime, so only the daemon's runtime can be stalled.
18647        let module_process = std::thread::spawn(move || {
18648            held_rx
18649                .recv_timeout(Duration::from_secs(30))
18650                .expect("the restart reached the replacement's record write");
18651            let conn = ConnectionInfo {
18652                schema: SCHEMA_VERSION,
18653                wire_version: None,
18654                endpoints: vec![Endpoint {
18655                    host: "127.0.0.1".to_owned(),
18656                    port,
18657                }],
18658                key,
18659                daemon_id,
18660                pid: std::process::id(),
18661                daemon_ver: "test".to_owned(),
18662            };
18663            tokio::runtime::Builder::new_current_thread()
18664                .enable_all()
18665                .build()
18666                .unwrap()
18667                .block_on(async {
18668                    let started = Instant::now();
18669                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18670                        .await
18671                        .unwrap();
18672                    let result =
18673                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18674                    (result, started.elapsed())
18675                })
18676        });
18677
18678        module.restart(Some(0)).await.unwrap();
18679        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18680            .await
18681            .unwrap();
18682        assert!(
18683            auth.is_ok(),
18684            "the daemon must answer a module's handshake while a restart waits on the \
18685             live-children record write; got {auth:?} after {elapsed:?}"
18686        );
18687
18688        tokio::time::timeout(Duration::from_secs(20), async {
18689            while lock_snapshot(&module.inner.snapshot)
18690                .unwrap()
18691                .reported_pid()
18692                .is_none_or(|pid| pid == old_pid)
18693            {
18694                tokio::time::sleep(Duration::from_millis(10)).await;
18695            }
18696        })
18697        .await
18698        .expect("the restart published its replacement");
18699        module.stop().await.unwrap();
18700        server.abort();
18701    }
18702}
18703
18704#[cfg(all(test, unix))]
18705mod retained_status_tests {
18706    use super::*;
18707
18708    #[tokio::test]
18709    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18710        use crate::live_children::{ContainerIdentity, RetainedReason};
18711        for (reason, carried, expected) in [
18712            (
18713                RetainedReason::ContainerSurvivors(vec![4242]),
18714                true,
18715                Some("waiting: container survivors (pids [4242])".to_owned()),
18716            ),
18717            (
18718                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18719                true,
18720                Some(
18721                    "waiting: container survivors could not be verified: member inspection denied"
18722                        .to_owned(),
18723                ),
18724            ),
18725            (RetainedReason::Survived, false, None),
18726        ] {
18727            let supervisor =
18728                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18729            let spec = ModuleSpec {
18730                module_id: "retained-status-fixture".into(),
18731                program: PathBuf::from("/not-spawned"),
18732                args: Vec::new(),
18733                env: Vec::new(),
18734                reserved: false,
18735                reserved_prefixes: Vec::new(),
18736                protocol: ModuleProtocol::None,
18737                overlap: ModuleOverlap::Exclusive,
18738                carry_containment: carried,
18739            };
18740            let module = supervisor
18741                .supervise_configured(spec.clone(), false)
18742                .unwrap();
18743            // No process is launched. Invalid group zero is deliberate: if a
18744            // status read wrongly rechecks, verification changes the reason
18745            // and fails the assertions without signalling any real process.
18746            let guard = supervisor
18747                .child_roster
18748                .admit(
18749                    spec.module_id,
18750                    0,
18751                    ModuleProtocol::None,
18752                    None,
18753                    crate::child_roster::RecordedIdentity {
18754                        container: carried.then_some(ContainerIdentity {
18755                            pgid: Some(0),
18756                            created_start_time: Some(0),
18757                            cgroup_path: None,
18758                            ..ContainerIdentity::test_metadata()
18759                        }),
18760                        ..crate::child_roster::RecordedIdentity::default()
18761                    },
18762                )
18763                .unwrap();
18764            guard.retain(reason);
18765            let status = module.status().unwrap();
18766            assert_eq!(status.waiting_for_previous_detail, expected.clone());
18767            assert_eq!(status.health.detail, expected);
18768            assert!(!status.process_alive);
18769        }
18770    }
18771}