Skip to main content

subc_daemon/
supervise.rs

1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5    collections::{HashMap, HashSet, VecDeque},
6    error::Error,
7    fmt, io,
8    path::PathBuf,
9    process::{ExitStatus, Stdio},
10    sync::{Arc, Mutex, OnceLock},
11    time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22    manifest::{SelfSignalKind, SignalAnchor},
23    session::{
24        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25        MODULE_CONTROL_OP_HEALTH_CHECK,
26    },
27    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30    process::{Child, Command},
31    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32    task::JoinHandle,
33    time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38    child_roster::ChildRoster,
39    daemon_config::{
40        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41    },
42    forwarding::{
43        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44        ModuleDrainTarget, PendingModuleControlRpc,
45    },
46    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47    registry::{ConnectionId, RegistrationEndReason, RegistryError},
48    stderr_tail::{
49        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50        StderrTailSnapshot,
51    },
52    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53    Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75
76#[cfg(all(test, unix))]
77mod slow_retry_tests;
78
79/// Command-line flag used by supervised modules to find subc.
80///
81/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
82/// The path points at the TCP+key connection file; it is not an ambient signal and
83/// is never inherited by standalone children.
84pub const SUBC_ARG: &str = "--subc";
85
86const DEFAULT_MAX_RESTARTS: u32 = 3;
87const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
88const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
89/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
90/// to contain a real crash loop (which respawns in seconds) and short enough
91/// that unrelated crashes hours apart never accumulate into a permanent stop.
92const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
93/// How long a drain waits for already-dispatched requests to finalize before
94/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
95/// not RPC-scale: the original 2s value silently cut nearly every real tool
96/// call at the fence, making the wait-for-finalize design decorative for the
97/// workloads it existed for. Quiescence short-circuits, so an idle module
98/// restarts immediately regardless of this value; the budget is spent only
99/// when a genuine in-flight request is worth finishing. Per-module override:
100/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
101/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
102/// where a stuck request will never settle).
103pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
104pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
105const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
106const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
107/// How long a drain waits after one busy-gauge probe completes before it asks
108/// again.
109///
110/// A drain re-probes on this short interval of its own rather than on the
111/// module's health cadence. The default cadence (30 s) equals the default
112/// drain ceiling, so a cadence-paced drain that found the module busy at its
113/// first probe never asked a second time: it waited out the whole ceiling and
114/// forced GOODBYE even after the gauges had read zero. The interval is counted
115/// from the moment the previous probe finished, and probes run one at a time,
116/// so a slow module is never asked more often than it can answer. Periodic
117/// health checks outside a drain keep using the configured cadence.
118const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
119/// How long a restart waits for the exited process's output readers.
120///
121/// The restart does not depend on either output reader finishing. A stderr
122/// reader still running at this bound is left running, and its later output goes
123/// into the exited process's own section of the stderr ring (see
124/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
125/// bound no longer decides whether a crash's last lines are kept: under load
126/// the reader may simply not have been scheduled yet, and cutting it there
127/// discarded exactly the lines that explained the crash.
128///
129/// What the bound still decides is when the tail starts reporting
130/// `Incomplete`: a pipe open past it usually means a descendant of the exited
131/// process still holds it, and the tail cannot claim to be whole until that
132/// pipe closes. That is also the only case in which the wait costs the restart
133/// anything, because a pipe with no other holder reaches EOF when the process
134/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
135/// `Captured` at EOF with nothing lost.
136///
137/// The stdout reader carries no ring, only the capture file. It also continues
138/// to EOF after this bound: child exit closes the writer but does not mean the
139/// reader has consumed all buffered bytes. Late output from either pipe can
140/// therefore reach the shared capture file after the successor starts.
141const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
142/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
143pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
144const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
145/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
146/// dropped because the subscriber stopped draining its frames. The detail's
147/// `first_undelivered_cursor` names the first event it did not receive; the
148/// client resubscribes from the last cursor it did receive.
149pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
150
151#[cfg(target_os = "macos")]
152struct PrivacyExec {
153    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
154    deadline: tokio::time::Instant,
155    expected: Option<subc_os::FileIdentity>,
156    trampoline: Option<subc_os::FileIdentity>,
157    script: bool,
158    module_id: String,
159    #[cfg(test)]
160    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
161}
162
163#[cfg(all(test, target_os = "macos"))]
164pub(crate) fn test_privacy_trampoline() -> PathBuf {
165    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
166    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
167    let path = std::env::current_exe()
168        .unwrap()
169        .parent()
170        .unwrap()
171        .parent()
172        .unwrap()
173        .join("privacy-trampoline-fixture");
174    // Without the fixture every macOS spawn is refused, and the tests that
175    // spawn fail later as a module in state Failed, which names the wrong
176    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
177    assert!(
178        path.exists(),
179        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
180         --bins --features test-support` or `cargo test -p subc-daemon` first",
181        path.display()
182    );
183    path
184}
185
186#[cfg(target_os = "macos")]
187fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
188    use std::io::Read;
189    let mut probe = std::process::Command::new(path)
190        .args(["__disclaim-exec", "--probe"])
191        .stdin(Stdio::null())
192        .stdout(Stdio::piped())
193        .stderr(Stdio::piped())
194        .spawn()
195        .map_err(|error| {
196            format!(
197                "privacy trampoline probe failed for {}: {error}",
198                path.display()
199            )
200        })?;
201    let deadline = std::time::Instant::now() + Duration::from_secs(5);
202    let status = loop {
203        match probe.try_wait() {
204            Ok(Some(status)) => break status,
205            Ok(None) if std::time::Instant::now() < deadline => {
206                std::thread::sleep(Duration::from_millis(5))
207            }
208            result => {
209                let _ = probe.kill();
210                let _ = probe.wait();
211                return Err(format!(
212                    "privacy trampoline probe failed or timed out for {}: {result:?}",
213                    path.display()
214                ));
215            }
216        }
217    };
218    let mut answer = String::new();
219    if let Some(stdout) = probe.stdout.take() {
220        let _ = stdout.take(256).read_to_string(&mut answer);
221    }
222    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
223        return Ok(());
224    }
225    let mut diagnostic = String::new();
226    if let Some(stderr) = probe.stderr.take() {
227        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
228    }
229    let cause = diagnostic
230        .trim()
231        .strip_prefix("ck-subc: own privacy identity refused: ")
232        .unwrap_or("binary does not implement the privacy trampoline protocol");
233    Err(format!(
234        "{cause}: probe of {} exited {status}",
235        path.display()
236    ))
237}
238
239#[cfg(target_os = "macos")]
240fn privacy_command(
241    spec: &ModuleSpec,
242    roster: &ChildRoster,
243) -> Result<
244    (
245        Command,
246        Option<PrivacyExec>,
247        subc_os::privacy_identity::ExecAcknowledgement,
248    ),
249    SuperviseError,
250> {
251    let failure = |cause: String| {
252        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
253        SuperviseError::Spawn {
254            program: spec.program.clone(),
255            source: io::Error::other(cause),
256            cgroup_path: None,
257        }
258    };
259    let trampoline = roster.privacy_trampoline().map_err(failure)?;
260    // Resolve PATH with the same environment the Command will receive. For
261    // scripts retain the existing orphan-identity rule: the kernel chooses
262    // the interpreter, and its observed image is the one recorded. Do not
263    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
264    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
265        let path = spec
266            .env
267            .iter()
268            .find(|(key, _)| key == "PATH")
269            .map(|(_, value)| std::ffi::OsString::from(value))
270            .or_else(|| std::env::var_os("PATH"))
271            .unwrap_or_else(|| "/usr/bin:/bin".into());
272        std::env::split_paths(&path)
273            .map(|dir| dir.join(&spec.program))
274            .find(|path| path.is_file())
275            .unwrap_or_else(|| spec.program.clone())
276    } else {
277        spec.program.clone()
278    };
279    let expected = subc_os::file_identity(&program);
280    let trampoline_image = subc_os::file_identity(&trampoline);
281    let script = {
282        use std::io::Read;
283        let mut prefix = [0u8; 2];
284        std::fs::File::open(&program)
285            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
286    };
287    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
288        return Err(failure(
289            "privacy identity module executable is missing or is the trampoline itself".to_string(),
290        ));
291    }
292    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
293        .map_err(|error| failure(error.to_string()))?;
294    let reader =
295        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
296    let mut command = Command::new(&trampoline);
297    command
298        .arg("__disclaim-exec")
299        .arg(ack.fd().to_string())
300        .arg(&program);
301    Ok((
302        command,
303        Some(PrivacyExec {
304            reader,
305            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
306            expected,
307            trampoline: trampoline_image,
308            script,
309            module_id: spec.module_id.clone(),
310            #[cfg(test)]
311            confirmation_pause: spec
312                .env
313                .iter()
314                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
315                .map(|(_, address)| (address.clone(), None)),
316        }),
317        ack,
318    ))
319}
320
321struct SupervisedChild {
322    subtree_target: crate::subtree_resources::Target,
323    child: containment::ProcessChild,
324    exec: crate::activity::ExecOwner,
325    #[cfg(windows)]
326    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
327    #[cfg(target_os = "macos")]
328    privacy_exec: Option<PrivacyExec>,
329    /// Set once this launch's exec acknowledgement confirms the module image.
330    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
331    /// `subc_os::privacy_identity`), which then replaces itself with the
332    /// module. The supervisor owns and can kill that pid from spawn, but
333    /// status readers report it only after this latch is set, so nothing
334    /// reports the trampoline's image as the module's.
335    #[cfg(target_os = "macos")]
336    report_ready: Arc<OnceLock<()>>,
337    /// Supervisor failure detail retained for the terminal record, including
338    /// image refusals and first-HELLO timeouts after a successful spawn.
339    spawn_failure: Option<String>,
340    /// The protocol this process was launched with. A reload can store a new
341    /// launch spec with a different protocol, but that takes effect only at the
342    /// next spawn, so this process keeps being handled by the protocol it
343    /// actually speaks.
344    protocol: ModuleProtocol,
345    /// This container's cgroup name: a bounded module/slot label followed by a
346    /// unique suffix (when cgroup placement is on). Carried replacements reuse
347    /// the name; ordinary launches each get a fresh container. A
348    /// retired process in a slot may still be draining when a later one is
349    /// spawned into that slot, so the suffix keeps the later process out of
350    /// the retired one's cgroup, which is the domain a kill applies to.
351    #[cfg(target_os = "linux")]
352    module_id: String,
353    #[cfg(target_os = "linux")]
354    cgroup_placement: Option<subc_cgroup::Placement>,
355    /// The job for an ordinary, non-carried launch. For carried
356    /// launches the actor owns it and the child holds only a weak container link.
357    ///
358    /// Dropping this handle is what reaps a surviving tree when no supervisor
359    /// code runs — a daemon crash — because the job carries
360    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
361    ///
362    /// That limit is not crash-only, and the difference is worth knowing: a
363    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
364    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
365    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
366    /// module at once. Before this change they survived that, saw EOF on the
367    /// control socket, and ran their own teardown; Unix keeps that path
368    /// deliberately, so a module can seal a WAL or close a capture rather than
369    /// be killed mid-write. So this trades graceful teardown on every Windows
370    /// daemon stop for containment on a crash, which is the right way round
371    /// today: orphaned GPU workers are a reported, recurring problem, and the
372    /// modules that write most heavily do not run on Windows.
373    ///
374    /// The fix is a real Windows stop path — the daemon draining before it
375    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
376    /// reaches only what the drain left behind, which is what it should reach.
377    #[cfg(windows)]
378    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
379    stdout_pump: Option<JoinHandle<()>>,
380    stderr_pump: Option<StderrPump>,
381    stderr_ring: Arc<Mutex<StderrRing>>,
382    /// Conservative monotonic bound sampled before `command.spawn()`, not when
383    /// identity recording finishes or the spawn generation is published.
384    spawned_at: Instant,
385    /// Ordinary spawns must send their first HELLO within this budget. Once
386    /// accepted, a later disconnect must not restart the registration clock.
387    registration_deadline: Option<Instant>,
388    registration_watch: Option<RegistrationWatch>,
389    spawned_at_ms: u64,
390    spawned_from: PathBuf,
391    launch_spec_digest: LaunchSpecDigest,
392    spawned_file_identity: Option<SpawnedFileIdentity>,
393    process_start_time: Option<u64>,
394    process_identity: Option<ProcessIdentity>,
395    pid: u32,
396    /// This process's entry in the daemon's child roster, released when the
397    /// process is reaped or this handle is dropped.
398    roster_guard: Option<crate::child_roster::RosterGuard>,
399}
400
401impl SupervisedChild {
402    fn registered_since_spawn(&self) -> bool {
403        self.registration_watch
404            .as_ref()
405            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
406    }
407
408    fn id(&self) -> Option<u32> {
409        Some(self.pid)
410    }
411
412    fn process_identity(&self) -> Option<ProcessIdentity> {
413        self.process_identity
414    }
415
416    async fn wait(&mut self) -> io::Result<ExitStatus> {
417        #[cfg(target_os = "macos")]
418        self.confirm_privacy_exec().await;
419        // The roster entry is NOT released here. A daemon shutdown waits for the
420        // roster to empty and then exits the process, so releasing at the reap
421        // let it exit before the exit handler wrote this child's terminal record
422        // (the stderr drain and snapshot update sit in between), and the
423        // shutdown's own `daemon_shutdown` record was intermittently lost. The
424        // caller releases it after recording the exit (`release_roster`), and
425        // dropping the handle releases it too.
426        let result = self.child.wait().await;
427        #[cfg(target_os = "linux")]
428        if result.is_ok() && self.child.container().is_none() {
429            if let Some(placement) = self.cgroup_placement.as_ref() {
430                // Ordinary children still own a per-spawn cgroup: returning
431                // from wait must leave it empty and removed, as before carry.
432                // Keep cleanup ownership across cancellation until it finishes.
433                cleanup_reaped_cgroup(placement, &self.module_id).await;
434                self.cgroup_placement = None;
435            }
436        }
437        result
438    }
439
440    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
441        if self.child.container().is_some() {
442            if !carry {
443                self.child.terminate_container().await;
444            }
445        } else {
446            #[cfg(target_os = "linux")]
447            if let Some(placement) = self.cgroup_placement.as_ref() {
448                cleanup_reaped_cgroup(placement, &self.module_id).await;
449                self.cgroup_placement = None;
450            }
451        }
452        self.child.reap().await?;
453        self.release_roster();
454        Ok(())
455    }
456
457    #[cfg(target_os = "macos")]
458    async fn confirm_privacy_exec(&mut self) {
459        let Some(pending) = &mut self.privacy_exec else {
460            return;
461        };
462        let result = tokio::time::timeout_at(pending.deadline, async {
463            let mut record = Vec::new();
464            loop {
465                let mut ready = pending.reader.readable().await?;
466                let read = ready.try_io(|reader| {
467                    use std::io::Read;
468                    let mut reader = reader.get_ref();
469                    let mut buffer = [0u8; 256];
470                    reader.read(&mut buffer).map(|count| (count, buffer))
471                });
472                match read {
473                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
474                    Ok(Ok((count, buffer))) => {
475                        if record.len() + count > 1024 {
476                            return Err(io::Error::other(
477                                "privacy exec refusal record is too long",
478                            ));
479                        }
480                        record.extend_from_slice(&buffer[..count]);
481                    }
482                    Ok(Err(error)) => return Err(error),
483                    Err(_) => continue,
484                }
485            }
486        })
487        .await;
488        #[cfg(test)]
489        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
490            use tokio::io::{AsyncReadExt, AsyncWriteExt};
491            if stream.is_none() {
492                *stream = Some(
493                    tokio::net::TcpStream::connect(address.as_str())
494                        .await
495                        .unwrap(),
496                );
497                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
498                stream
499                    .as_mut()
500                    .unwrap()
501                    .write_all(&self.pid.to_ne_bytes())
502                    .await
503                    .unwrap();
504            }
505            let mut reply = [0];
506            stream
507                .as_mut()
508                .unwrap()
509                .read_exact(&mut reply)
510                .await
511                .unwrap();
512            pending.confirmation_pause = None;
513            if &reply == b"F" {
514                Ok(Err(io::Error::other("injected late exec refusal")))
515            } else {
516                assert_eq!(&reply, b"X");
517                result
518            }
519        } else {
520            result
521        };
522        // Keep the reader in self across await: select cancellation must not
523        // discard the handshake or reset its original five-second deadline.
524        let pending = self.privacy_exec.as_ref().expect("pending exec");
525        let cause = match result {
526            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
527            Ok(Err(error)) => Some(format!(
528                "privacy identity exec acknowledgement failed: {error}"
529            )),
530            Ok(Ok(record)) if !record.is_empty() => Some(
531                std::str::from_utf8(&record)
532                    .ok()
533                    .and_then(|record| {
534                        record
535                            .trim()
536                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
537                    })
538                    .filter(|cause| !cause.is_empty())
539                    .unwrap_or("invalid privacy exec refusal record")
540                    .to_string(),
541            ),
542            Ok(Ok(_)) => match self.child.try_wait() {
543                // Empty EOF is the exec acknowledgement. A real module may exit
544                // immediately, including with a reserved trampoline status; no
545                // image is admitted, and its ordinary exit contract stays intact.
546                Ok(Some(_status)) => None,
547                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
548                Ok(None) => {
549                    let image = observe_spawned_image(self.pid);
550                    if let Some(image) = image.filter(|image| {
551                        image.executable.is_some()
552                            && image.executable != pending.trampoline
553                            && (image.executable == pending.expected || pending.script)
554                    }) {
555                        if let Err(error) = self
556                            .roster_guard
557                            .as_ref()
558                            .map(|guard| guard.confirm_executable(image))
559                            .transpose()
560                        {
561                            Some(format!(
562                                "could not durably record confirmed module image: {error}"
563                            ))
564                        } else {
565                            let _ = self.report_ready.set(());
566                            info!(module_id = %pending.module_id, pid = self.pid,
567                                "module spawned with own privacy identity (responsibility disclaimed)");
568                            None
569                        }
570                    } else if image.is_none()
571                        || image.is_some_and(|image| image.executable.is_none())
572                    {
573                        // A process can exit between try_wait and the kernel
574                        // image read. Empty EOF already acknowledged exec, so
575                        // preserve that module's ordinary exit rather than
576                        // mislabel a disappearing image as trampoline refusal.
577                        // Keep pending in self across await so cancellation does
578                        // not discard validation or reset its original deadline.
579                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
580                            Ok(Ok(_status)) => None,
581                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
582                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
583                        }
584                    } else {
585                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
586                    }
587                }
588            },
589        };
590        let pending = self.privacy_exec.take().expect("pending exec");
591        // Exec confirmation describes launch success, not roster retirement.
592        // Keep even a reaped child recorded until its terminal history exists.
593        self.exec.0.finish(if cause.is_some() {
594            crate::activity::ExecState::Refused
595        } else {
596            crate::activity::ExecState::CommittedExec
597        });
598        if let Some(cause) = cause {
599            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
600            self.spawn_failure = Some(cause);
601            // No image is admitted on failure. Reach the entire fresh process
602            // group, including a module which spawned a helper before refusal.
603            if let Some(container) = self.child.container() {
604                container.hard_kill();
605            } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
606                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
607            }
608            let _ = self.child.start_kill();
609        }
610    }
611
612    /// Releases the leader's roster entry after recording and finalizing its
613    /// exit. A carried container retains its separate roster entry and, for a
614    /// Unix process group, its unreaped first leader that reserves the group ID.
615    fn release_roster(&mut self) {
616        self.roster_guard = None;
617    }
618
619    /// Kill the child and, where containment is available, its process tree.
620    ///
621    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
622    /// helper process leaked the helper — the Synapse embedding module's CUDA
623    /// worker holds the GPU allocation, so the leak cost VRAM until the next
624    /// restart of something else. Terminating the job reaches grandchildren that
625    /// a tree walk cannot, including one whose parent has already exited and
626    /// been reparented away.
627    ///
628    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
629    /// direct-child kill still decides the outcome, so containment can never
630    /// change whether a module is reported as stopped.
631    fn start_kill(&mut self) -> io::Result<()> {
632        if let Some(container) = self.child.container() {
633            container.hard_kill();
634            return self.kill_leader();
635        }
636        #[cfg(windows)]
637        if let Some(job) = &self.job {
638            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
639                debug!(
640                    error = %error,
641                    "job termination failed; the direct-child kill still owns the outcome"
642                );
643            }
644        }
645        #[cfg(target_os = "linux")]
646        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
647        self.child.start_kill()
648    }
649
650    fn kill_leader(&mut self) -> io::Result<()> {
651        self.child.start_kill()
652    }
653
654    async fn drain_stderr(&mut self, module_id: &str) {
655        if let Some(pump) = self.stdout_pump.take() {
656            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
657        }
658
659        let Some(pump) = self.stderr_pump.take() else {
660            return;
661        };
662        settle_stderr_pump(
663            module_id,
664            &self.stderr_ring,
665            pump,
666            STDERR_PUMP_DRAIN_TIMEOUT,
667        )
668        .await;
669    }
670}
671
672/// Bound the restart's wait, not the lifetime of the exited child's stdout
673/// reader. Slow pipe reads or capture writes must not turn the bound into lost
674/// output; dropping the handle lets the reader finish at EOF, like stderr.
675async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
676    match timeout(bound, &mut pump).await {
677        Ok(Ok(())) => {}
678        Ok(Err(error)) => {
679            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
680        }
681        Err(_) => {
682            drop(pump);
683            warn!(
684                module_id,
685                waited = ?bound,
686                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
687            );
688        }
689    }
690}
691
692/// The reader task for one process's stderr, with the ring generation its
693/// lines are attributed to.
694struct StderrPump {
695    task: JoinHandle<()>,
696    generation: u64,
697}
698
699/// Retire an exited process's stderr reader and wait up to `bound` for it to
700/// reach EOF. A reader still running at the bound is detached, not stopped: it
701/// keeps filling the exited process's section of the ring until its pipe
702/// closes, and the tail reads `Incomplete` until then. See
703/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
704async fn settle_stderr_pump(
705    module_id: &str,
706    ring: &Arc<Mutex<StderrRing>>,
707    pump: StderrPump,
708    bound: Duration,
709) {
710    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
711    let StderrPump {
712        mut task,
713        generation,
714    } = pump;
715    lock().retire_pump(generation);
716    match timeout(bound, &mut task).await {
717        Ok(Ok(())) => {}
718        Ok(Err(err)) => {
719            let mut ring = lock();
720            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
721            ring.finish_pump(generation);
722            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
723        }
724        Err(_) => {
725            // Dropping the handle detaches the task; it ends at EOF on its pipe.
726            drop(task);
727            lock().mark_pump_late(
728                generation,
729                format!(
730                    "stderr of the exited process had not reached EOF {bound:?} after it was \
731                     retired (a descendant may still hold the pipe open); lines it still \
732                     writes are kept in that process's section"
733                ),
734            );
735            warn!(
736                module_id,
737                waited = ?bound,
738                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
739            );
740        }
741    }
742}
743
744fn registration_release_events() -> &'static watch::Sender<u64> {
745    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
746    EVENTS.get_or_init(|| {
747        let (sender, _receiver) = watch::channel(0);
748        sender
749    })
750}
751
752pub(crate) fn notify_registration_release() {
753    let events = registration_release_events();
754    let next_generation = (*events.borrow()).wrapping_add(1);
755    events.send_replace(next_generation);
756}
757
758/// How to launch one singleton module process.
759#[derive(Debug, Clone, PartialEq, Eq)]
760pub struct ModuleSpec {
761    pub module_id: String,
762    pub program: PathBuf,
763    pub args: Vec<String>,
764    pub env: Vec<(String, String)>,
765    /// When true this is a reserved module: each spawn gets a fresh one-time launch
766    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
767    /// process can register this module_id (a security-boundary module like the
768    /// credential vault must not be impersonable while it is down/restarting).
769    pub reserved: bool,
770    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
771    /// Prefixes come from daemon config and must end in `:` before they reach the
772    /// supervisor; the owner module's current spawn nonce authorizes claims under
773    /// each prefix.
774    pub reserved_prefixes: Vec<String>,
775    /// The wire protocol this module speaks, as DECLARED in daemon config.
776    ///
777    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
778    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
779    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
780    /// and NO launch nonce, and a clean exit the daemon did not request is
781    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
782    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
783    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
784    /// because a process ignores an environment variable it does not read.
785    ///
786    /// The argument is the part that cannot be "harmless to a process that
787    /// ignores it": a stock binary exits on an unknown flag before it listens
788    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
789    /// first conformance run against this mode found it. The nonce is withheld
790    /// because a process that will never present it gains nothing from holding
791    /// it, and a secret in the environment of a process that does not need it is
792    /// a leak surface for no benefit.
793    pub protocol: ModuleProtocol,
794    /// Whether two processes of this module may run at once, which is what a
795    /// blue/green swap does for the length of its overlap. Declared in daemon
796    /// config because the daemon must be able to answer it while the module is
797    /// down, and so a module cannot talk itself into it after registering.
798    pub overlap: ModuleOverlap,
799    /// Keep module-started helpers in the same container across replacements.
800    /// This is a daemon-config policy, fixed for the lifetime of a container;
801    /// stopping the module or losing its supervisor still destroys the container.
802    pub carry_containment: bool,
803}
804
805/// Only the configured launch inputs enter this value; spawned connection paths,
806/// nonces and inherited environment never do. Its bytes remain daemon-private.
807#[derive(Clone, PartialEq, Eq)]
808pub(crate) struct LaunchSpecDigest([u8; 32]);
809
810impl fmt::Debug for LaunchSpecDigest {
811    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
812        f.write_str("LaunchSpecDigest([redacted])")
813    }
814}
815
816impl LaunchSpecDigest {
817    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
818        fn field(hash: &mut Sha256, bytes: &[u8]) {
819            hash.update((bytes.len() as u64).to_le_bytes());
820            hash.update(bytes);
821        }
822        let mut hash = Sha256::new();
823        if let Some(program) = spec.program.to_str() {
824            field(&mut hash, b"utf8-program");
825            field(&mut hash, program.as_bytes());
826        } else {
827            // The daemon config is UTF-8, but in-process callers may provide
828            // native paths: lossy conversion would collapse distinct programs.
829            #[cfg(unix)]
830            {
831                use std::os::unix::ffi::OsStrExt;
832                field(&mut hash, b"unix-program");
833                field(&mut hash, spec.program.as_os_str().as_bytes());
834            }
835            #[cfg(windows)]
836            {
837                use std::os::windows::ffi::OsStrExt;
838                field(&mut hash, b"windows-program");
839                let bytes = spec
840                    .program
841                    .as_os_str()
842                    .encode_wide()
843                    .flat_map(u16::to_le_bytes)
844                    .collect::<Vec<_>>();
845                field(&mut hash, &bytes);
846            }
847            #[cfg(not(any(unix, windows)))]
848            field(&mut hash, spec.program.to_string_lossy().as_bytes());
849        }
850        hash.update((spec.args.len() as u64).to_le_bytes());
851        for arg in &spec.args {
852            field(&mut hash, arg.as_bytes());
853        }
854        let mut env = spec.env.iter().collect::<Vec<_>>();
855        env.sort_unstable();
856        hash.update((env.len() as u64).to_le_bytes());
857        for (key, value) in env {
858            field(&mut hash, key.as_bytes());
859            field(&mut hash, value.as_bytes());
860        }
861        field(
862            &mut hash,
863            match spec.protocol {
864                ModuleProtocol::Subc => b"subc",
865                ModuleProtocol::None => b"none",
866            },
867        );
868        Self(hash.finalize().into())
869    }
870}
871
872/// Whether a module tolerates a second process of itself running alongside.
873///
874/// Most modules are single-writer on their store (a WAL, a capture log, a
875/// resident index behind a writer barrier), and two processes on one store
876/// corrupt it. So a swap, which overlaps the old and new process by design,
877/// is refused unless the module's config opts in with `overlap: "safe"`.
878#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
879pub enum ModuleOverlap {
880    /// Never run two processes of this module at once. The default.
881    #[default]
882    Exclusive,
883    /// The module has said a second process of itself is harmless for the
884    /// length of a swap.
885    ///
886    /// Declare it only if a second instance can run for a few seconds without
887    /// touching ANY single-writer store: every database, WAL, index, projector
888    /// and scheduled job the module owns. A lease on part of that state is not
889    /// enough. broca's session lease guards WAL appends while its run index, its
890    /// store projector and its archive fold timer (which unlinks live WAL files)
891    /// stay single-writer, so broca is exclusive despite holding a lease. The
892    /// refusal only fires after this has been decided, so the decision is the
893    /// check.
894    Safe,
895}
896
897impl ModuleOverlap {
898    pub fn as_str(self) -> &'static str {
899        match self {
900            Self::Exclusive => "exclusive",
901            Self::Safe => "safe",
902        }
903    }
904}
905
906/// Environment variable telling a spawned module which case it was started
907/// for, before it sends HELLO. Only a swap candidate carries it, as
908/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
909///
910/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
911/// longer because nobody waits on it, while a plain restart must flip ready
912/// quickly because callers see `module_warming` until it does. Absence means
913/// plain restart, the safe reading. The daemon trusts nothing about it; the
914/// candidate is proven by its launch nonce at HELLO.
915pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
916/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
917pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
918/// How long a swap waits for its candidate to register and declare itself
919/// ready when the operator does not say. A module warming as a swap candidate
920/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
921/// daemon allows that plus time to start the process and send HELLO.
922pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
923
924/// Bounded restart policy for crash exits.
925///
926/// `max_restarts` is the number of replacement processes allowed after the
927/// initial spawn WITHIN `window`. After that many crash restarts inside one
928/// window the module enters [`ModuleState::Failed`] and the supervisor stops
929/// the crash loop.
930///
931/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
932/// and that only survived because crashes were rare: a module that crashed
933/// three times across a week was disabled forever by crashes that had nothing
934/// to do with each other. That stopped being survivable once modules began
935/// exiting non-zero whenever the daemon's connection to them drops, because
936/// then every daemon-side connection drop spends a unit of the same budget and
937/// one flappy hour permanently stops a healthy module. Restarts older than
938/// `window` release their slot, so a module that crashed twice yesterday has a
939/// full budget today, while a genuine crash loop -- which is fast by
940/// definition -- still reaches the cap and stops.
941#[derive(Debug, Clone, Copy, PartialEq, Eq)]
942pub struct RestartPolicy {
943    pub max_restarts: u32,
944    /// Base delay before a crash replacement. The actual delay escalates with
945    /// the number of recent crash replacements and is capped by `max_backoff`.
946    pub backoff: Duration,
947    /// Maximum delay before a crash replacement.
948    pub max_backoff: Duration,
949    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
950    /// budget effectively infinite (nothing is ever in-window), which is why
951    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
952    pub window: Duration,
953}
954
955impl RestartPolicy {
956    /// A policy with the default crash window. Callers that care about the
957    /// window say so with [`Self::with_window`]; the ones that do not are
958    /// asking for the standard rate limit, not for no limit.
959    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
960        Self {
961            max_restarts,
962            backoff,
963            max_backoff: DEFAULT_MAX_BACKOFF,
964            window: DEFAULT_RESTART_WINDOW,
965        }
966    }
967
968    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
969        self.max_backoff = max_backoff;
970        self
971    }
972
973    pub fn with_window(mut self, window: Duration) -> Self {
974        self.window = window;
975        self
976    }
977
978    /// Calculate the capped exponential delay for the next crash replacement.
979    /// `restart_in_window` is zero for the first replacement after an operator
980    /// action (restart, reload, re-enable) cleared the crash ring, or after all
981    /// older crash replacements have aged out of the window.
982    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
983        if self.backoff.is_zero() || self.max_backoff.is_zero() {
984            return Duration::ZERO;
985        }
986
987        let mut delay = self.backoff;
988        for _ in 0..restart_in_window {
989            if delay >= self.max_backoff {
990                return self.max_backoff;
991            }
992            delay = delay
993                .checked_mul(10)
994                .unwrap_or(self.max_backoff)
995                .min(self.max_backoff);
996        }
997        delay.min(self.max_backoff)
998    }
999
1000    /// The one sentence that explains a budget-exhausted stop, used for both the
1001    /// log line and the terminal record so the two cannot drift. It names the
1002    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1003    /// exactly what this budget is not.
1004    fn budget_exhausted_detail(&self) -> String {
1005        format!(
1006            "crash budget exhausted: max_restarts={} within window_secs={}",
1007            self.max_restarts,
1008            self.window.as_secs()
1009        )
1010    }
1011}
1012
1013impl Default for RestartPolicy {
1014    fn default() -> Self {
1015        Self {
1016            max_restarts: DEFAULT_MAX_RESTARTS,
1017            backoff: DEFAULT_BACKOFF,
1018            max_backoff: DEFAULT_MAX_BACKOFF,
1019            window: DEFAULT_RESTART_WINDOW,
1020        }
1021    }
1022}
1023
1024#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1025struct CrashRestartSchedule {
1026    restart_in_window: u32,
1027    delay: Duration,
1028}
1029
1030/// Whether the daemon itself will bring this module back after the exit being
1031/// handled: it is enabled AND its in-window crash restarts are below the cap.
1032///
1033/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1034/// the window are dropped here rather than by a timer, so the count is right
1035/// the moment somebody asks and no bookkeeping runs for idle modules.
1036fn daemon_will_restart(
1037    state: &mut SupervisorSnapshot,
1038    policy: &RestartPolicy,
1039    now: Instant,
1040) -> bool {
1041    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1042}
1043
1044const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1045const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1046const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1047const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1048
1049#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1050pub enum HealthAction {
1051    Report,
1052    Restart,
1053    Alert,
1054}
1055
1056impl fmt::Display for HealthAction {
1057    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1058        f.write_str(match self {
1059            Self::Report => "report",
1060            Self::Restart => "restart",
1061            Self::Alert => "alert",
1062        })
1063    }
1064}
1065
1066#[derive(Debug, Clone, PartialEq, Eq)]
1067pub struct HealthConfig {
1068    /// Optional loopback HTTP endpoint for a managed non-wire process.
1069    /// Changing it applies live on rescan; the process protocol changes only
1070    /// at its next spawn.
1071    pub http: Option<String>,
1072    pub cadence: Duration,
1073    pub deadline: Duration,
1074    pub failure_threshold: u32,
1075    pub on_degraded: HealthAction,
1076    pub on_failing: HealthAction,
1077    pub critical: bool,
1078}
1079
1080impl Default for HealthConfig {
1081    fn default() -> Self {
1082        Self {
1083            http: None,
1084            cadence: DEFAULT_HEALTH_CADENCE,
1085            deadline: DEFAULT_HEALTH_DEADLINE,
1086            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1087            on_degraded: HealthAction::Report,
1088            on_failing: HealthAction::Report,
1089            critical: false,
1090        }
1091    }
1092}
1093
1094/// The supervisor's view of one module's health, relayed to clients over
1095/// channel-0 and rendered by `ck health`.
1096///
1097/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1098/// stated here rather than only at the wire type a consumer reads. A reader can
1099/// look up what `None` means; only a writer can silently change it, and the
1100/// writer has no reason to go looking at a downstream contract before editing.
1101///
1102/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1103/// back to `None` on re-registration precisely so a respawned module does not
1104/// carry its predecessor's timestamp — so an old value and an absent one call for
1105/// opposite readings, and anything that defaulted this to a number would make a
1106/// never-probed module indistinguishable from one probed at the epoch.
1107///
1108/// `detail` and `metrics` are `None` when the module published none on this
1109/// probe, which does not mean it reported nothing wrong — it is also the shape
1110/// when the probe never reached it. `last_probe_ms` is what separates those.
1111#[derive(Debug, Clone, PartialEq)]
1112pub struct ModuleHealthStatus {
1113    pub status: SupervisorHealthStatus,
1114    pub last_probe_ms: Option<u64>,
1115    pub detail: Option<String>,
1116    pub metrics: Option<Value>,
1117    pub consecutive_failures: u32,
1118    /// Number of replies received after a recurring health probe's deadline.
1119    /// Unlike a timeout, every increment proves the module was alive.
1120    pub late_answer_count: u64,
1121    /// End-to-end latency of the newest late reply, measured from probe start.
1122    pub last_late_answer_latency_ms: Option<u64>,
1123    pub last_action: Option<String>,
1124    /// Set together with `last_action`; the pair moves as one, and both being
1125    /// absent means no escalation has ever been taken rather than that the last
1126    /// one succeeded.
1127    pub last_action_ms: Option<u64>,
1128}
1129
1130impl Default for ModuleHealthStatus {
1131    fn default() -> Self {
1132        Self {
1133            status: SupervisorHealthStatus::Unknown,
1134            last_probe_ms: None,
1135            detail: None,
1136            metrics: None,
1137            consecutive_failures: 0,
1138            late_answer_count: 0,
1139            last_late_answer_latency_ms: None,
1140            last_action: None,
1141            last_action_ms: None,
1142        }
1143    }
1144}
1145
1146/// Typed lifecycle state for a supervised module.
1147#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1148pub enum ModuleState {
1149    Starting,
1150    Running,
1151    Unresponsive,
1152    Restarting,
1153    Draining,
1154    Stopped,
1155    Failed,
1156    Disabled,
1157}
1158
1159impl fmt::Display for ModuleState {
1160    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1161        f.write_str(match self {
1162            Self::Starting => "starting",
1163            Self::Running => "running",
1164            Self::Unresponsive => "unresponsive",
1165            Self::Restarting => "restarting",
1166            Self::Draining => "draining",
1167            Self::Stopped => "stopped",
1168            Self::Failed => "failed",
1169            Self::Disabled => "disabled",
1170        })
1171    }
1172}
1173
1174/// Supervisor classification of a child-process exit.
1175#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1176pub enum ExitKind {
1177    Clean,
1178    Crash,
1179    DeliberateSeverance,
1180}
1181
1182impl From<ExitKind> for TerminalExitKind {
1183    fn from(kind: ExitKind) -> Self {
1184        match kind {
1185            ExitKind::Clean => Self::Clean,
1186            ExitKind::Crash => Self::Crash,
1187            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1188        }
1189    }
1190}
1191
1192/// Exact process identity retained when a supervised module registers its
1193/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1194#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1195pub(crate) struct ProcessIdentity {
1196    pub(crate) pid: u32,
1197    pub(crate) start_time: u64,
1198}
1199
1200/// Last observed child exit, if any.
1201#[derive(Debug, Clone, PartialEq, Eq)]
1202pub struct ExitReport {
1203    pub kind: ExitKind,
1204    pub code: Option<i32>,
1205    pub signal: Option<i32>,
1206    pub at_ms: u64,
1207}
1208
1209/// Point-in-time module status answerable by subc without forwarding to the
1210/// module process.
1211#[derive(Debug, Clone, PartialEq)]
1212pub struct ModuleStatus {
1213    pub module_id: String,
1214    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1215    pub waiting_for_previous_pid: Option<u32>,
1216    /// Why the previous instance still prevents a new start.
1217    pub waiting_for_previous_detail: Option<String>,
1218    /// The daemon's Windows per-spawn record: `pipe` after the process it
1219    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1220    /// List/status reports use this record, not the source the module sends
1221    /// in its registration message's manifest.provenance.launch_nonce_source.
1222    pub launch_nonce_source: Option<String>,
1223    pub state: ModuleState,
1224    pub enabled: bool,
1225    pub process_alive: bool,
1226    pub registration_active: bool,
1227    /// The module's declared wire protocol, carried beside `live` because it is
1228    /// what makes `live` readable: the two fields answer one question together.
1229    /// While a process is alive this is its launch declaration, not a later
1230    /// pending-reload edit. When down it is the configured next launch protocol.
1231    pub protocol: ModuleProtocol,
1232    /// Whether the module is serving, under the strongest definition the daemon
1233    /// can assert for its protocol.
1234    ///
1235    /// A subc module must also be REGISTERED: its process being alive says
1236    /// nothing about whether it can take a request. A `protocol: "none"` module
1237    /// never registers, so that term is dropped and this falls back to "enabled,
1238    /// running, and the process the daemon launched is alive" -- which is all
1239    /// the daemon observes about a process that speaks no subc wire. It stays a
1240    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1241    /// rather than printing it bare.
1242    pub live: bool,
1243    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1244    /// restarts have already released their slot, so this count can go down
1245    /// without anybody touching the module.
1246    pub restart_count: u32,
1247    /// Replacement processes spawned over this module's entire supervisor lifetime;
1248    /// unlike `restart_count`, this value is never reset by an operator action
1249    /// and never falls out of a window.
1250    pub lifetime_restarts: u32,
1251    pub spawn_generation: u64,
1252    /// The budget `restart_count` is spent against. Carried alongside the count
1253    /// because the count alone does not say how close the module is to being
1254    /// disabled, and reporting one without the other is what makes an
1255    /// about-to-be-retired module look ordinary.
1256    pub max_restarts: u32,
1257    /// The span `restart_count` is counted over. Carried with the pair above for
1258    /// the same reason they are carried together: "2 of 3" means one thing for a
1259    /// ten-minute window and something else entirely for a lifetime.
1260    pub restart_window: Duration,
1261    /// Effective drain and restart timing policy used by this running module.
1262    /// These values are carried together with the restart budget so status
1263    /// readers can compare configured intent with what the supervisor applied.
1264    pub drain_timeout: Duration,
1265    pub restart_backoff: Duration,
1266    pub restart_max_backoff: Duration,
1267    /// The module's process. On macOS this stays absent while the `ck-subc`
1268    /// launch trampoline is still running in that pid, and appears once the
1269    /// exec acknowledgement confirms the module image has replaced it. Launch
1270    /// time and the supervisor's own hold on the process are unaffected.
1271    pub pid: Option<u32>,
1272    pub spawned_at_ms: Option<u64>,
1273    pub spawned_from: Option<PathBuf>,
1274    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1275    pub process_start_time: Option<u64>,
1276    pub last_exit: Option<ExitReport>,
1277    pub health: ModuleHealthStatus,
1278}
1279
1280#[cfg(test)]
1281#[derive(Debug, Clone, Copy, PartialEq)]
1282struct ActorSelectCheckpoint {
1283    generation: u64,
1284    turn: u64,
1285    registered_connection: Option<ConnectionId>,
1286    next_probe_at: Option<Instant>,
1287    wake_after: Option<Duration>,
1288}
1289
1290#[derive(Debug, Clone, PartialEq)]
1291struct SupervisorSnapshot {
1292    subtree_target: crate::subtree_resources::Target,
1293    failed_retry: FailedRetry,
1294    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1295    #[cfg(windows)]
1296    pipe_consumed: Option<Arc<OnceLock<()>>>,
1297    /// Counts running-child loop turns, not executor polls of a parked wait.
1298    #[cfg(test)]
1299    actor_turns: u64,
1300    /// Running select after exec confirmation and with no unconsumed registry
1301    /// event. Recording a spawn or starting a loop turn is not this barrier.
1302    #[cfg(test)]
1303    actor_select: Option<ActorSelectCheckpoint>,
1304    state: ModuleState,
1305    enabled: bool,
1306    process_alive: bool,
1307    spawned_protocol: Option<ModuleProtocol>,
1308    spawn_failure: Option<String>,
1309    waiting_for_previous_pid: Option<u32>,
1310    waiting_for_previous_detail: Option<String>,
1311    /// When each crash restart was spent, oldest first. This IS the crash
1312    /// budget: its in-window length is the count an operator sees and the count
1313    /// the restart decision is made against, so there is no second counter that
1314    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1315    /// operator actions that used to zero the old lifetime counter.
1316    crash_restarts: VecDeque<Instant>,
1317    lifetime_restarts: u32,
1318    /// Successful child spawns in this daemon incarnation.
1319    ///
1320    /// `lifetime_restarts` was considered and rejected: it starts at zero
1321    /// (line 640), successful initial/operator spawns in `set_running` do not
1322    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1323    /// increments before a successful replacement exists (lines 604, 3846,
1324    /// and 3921), so a failed spawn can consume it. This counter moves only
1325    /// when a live PID is accepted below.
1326    spawn_generation: u64,
1327    pid: Option<u32>,
1328    #[cfg(target_os = "macos")]
1329    report_ready: Option<Arc<OnceLock<()>>>,
1330    /// Last reaped child, retained after current process facts are cleared.
1331    reaped_pid: Option<u32>,
1332    /// Whether a replacement has already been scheduled for the current process.
1333    respawn_pending: bool,
1334    /// A second restart is waiting for the replacement already scheduled.
1335    coalesced_restart_pending: bool,
1336    spawned_at_ms: Option<u64>,
1337    spawned_from: Option<PathBuf>,
1338    launch_spec_digest: Option<LaunchSpecDigest>,
1339    spawned_file_identity: Option<SpawnedFileIdentity>,
1340    process_start_time: Option<u64>,
1341    deliberate_severance: Option<ProcessIdentity>,
1342    last_exit: Option<ExitReport>,
1343    /// Diagnostic attached to the next drain's terminal record, if any.
1344    drain_disposition_detail: Option<String>,
1345    health: ModuleHealthStatus,
1346    /// Whether the current process was started as a swap candidate and so
1347    /// lives in the module's alternate cgroup. The next swap's candidate takes
1348    /// the other one, so the two processes of a swap never share a cgroup. A
1349    /// plain spawn always uses the primary cgroup.
1350    in_alternate_slot: bool,
1351    /// Whether the current `Draining` state ends in a replacement process
1352    /// (restart, reload, health restart) rather than a stop. Only meaningful
1353    /// while `state` is `Draining`; every entry into that state rewrites it.
1354    /// It is what lets route.open answer the retryable `module_reloading` to a
1355    /// consumer that reaches a still-registered process mid-restart, instead of
1356    /// the `supervisor_not_live` a stop or disable deserves.
1357    draining_to_replace: bool,
1358    /// Whether a configuration update has been applied since the current
1359    /// process was spawned, so that process runs an older spec than the one
1360    /// the supervisor now holds. A queued restart is only coalesced into a
1361    /// fresher process when this is false: a restart requested to pick up a
1362    /// new configuration must not be satisfied by a process that predates it.
1363    configuration_updated_since_spawn: bool,
1364}
1365
1366impl SupervisorSnapshot {
1367    /// The pid that status, provenance and resource readings may report. While
1368    /// the launch trampoline still runs in the pid, reading its executable or
1369    /// resource use would describe `ck-subc`, not the module, so none is
1370    /// reported until the exec acknowledgement confirms the module image.
1371    fn reported_pid(&self) -> Option<u32> {
1372        #[cfg(target_os = "macos")]
1373        if self
1374            .report_ready
1375            .as_ref()
1376            .is_some_and(|ready| ready.get().is_none())
1377        {
1378            return None;
1379        }
1380        self.pid
1381    }
1382
1383    fn starting() -> Self {
1384        Self::new(ModuleState::Starting, true)
1385    }
1386
1387    fn disabled() -> Self {
1388        Self::new(ModuleState::Disabled, false)
1389    }
1390
1391    /// Crash restarts still inside `window`, having dropped the ones that are
1392    /// not. Pruning on read is what makes the budget a rate: an instant older
1393    /// than the window stops holding a slot the moment anybody counts.
1394    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1395        while let Some(oldest) = self.crash_restarts.front() {
1396            if now.duration_since(*oldest) > window {
1397                self.crash_restarts.pop_front();
1398            } else {
1399                break;
1400            }
1401        }
1402        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1403    }
1404
1405    /// Spend one unit of the crash budget and record the restart in the ledger.
1406    ///
1407    /// The ring is bounded by the cap because more than `max_restarts` in-window
1408    /// instants can never be reached (the caller refuses the restart first), so
1409    /// anything beyond that is an unbounded queue waiting to happen.
1410    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1411        self.crash_restarts.push_back(now);
1412        while self.crash_restarts.len() > policy.max_restarts as usize {
1413            self.crash_restarts.pop_front();
1414        }
1415        self.lifetime_restarts += 1;
1416    }
1417
1418    /// Reserve one crash-restart slot and calculate the delay before respawning.
1419    /// The count is captured before recording this restart, so the first retry
1420    /// uses the base delay and each later in-window retry escalates once.
1421    fn next_crash_restart(
1422        &mut self,
1423        policy: &RestartPolicy,
1424        now: Instant,
1425    ) -> Option<CrashRestartSchedule> {
1426        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1427        if restart_in_window >= policy.max_restarts {
1428            return None;
1429        }
1430        self.record_crash_restart(policy, now);
1431        Some(CrashRestartSchedule {
1432            restart_in_window,
1433            delay: policy.delay_for_restart(restart_in_window),
1434        })
1435    }
1436
1437    /// Give the module its full budget back, as an operator restart, reload, or
1438    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1439    /// ledger of what actually happened, and an operator action does not unmake
1440    /// the crashes.
1441    fn clear_crash_restarts(&mut self) {
1442        self.crash_restarts.clear();
1443    }
1444
1445    fn new(state: ModuleState, enabled: bool) -> Self {
1446        Self {
1447            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1448            failed_retry: FailedRetry::default(),
1449            exec_outcome: None,
1450            #[cfg(test)]
1451            actor_turns: 0,
1452            #[cfg(test)]
1453            actor_select: None,
1454            state,
1455            enabled,
1456            process_alive: false,
1457            #[cfg(windows)]
1458            pipe_consumed: None,
1459            spawned_protocol: None,
1460            spawn_failure: None,
1461            waiting_for_previous_pid: None,
1462            waiting_for_previous_detail: None,
1463            crash_restarts: VecDeque::new(),
1464            lifetime_restarts: 0,
1465            spawn_generation: 0,
1466            pid: None,
1467            #[cfg(target_os = "macos")]
1468            report_ready: None,
1469            reaped_pid: None,
1470            respawn_pending: false,
1471            coalesced_restart_pending: false,
1472            spawned_at_ms: None,
1473            spawned_from: None,
1474            launch_spec_digest: None,
1475            spawned_file_identity: None,
1476            process_start_time: None,
1477            deliberate_severance: None,
1478            last_exit: None,
1479            drain_disposition_detail: None,
1480            health: ModuleHealthStatus::default(),
1481            in_alternate_slot: false,
1482            draining_to_replace: false,
1483            configuration_updated_since_spawn: false,
1484        }
1485    }
1486}
1487
1488type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1489
1490type SpawnSubscriberKey = (ConnectionId, u64);
1491
1492#[derive(Debug)]
1493struct SpawnSubscriber {
1494    version: u8,
1495    frames: mpsc::Sender<Frame>,
1496    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1497    /// from which event. The full frame channel cannot carry that news, so it
1498    /// travels beside it; see `SpawnEventFeed::subscribe`.
1499    lagged: Option<oneshot::Sender<SpawnCursor>>,
1500}
1501
1502#[derive(Debug)]
1503struct SpawnEventState {
1504    daemon_incarnation: String,
1505    seq: u64,
1506    capacity: usize,
1507    live: HashMap<String, LiveSpawn>,
1508    generations: HashMap<String, u64>,
1509    events: VecDeque<SpawnEvent>,
1510    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1511}
1512
1513impl Default for SpawnEventState {
1514    fn default() -> Self {
1515        Self {
1516            daemon_incarnation: "unconfigured".to_string(),
1517            seq: 0,
1518            capacity: SPAWN_EVENT_RING_CAPACITY,
1519            live: HashMap::new(),
1520            generations: HashMap::new(),
1521            events: VecDeque::new(),
1522            subscribers: HashMap::new(),
1523        }
1524    }
1525}
1526
1527#[derive(Debug, Clone, Default)]
1528struct SpawnEventFeed(
1529    Arc<Mutex<SpawnEventState>>,
1530    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1531);
1532
1533#[derive(Debug, Clone, PartialEq, Eq)]
1534pub(crate) enum SpawnSubscribeRefusal {
1535    ForeignIncarnation { current: String },
1536    TooOld { oldest: SpawnCursor },
1537    Frame(String),
1538}
1539
1540impl SpawnEventFeed {
1541    fn configure_incarnation(&self, daemon_incarnation: String) {
1542        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1543        state.daemon_incarnation = daemon_incarnation;
1544        state.seq = 0;
1545        state.live.clear();
1546        state.generations.clear();
1547        state.events.clear();
1548        state.subscribers.clear();
1549    }
1550
1551    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1552        SpawnCursor {
1553            daemon_incarnation: state.daemon_incarnation.clone(),
1554            seq: state.seq,
1555        }
1556    }
1557
1558    fn snapshot(&self) -> SpawnSnapshot {
1559        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1560        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1561        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1562        SpawnSnapshot {
1563            cursor: Self::cursor(&state),
1564            ring_bound: state.capacity as u64,
1565            live,
1566        }
1567    }
1568
1569    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1570        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1571        let generation = state
1572            .generations
1573            .get(module_id)
1574            .copied()
1575            .unwrap_or(0)
1576            .checked_add(1)
1577            .expect("spawn generation exhausted");
1578        state.generations.insert(module_id.to_string(), generation);
1579        let live = LiveSpawn {
1580            module_id: module_id.to_string(),
1581            spawn_generation: generation,
1582            pid,
1583            spawned_at_ms,
1584        };
1585        state.live.insert(module_id.to_string(), live);
1586        Self::emit_locked(
1587            &mut state,
1588            SpawnEventKind::Spawned,
1589            module_id.to_string(),
1590            generation,
1591            pid,
1592            None,
1593            None,
1594        );
1595        generation
1596    }
1597
1598    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1599        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1600        let Some(live) = state.live.remove(module_id) else {
1601            warn!(
1602                module_id,
1603                "terminal record had no live spawn event identity"
1604            );
1605            return;
1606        };
1607        Self::emit_locked(
1608            &mut state,
1609            SpawnEventKind::Exited,
1610            module_id.to_string(),
1611            live.spawn_generation,
1612            live.pid,
1613            exit_code,
1614            exit_signal,
1615        );
1616    }
1617
1618    /// Report the exit of a process that a swap has already replaced.
1619    ///
1620    /// `emit_exited` removes the module's live entry, which after a swap's
1621    /// cutover describes the promoted candidate, not the old process now
1622    /// exiting. This emits the old generation's exit and leaves the live entry
1623    /// alone unless it still names that generation.
1624    fn emit_superseded_exited(
1625        &self,
1626        module_id: &str,
1627        spawn_generation: u64,
1628        pid: u32,
1629        exit_code: Option<i32>,
1630        exit_signal: Option<i32>,
1631    ) {
1632        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1633        if state
1634            .live
1635            .get(module_id)
1636            .is_some_and(|live| live.spawn_generation == spawn_generation)
1637        {
1638            state.live.remove(module_id);
1639        }
1640        Self::emit_locked(
1641            &mut state,
1642            SpawnEventKind::Exited,
1643            module_id.to_string(),
1644            spawn_generation,
1645            pid,
1646            exit_code,
1647            exit_signal,
1648        );
1649    }
1650
1651    #[allow(clippy::too_many_arguments)]
1652    fn emit_locked(
1653        state: &mut SpawnEventState,
1654        kind: SpawnEventKind,
1655        module_id: String,
1656        spawn_generation: u64,
1657        pid: u32,
1658        exit_code: Option<i32>,
1659        exit_signal: Option<i32>,
1660    ) {
1661        state.seq = state
1662            .seq
1663            .checked_add(1)
1664            .expect("spawn event sequence exhausted");
1665        let event = SpawnEvent {
1666            cursor: Self::cursor(state),
1667            kind,
1668            module_id,
1669            spawn_generation,
1670            pid,
1671            exit_code,
1672            exit_signal,
1673        };
1674        state.events.push_back(event.clone());
1675        while state.events.len() > state.capacity {
1676            state.events.pop_front();
1677        }
1678        let body = match serde_json::to_vec(&event) {
1679            Ok(body) => body,
1680            Err(error) => {
1681                error!(%error, "failed to serialize supervisor spawn event");
1682                return;
1683            }
1684        };
1685        state.subscribers.retain(|(connection_id, corr), subscriber| {
1686            let frame = Frame::build_with_version(
1687                subscriber.version,
1688                FrameType::StreamData,
1689                control_flags(),
1690                0,
1691                0,
1692                *corr,
1693                body.clone(),
1694            );
1695            match frame {
1696                Ok(frame) => {
1697                    if subscriber.frames.try_send(frame).is_ok() {
1698                        true
1699                    } else {
1700                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1701                        if let Some(lagged) = subscriber.lagged.take() {
1702                            let _ = lagged.send(event.cursor.clone());
1703                        }
1704                        false
1705                    }
1706                }
1707                Err(error) => {
1708                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1709                    false
1710                }
1711            }
1712        });
1713    }
1714
1715    fn subscribe(
1716        &self,
1717        connection_id: ConnectionId,
1718        corr: u64,
1719        version: u8,
1720        since: Option<SpawnCursor>,
1721        sink: FrameSink,
1722    ) -> Result<(), SpawnSubscribeRefusal> {
1723        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1724        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1725        {
1726            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1727            let replay = if let Some(since) = since {
1728                if since.daemon_incarnation != state.daemon_incarnation {
1729                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1730                        current: state.daemon_incarnation.clone(),
1731                    });
1732                }
1733                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1734                    if since.seq < oldest.seq.saturating_sub(1) {
1735                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1736                    }
1737                }
1738                state
1739                    .events
1740                    .iter()
1741                    .filter(|event| event.cursor.seq > since.seq)
1742                    .cloned()
1743                    .collect::<Vec<_>>()
1744            } else {
1745                Vec::new()
1746            };
1747            for event in replay {
1748                let body = serde_json::to_vec(&event)
1749                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1750                let frame = Frame::build_with_version(
1751                    version,
1752                    FrameType::StreamData,
1753                    control_flags(),
1754                    0,
1755                    0,
1756                    corr,
1757                    body,
1758                )
1759                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1760                frames
1761                    .try_send(frame)
1762                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1763            }
1764            state.subscribers.insert(
1765                (connection_id, corr),
1766                SpawnSubscriber {
1767                    version,
1768                    frames,
1769                    lagged: Some(lagged),
1770                },
1771            );
1772        }
1773        // The lagged terminal is sent here, by the forwarder, rather than by
1774        // the emitter: at the moment of the drop the subscriber's own channel
1775        // is full, and writing to the connection sink directly from the emitter
1776        // would put the Error AHEAD of the events still queued in that channel
1777        // (and the emitter holds the feed lock, so it cannot await the sink).
1778        // Dropping the subscriber drops the only sender, so `recv` drains every
1779        // queued event and then returns `None`; only then is the Error sent, so
1780        // the client sees each event it can keep, then the reason it was cut.
1781        // Cancel and connection removal drop the oneshot unsent, so they end
1782        // the stream with no Error.
1783        tokio::spawn(async move {
1784            while let Some(frame) = receiver.recv().await {
1785                if sink.send(frame).await.is_err() {
1786                    return;
1787                }
1788            }
1789            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1790                return;
1791            };
1792            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1793                Ok(frame) => {
1794                    let _ = sink.send(frame).await;
1795                }
1796                Err(error) => {
1797                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1798                }
1799            }
1800        });
1801        Ok(())
1802    }
1803
1804    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1805        let Some(subscriber) = self
1806            .0
1807            .lock()
1808            .unwrap_or_else(|p| p.into_inner())
1809            .subscribers
1810            .remove(&(connection_id, corr))
1811        else {
1812            return false;
1813        };
1814        if let Ok(frame) = Frame::build_with_version(
1815            subscriber.version,
1816            FrameType::StreamEnd,
1817            control_flags(),
1818            0,
1819            0,
1820            corr,
1821            Vec::new(),
1822        ) {
1823            tokio::spawn(async move {
1824                let _ = subscriber.frames.send(frame).await;
1825            });
1826        }
1827        true
1828    }
1829
1830    fn remove_connection(&self, connection_id: ConnectionId) {
1831        self.0
1832            .lock()
1833            .unwrap_or_else(|p| p.into_inner())
1834            .subscribers
1835            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1836    }
1837
1838    #[cfg(any(test, feature = "test-support"))]
1839    fn set_capacity(&self, capacity: usize) {
1840        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1841    }
1842
1843    #[cfg(any(test, feature = "test-support"))]
1844    fn subscriber_count(&self) -> usize {
1845        self.0
1846            .lock()
1847            .unwrap_or_else(|p| p.into_inner())
1848            .subscribers
1849            .len()
1850    }
1851}
1852
1853/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1854/// The terminal Error a lagged spawn subscriber receives after its queued events.
1855fn spawn_subscriber_lagged_frame(
1856    version: u8,
1857    corr: u64,
1858    first_undelivered: SpawnCursor,
1859) -> Result<Frame, String> {
1860    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1861    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1862        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1863        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1864            .to_string(),
1865        detail: Some(serde_json::json!({
1866            "first_undelivered_cursor": first_undelivered
1867        })),
1868    })
1869    .map_err(|error| error.to_string())?;
1870    Frame::build_with_version(
1871        version,
1872        FrameType::Error,
1873        crate::router::daemon_error_flags(),
1874        0,
1875        0,
1876        corr,
1877        body,
1878    )
1879    .map_err(|error| error.to_string())
1880}
1881
1882pub trait ModuleProcessLiveness: Send + Sync {
1883    fn process_live(&self, module_id: &str) -> Option<bool>;
1884
1885    /// Whether the supervisor is replacing this module's process right now: an
1886    /// operator restart or reload, a health restart, or a crash respawn whose
1887    /// backoff is running. A module in that state is not live, but a consumer
1888    /// refused now should retry shortly rather than treat the target as gone.
1889    /// Stopped, failed, and disabled modules are not replacing.
1890    fn process_replacing(&self, _module_id: &str) -> bool {
1891        false
1892    }
1893}
1894
1895/// Shared process-liveness registry keyed by supervised `module_id`.
1896#[derive(Debug, Clone, Default)]
1897pub struct SupervisorProcessLiveness {
1898    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1899}
1900
1901impl SupervisorProcessLiveness {
1902    pub fn new() -> Self {
1903        Self::default()
1904    }
1905
1906    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1907        let mut snapshots = self
1908            .snapshots
1909            .lock()
1910            .unwrap_or_else(|poisoned| poisoned.into_inner());
1911        snapshots.insert(module_id, snapshot);
1912    }
1913
1914    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1915        let mut snapshots = self
1916            .snapshots
1917            .lock()
1918            .unwrap_or_else(|poisoned| poisoned.into_inner());
1919        let is_current = snapshots
1920            .get(module_id)
1921            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1922            .unwrap_or(false);
1923        if is_current {
1924            snapshots.remove(module_id);
1925        }
1926    }
1927}
1928
1929impl ModuleProcessLiveness for SupervisorProcessLiveness {
1930    fn process_live(&self, module_id: &str) -> Option<bool> {
1931        let snapshot = {
1932            let snapshots = self
1933                .snapshots
1934                .lock()
1935                .unwrap_or_else(|poisoned| poisoned.into_inner());
1936            snapshots.get(module_id).cloned()
1937        }?;
1938        let snapshot = snapshot
1939            .lock()
1940            .unwrap_or_else(|poisoned| poisoned.into_inner());
1941        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1942    }
1943
1944    fn process_replacing(&self, module_id: &str) -> bool {
1945        let Some(snapshot) = self
1946            .snapshots
1947            .lock()
1948            .unwrap_or_else(|poisoned| poisoned.into_inner())
1949            .get(module_id)
1950            .cloned()
1951        else {
1952            return false;
1953        };
1954        let snapshot = snapshot
1955            .lock()
1956            .unwrap_or_else(|poisoned| poisoned.into_inner());
1957        snapshot.enabled
1958            && match snapshot.state {
1959                ModuleState::Restarting => true,
1960                ModuleState::Draining => snapshot.draining_to_replace,
1961                ModuleState::Starting
1962                | ModuleState::Running
1963                | ModuleState::Unresponsive
1964                | ModuleState::Stopped
1965                | ModuleState::Failed
1966                | ModuleState::Disabled => false,
1967            }
1968    }
1969}
1970
1971#[cfg(test)]
1972#[derive(Debug, Default)]
1973struct ReloadExitRecordGate {
1974    reached: tokio::sync::Notify,
1975    resume: tokio::sync::Notify,
1976}
1977
1978#[derive(Debug, Clone, Copy)]
1979enum RespawnKind {
1980    Spawn,
1981    Reload,
1982}
1983
1984#[derive(Debug, Clone, Copy)]
1985struct PendingRespawn {
1986    deadline: Instant,
1987    kind: RespawnKind,
1988    daemon_initiated: bool,
1989}
1990
1991type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1992
1993#[derive(Debug, Clone)]
1994struct SupervisorRuntimeConfig {
1995    registry: Arc<Registry>,
1996    /// Retained across an operation's deferred backoff, until its child starts.
1997    recovery_is_daemon: bool,
1998    start_requester: Option<Principal>,
1999    pending_activity_restart: bool,
2000    provenance_probe: ExecutableIdentityProbe,
2001    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2002    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2003    /// A reload acknowledges completion only after its replacement registers.
2004    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2005    restart_policy: RestartPolicy,
2006    register_timeout: Duration,
2007    /// This module's RESOLVED drain budget: per-module config when present,
2008    /// else `default_drain_timeout`.
2009    drain_timeout: Duration,
2010    /// Shared with the status handle so the attested value changes atomically
2011    /// when a rescan updates the running drain policy.
2012    effective_drain_timeout: Arc<Mutex<Duration>>,
2013    /// The supervisor-wide fallback, kept so a configuration update that
2014    /// REMOVES the per-module override can re-resolve to it.
2015    default_drain_timeout: Duration,
2016    health: HealthConfig,
2017    connection_file_path: Option<PathBuf>,
2018    capture_logs_dir: Option<PathBuf>,
2019    forwarding: Option<Arc<ForwardingTable>>,
2020    /// The shared handle, so every spawn path (initial, restart, reload) records the
2021    /// reserved-module launch nonce the HELLO verifier checks against.
2022    supervisor_handle: Option<SupervisorHandle>,
2023    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2024    /// status queries.
2025    ///
2026    /// One ring per module, held across every respawn. The lines explaining an exit
2027    /// are written BEFORE that exit, so a ring recreated per process would be empty
2028    /// exactly when it is asked for.
2029    stderr_ring: Arc<Mutex<StderrRing>>,
2030    terminal_ring: Arc<Mutex<TerminalRing>>,
2031    spawn_events: SpawnEventFeed,
2032    child_roster: ChildRoster,
2033    #[cfg(target_os = "linux")]
2034    cgroup_placement: Option<subc_cgroup::Placement>,
2035    #[cfg(test)]
2036    test_seed_stale_facts_before_enable_spawn: bool,
2037    #[cfg(test)]
2038    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2039}
2040
2041impl SupervisorRuntimeConfig {
2042    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2043        FailureContext {
2044            daemon_initiated,
2045            shutdown: self.child_roster.shutdown_flag(),
2046        }
2047    }
2048}
2049
2050#[derive(Debug, Clone, PartialEq, Eq)]
2051struct SupervisedConfiguration {
2052    spec: ModuleSpec,
2053    health: HealthConfig,
2054}
2055
2056/// Shared daemon lookup table for supervised module handles.
2057///
2058/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2059/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2060/// launch nonces recorded at spawn are checked by the same daemon instance.
2061#[derive(Debug, Clone, Default)]
2062pub struct SupervisorHandle {
2063    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2064    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2065    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2066    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2067    /// Module ids the supervisor has taken on. An id is added BEFORE the
2068    /// module's first process is spawned and removed only when the module
2069    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2070    /// the keys of `modules`.
2071    ///
2072    /// `modules` cannot answer "is this module configured?" on its own: a
2073    /// [`SupervisedModule`] only exists once its process has been spawned, and
2074    /// a fast child can connect, register, sync its scopes and ask about them
2075    /// before the supervisor has inserted it. Answering "not configured" in that
2076    /// gap makes scope admission refuse with the terminal "will never sync"
2077    /// instead of the retryable "has not synced yet".
2078    configured_ids: Arc<Mutex<HashSet<String>>>,
2079    spawn_events: SpawnEventFeed,
2080    /// The current expected launch nonce for each reserved module_id. Set when the
2081    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2082    /// non-reserved module never has an entry here and is never nonce-checked.
2083    /// Reserved module ids and the nonce that authorizes their next HELLO.
2084    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2085    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2086    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2087    /// had NO entry and admitted anyone: the reservation protected the nonce
2088    /// holder, not the NAME (found live by CKCRED's canary probe registering
2089    /// against a reserved scratch id).
2090    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2091    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2092    ///
2093    /// This is deliberately in-memory only: subc is state-free across daemon
2094    /// restarts, and the tombstone only explains the hours-after-removal window
2095    /// while this executing daemon is still alive. Do not persist it in a store.
2096    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2097    /// The current launch nonce for every supervised spawn. This is separate from
2098    /// reserved_nonces because consumer route.open attestation applies to all spawned
2099    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2100    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2101    /// Reserved namespace prefixes mapped to the supervised owner module whose
2102    /// current spawn nonce authorizes HELLO claims below the prefix.
2103    ///
2104    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2105    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2106    /// accidental collisions and lower-trust processes from squatting protected
2107    /// namespaces.
2108    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2109    /// Blue/green swaps in progress, by module id. An entry exists from just
2110    /// before the candidate process is spawned until the swap has failed, or
2111    /// has cut over and the old process is gone. While it exists, HELLO for the
2112    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2113    /// consumer attestation accepts both processes' nonces.
2114    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2115    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2116    promotion_observer: PromotionObserverSlot,
2117    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2118    /// this daemon-wide ordering, a rescan could retire or update a module while a
2119    /// concurrent reload still held its old handle and launch specification.
2120    operation_lock: Arc<AsyncMutex<()>>,
2121}
2122
2123/// Told when a swap has promoted its candidate to be the module's active
2124/// registration.
2125///
2126/// An ordinary HELLO runs the control plane's registration side effects (the
2127/// capability cache, the deny census, the requirement recompute) as it
2128/// registers. A swap candidate's HELLO does not, because it is not routable;
2129/// promotion is when those must run instead, and promotion happens in the
2130/// supervisor, which has no other way into the control handler.
2131pub(crate) trait SwapPromotionObserver: Send + Sync {
2132    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2133}
2134
2135/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2136/// control handler) owns this handle, so a strong reference back would be a
2137/// cycle that keeps both alive.
2138#[derive(Clone, Default)]
2139struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2140
2141impl fmt::Debug for PromotionObserverSlot {
2142    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2143        f.write_str("PromotionObserverSlot")
2144    }
2145}
2146
2147/// The nonces of one open swap.
2148#[derive(Debug, Clone)]
2149struct OpenSwap {
2150    /// The launch nonce minted for the candidate process. It is the swap
2151    /// token: the only thing that admits a HELLO into the candidate slot.
2152    candidate_nonce: String,
2153    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2154    /// here because cutover moves the module's recorded spawn nonce to the
2155    /// candidate while the incumbent is still draining and its consumers are
2156    /// still attesting with this one.
2157    incumbent_nonce: Option<String>,
2158    /// Set once a HELLO has been admitted with the swap token, so the token
2159    /// admits one registration and cannot be replayed after cutover empties
2160    /// the candidate slot.
2161    candidate_admitted: bool,
2162}
2163
2164/// What the swap gate says about a HELLO. See
2165/// [`SupervisorHandle::swap_hello_admission`].
2166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2167pub(crate) enum SwapHelloAdmission {
2168    /// No swap is open for the id (or the HELLO carries the incumbent's own
2169    /// nonce); the ordinary gates decide.
2170    NotSwapping,
2171    /// The HELLO carries the swap token: register it into the candidate slot.
2172    Candidate,
2173    /// A swap is open and the HELLO carries a nonce the supervisor did not
2174    /// mint for this id, no nonce, or a token already used.
2175    Refused,
2176}
2177
2178#[derive(Debug, Clone, PartialEq, Eq)]
2179pub(crate) enum ReservedHelloRejection {
2180    Exact {
2181        module_id: String,
2182    },
2183    Prefix {
2184        prefix: String,
2185        owner_module_id: String,
2186    },
2187}
2188
2189#[derive(Debug)]
2190struct DaemonRestartIntent {
2191    expires: Option<Duration>,
2192    window: Duration,
2193}
2194
2195impl Default for DaemonRestartIntent {
2196    fn default() -> Self {
2197        Self {
2198            expires: None,
2199            window: Duration::from_secs(120),
2200        }
2201    }
2202}
2203
2204impl SupervisorHandle {
2205    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2206        let mut intent = self
2207            .restart_intent
2208            .lock()
2209            .unwrap_or_else(|p| p.into_inner());
2210        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2211        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2212    }
2213
2214    #[cfg(feature = "test-support")]
2215    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2216        self.restart_intent
2217            .lock()
2218            .unwrap_or_else(|p| p.into_inner())
2219            .window = window;
2220    }
2221
2222    #[cfg(unix)]
2223    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2224        let expiry = self
2225            .restart_intent
2226            .lock()
2227            .unwrap_or_else(|p| p.into_inner())
2228            .expires
2229            .take();
2230        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2231            RouteCloseReason::Restart
2232        } else {
2233            RouteCloseReason::Disable
2234        }
2235    }
2236
2237    pub fn new() -> Self {
2238        Self::default()
2239    }
2240
2241    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2242        self.spawn_events.snapshot()
2243    }
2244
2245    pub(crate) fn subscribe_spawns(
2246        &self,
2247        connection_id: ConnectionId,
2248        corr: u64,
2249        version: u8,
2250        since: Option<SpawnCursor>,
2251        sink: FrameSink,
2252    ) -> Result<(), SpawnSubscribeRefusal> {
2253        self.spawn_events
2254            .subscribe(connection_id, corr, version, since, sink)
2255    }
2256
2257    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2258        self.spawn_events.cancel(connection_id, corr)
2259    }
2260
2261    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2262        self.spawn_events.remove_connection(connection_id);
2263    }
2264
2265    #[cfg(any(test, feature = "test-support"))]
2266    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2267        assert!(capacity > 0, "spawn event capacity must be non-zero");
2268        self.spawn_events.set_capacity(capacity);
2269    }
2270
2271    #[cfg(any(test, feature = "test-support"))]
2272    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2273        self.spawn_events.subscriber_count()
2274    }
2275
2276    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2277    /// a respawn invalidates stale consumer identities.
2278    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2279        self.spawn_nonces
2280            .lock()
2281            .unwrap_or_else(|poisoned| poisoned.into_inner())
2282            .insert(module_id.to_string(), nonce);
2283    }
2284
2285    /// Record the launch nonce expected from the next HELLO for a reserved module,
2286    /// replacing any prior nonce (a respawn invalidates the previous one).
2287    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2288        self.reserved_nonces
2289            .lock()
2290            .unwrap_or_else(|poisoned| poisoned.into_inner())
2291            .insert(module_id.to_string(), Some(nonce));
2292    }
2293
2294    /// Record namespace prefixes owned by a supervised module.
2295    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2296        let mut owners = self
2297            .reserved_prefix_owners
2298            .lock()
2299            .unwrap_or_else(|poisoned| poisoned.into_inner());
2300        owners.retain(|_, owner| owner != owner_module_id);
2301        for prefix in prefixes {
2302            owners.insert(prefix.clone(), owner_module_id.to_string());
2303        }
2304    }
2305
2306    /// The launch nonce most recently minted for a module's spawn, if any.
2307    #[cfg(test)]
2308    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2309        self.spawn_nonces
2310            .lock()
2311            .unwrap_or_else(|poisoned| poisoned.into_inner())
2312            .get(module_id)
2313            .cloned()
2314    }
2315
2316    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2317        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2318        let spawn_nonce = self
2319            .spawn_nonces
2320            .lock()
2321            .unwrap_or_else(|poisoned| poisoned.into_inner())
2322            .get(&spec.module_id)
2323            .cloned();
2324        let mut reserved_nonces = self
2325            .reserved_nonces
2326            .lock()
2327            .unwrap_or_else(|poisoned| poisoned.into_inner());
2328        if spec.reserved {
2329            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2330            // reserved name whose module has never spawned has no legitimate
2331            // holder, and the entry's absence is what used to leave the name
2332            // open to the first claimant.
2333            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2334        }
2335        drop(reserved_nonces);
2336        // A later unreserved declaration must not silently unreserve an id that
2337        // was retained after its reserved configuration was removed. The explicit
2338        // release ceremony is the only operation that retires that gate.
2339        self.removal_tombstones
2340            .lock()
2341            .unwrap_or_else(|poisoned| poisoned.into_inner())
2342            .remove(&spec.module_id);
2343    }
2344
2345    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2346    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2347    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2348    /// with no matching prefix are always authorized.
2349    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2350        self.reserved_hello_verification(module_id, presented)
2351            .is_ok()
2352    }
2353
2354    /// Whether HELLO passed a nonce check that was actually required by reserved
2355    /// module policy, or the refusal explaining why a required check failed.
2356    pub(crate) fn reserved_hello_verification(
2357        &self,
2358        module_id: &str,
2359        presented: Option<&str>,
2360    ) -> Result<bool, ReservedHelloRejection> {
2361        let nonces = self
2362            .reserved_nonces
2363            .lock()
2364            .unwrap_or_else(|poisoned| poisoned.into_inner());
2365        if let Some(expected) = nonces.get(module_id) {
2366            // `None` = reserved with no legitimate holder: refuse every
2367            // presentation, because no process can hold a nonce that was never
2368            // minted. Only a real minted nonce admits, in constant time.
2369            let authorized = match expected {
2370                Some(expected) => {
2371                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2372                }
2373                None => false,
2374            };
2375            if authorized {
2376                return Ok(true);
2377            }
2378            return Err(ReservedHelloRejection::Exact {
2379                module_id: module_id.to_string(),
2380            });
2381        }
2382        drop(nonces);
2383
2384        let matched_prefix = self
2385            .reserved_prefix_owners
2386            .lock()
2387            .unwrap_or_else(|poisoned| poisoned.into_inner())
2388            .iter()
2389            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2390            .max_by_key(|(prefix, _)| prefix.len())
2391            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2392        let Some((prefix, owner_module_id)) = matched_prefix else {
2393            return Ok(false);
2394        };
2395
2396        let authorized = presented.is_some_and(|presented| {
2397            self.spawn_nonces
2398                .lock()
2399                .unwrap_or_else(|poisoned| poisoned.into_inner())
2400                .get(&owner_module_id)
2401                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2402                // While the owner is being swapped, children started by
2403                // either of its two processes hold that process's nonce.
2404                || self.swap_nonce_matches(&owner_module_id, presented)
2405        });
2406        if authorized {
2407            Ok(true)
2408        } else {
2409            Err(ReservedHelloRejection::Prefix {
2410                prefix,
2411                owner_module_id,
2412            })
2413        }
2414    }
2415
2416    /// Whether a consumer connection proved it came from a daemon-spawned module.
2417    ///
2418    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2419    /// accepted only for module ids the supervisor has spawned.
2420    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2421        if presented.is_empty() {
2422            return false;
2423        }
2424        let nonces = self
2425            .spawn_nonces
2426            .lock()
2427            .unwrap_or_else(|poisoned| poisoned.into_inner());
2428        let current = nonces
2429            .get(module_id)
2430            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2431        drop(nonces);
2432        // During a swap two processes of the module are alive, and a consumer
2433        // started by either one presents that process's nonce. Accepting only
2434        // the recorded one would fail the incumbent's consumers for the whole
2435        // overlap once cutover moves the record to the candidate.
2436        current || self.swap_nonce_matches(module_id, presented)
2437    }
2438
2439    /// Whether `presented` is either nonce of an open swap for `module_id`.
2440    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2441        let swaps = self
2442            .swaps
2443            .lock()
2444            .unwrap_or_else(|poisoned| poisoned.into_inner());
2445        swaps.get(module_id).is_some_and(|swap| {
2446            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2447                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2448                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2449                })
2450        })
2451    }
2452
2453    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2454    /// Called before the candidate process exists.
2455    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2456        let incumbent_nonce = self
2457            .spawn_nonces
2458            .lock()
2459            .unwrap_or_else(|poisoned| poisoned.into_inner())
2460            .get(module_id)
2461            .cloned();
2462        self.swaps
2463            .lock()
2464            .unwrap_or_else(|poisoned| poisoned.into_inner())
2465            .insert(
2466                module_id.to_string(),
2467                OpenSwap {
2468                    candidate_nonce,
2469                    incumbent_nonce,
2470                    candidate_admitted: false,
2471                },
2472            );
2473    }
2474
2475    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2476    /// the module's recorded one.
2477    pub(crate) fn close_swap(&self, module_id: &str) {
2478        self.swaps
2479            .lock()
2480            .unwrap_or_else(|poisoned| poisoned.into_inner())
2481            .remove(module_id);
2482    }
2483
2484    /// Install the observer told about swap promotions, replacing any earlier
2485    /// one.
2486    pub(crate) fn set_swap_promotion_observer(
2487        &self,
2488        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2489    ) {
2490        *self
2491            .promotion_observer
2492            .0
2493            .lock()
2494            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2495    }
2496
2497    /// Tell the installed observer, if it is still alive, that a swap promoted
2498    /// `registration`.
2499    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2500        let observer = self
2501            .promotion_observer
2502            .0
2503            .lock()
2504            .unwrap_or_else(|poisoned| poisoned.into_inner())
2505            .as_ref()
2506            .and_then(std::sync::Weak::upgrade);
2507        if let Some(observer) = observer {
2508            observer.swap_promoted(registration);
2509        }
2510    }
2511
2512    /// Whether a swap is open for `module_id`.
2513    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2514        self.swaps
2515            .lock()
2516            .unwrap_or_else(|poisoned| poisoned.into_inner())
2517            .contains_key(module_id)
2518    }
2519
2520    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2521    /// respawn would, once cutover has made the candidate the module's process.
2522    /// The swap stays open so the incumbent's nonce keeps attesting until the
2523    /// incumbent has drained and exited.
2524    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2525        let candidate_nonce = self
2526            .swaps
2527            .lock()
2528            .unwrap_or_else(|poisoned| poisoned.into_inner())
2529            .get(module_id)
2530            .map(|swap| swap.candidate_nonce.clone());
2531        let Some(nonce) = candidate_nonce else {
2532            return;
2533        };
2534        self.set_spawn_nonce(module_id, nonce.clone());
2535        if reserved {
2536            self.set_reserved_nonce(module_id, nonce);
2537        }
2538    }
2539
2540    /// The swap gate for a HELLO claiming `module_id`.
2541    ///
2542    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2543    /// presents the candidate nonce, which the reserved gate (holding the
2544    /// incumbent's nonce) would refuse as `reserved_module` before swap
2545    /// admission was ever reached. And it applies to unreserved ids too: for an
2546    /// unreserved id the only thing that ever stopped a second process claiming
2547    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2548    /// refusal a swap lifts for its candidate.
2549    ///
2550    /// The incumbent's own nonce falls through to the ordinary gates, which
2551    /// treat it as they always have (a live incumbent is refused as a
2552    /// duplicate). Anything else while a swap is open is refused, including an
2553    /// absent nonce.
2554    pub(crate) fn swap_hello_admission(
2555        &self,
2556        module_id: &str,
2557        presented: Option<&str>,
2558    ) -> SwapHelloAdmission {
2559        let swaps = self
2560            .swaps
2561            .lock()
2562            .unwrap_or_else(|poisoned| poisoned.into_inner());
2563        let Some(swap) = swaps.get(module_id) else {
2564            return SwapHelloAdmission::NotSwapping;
2565        };
2566        let Some(presented) = presented else {
2567            return SwapHelloAdmission::Refused;
2568        };
2569        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2570            return if swap.candidate_admitted {
2571                SwapHelloAdmission::Refused
2572            } else {
2573                SwapHelloAdmission::Candidate
2574            };
2575        }
2576        if swap
2577            .incumbent_nonce
2578            .as_deref()
2579            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2580        {
2581            return SwapHelloAdmission::NotSwapping;
2582        }
2583        SwapHelloAdmission::Refused
2584    }
2585
2586    /// Record that the swap token has registered a candidate, so it admits no
2587    /// second HELLO.
2588    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2589        if let Some(swap) = self
2590            .swaps
2591            .lock()
2592            .unwrap_or_else(|poisoned| poisoned.into_inner())
2593            .get_mut(module_id)
2594        {
2595            swap.candidate_admitted = true;
2596        }
2597    }
2598
2599    /// Test/support lookup for the current launch nonce of a supervised spawn.
2600    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2601        self.spawn_nonces
2602            .lock()
2603            .unwrap_or_else(|poisoned| poisoned.into_inner())
2604            .get(module_id)
2605            .cloned()
2606    }
2607
2608    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2609    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2610        self.reserved_nonces
2611            .lock()
2612            .unwrap_or_else(|poisoned| poisoned.into_inner())
2613            .get(module_id)
2614            .cloned()
2615            .flatten()
2616    }
2617
2618    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2619        // Normally already marked before the process was spawned; marking here
2620        // too keeps `configured_ids` a superset of the roster for any caller
2621        // that inserts a module directly.
2622        self.mark_configured(module.module_id());
2623        let mut modules = self
2624            .modules
2625            .lock()
2626            .unwrap_or_else(|poisoned| poisoned.into_inner());
2627        modules.insert(module.module_id().to_string(), module)
2628    }
2629
2630    /// Record that the supervisor has taken on `module_id`. Called before the
2631    /// module's first process is spawned, so that by the time that process can
2632    /// register, [`Self::is_configured`] already answers true.
2633    fn mark_configured(&self, module_id: &str) {
2634        self.configured_ids
2635            .lock()
2636            .unwrap_or_else(|poisoned| poisoned.into_inner())
2637            .insert(module_id.to_string());
2638    }
2639
2640    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2641    /// before it was ever put on the roster. A module already on the roster
2642    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2643    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2644        let modules = self
2645            .modules
2646            .lock()
2647            .unwrap_or_else(|poisoned| poisoned.into_inner());
2648        if !modules.contains_key(module_id) {
2649            self.configured_ids
2650                .lock()
2651                .unwrap_or_else(|poisoned| poisoned.into_inner())
2652                .remove(module_id);
2653        }
2654    }
2655
2656    /// Whether `module_id` is a module this daemon supervises: on the roster,
2657    /// or about to be (its process is being spawned right now).
2658    ///
2659    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2660    /// for scopes: a supervised module's process can register and sync before
2661    /// [`Self::get`] can return it, and in that window it is still a module
2662    /// that will sync, not one that never will.
2663    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2664        self.configured_ids
2665            .lock()
2666            .unwrap_or_else(|poisoned| poisoned.into_inner())
2667            .contains(module_id)
2668    }
2669
2670    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2671        let modules = self
2672            .modules
2673            .lock()
2674            .unwrap_or_else(|poisoned| poisoned.into_inner());
2675        modules.get(module_id).cloned()
2676    }
2677
2678    pub(crate) fn record_late_health_answer(
2679        &self,
2680        module_id: &str,
2681        latency_ms: u64,
2682    ) -> Result<bool, SuperviseError> {
2683        let Some(module) = self.get(module_id) else {
2684            return Ok(false);
2685        };
2686        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2687            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2688            state.health.last_late_answer_latency_ms = Some(latency_ms);
2689            // A late answer is an answer: the module served the probe, just past
2690            // the deadline. Leaving the miss streak in place while logging
2691            // "proves the module is alive" is how a CPU-starved module that
2692            // answers every probe a few seconds late still marches to the
2693            // threshold and gets killed — the exact kill class `NoAnswer` is
2694            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2695            // is degradation, and degradation reports; it does not restart.
2696            state.health.consecutive_failures = 0;
2697        })?;
2698        Ok(true)
2699    }
2700
2701    /// Arm the one-shot marker for the module process that this caller
2702    /// deliberately initiated severance against. Generic connection teardown
2703    /// must not call this:
2704    /// a surviving process would otherwise retain an exemption for a later
2705    /// genuine crash.
2706    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2707        let Some(module) = self.get(module_id) else {
2708            return Ok(false);
2709        };
2710        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2711        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2712            return Ok(false);
2713        };
2714        drop(snapshot);
2715        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2716    }
2717
2718    pub fn list(&self) -> Vec<SupervisedModule> {
2719        let modules = self
2720            .modules
2721            .lock()
2722            .unwrap_or_else(|poisoned| poisoned.into_inner());
2723        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2724        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2725        modules
2726    }
2727
2728    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2729        self.activity_images
2730            .lock()
2731            .unwrap_or_else(|p| p.into_inner())
2732            .remove(module_id);
2733        self.spawn_nonces
2734            .lock()
2735            .unwrap_or_else(|poisoned| poisoned.into_inner())
2736            .remove(module_id);
2737        self.close_swap(module_id);
2738        let mut reserved_nonces = self
2739            .reserved_nonces
2740            .lock()
2741            .unwrap_or_else(|poisoned| poisoned.into_inner());
2742        if reserved_nonces.contains_key(module_id) {
2743            // The old nonce must die with the removed process, but the exact-id
2744            // gate remains until an operator explicitly releases it.
2745            reserved_nonces.insert(module_id.to_string(), None);
2746        }
2747        drop(reserved_nonces);
2748        self.reserved_prefix_owners
2749            .lock()
2750            .unwrap_or_else(|poisoned| poisoned.into_inner())
2751            .retain(|_, owner| owner != module_id);
2752        let removed = self
2753            .modules
2754            .lock()
2755            .unwrap_or_else(|poisoned| poisoned.into_inner())
2756            .remove(module_id);
2757        self.configured_ids
2758            .lock()
2759            .unwrap_or_else(|poisoned| poisoned.into_inner())
2760            .remove(module_id);
2761        removed
2762    }
2763
2764    /// Remember a module removed by a non-preview rescan so route.open can
2765    /// distinguish that intentional removal from an unknown id.
2766    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2767        self.removal_tombstones
2768            .lock()
2769            .unwrap_or_else(|poisoned| poisoned.into_inner())
2770            .insert(module_id.to_string(), unix_ms_now());
2771    }
2772
2773    /// Return how long ago a rescan removed this module in milliseconds.
2774    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2775        self.removal_tombstones
2776            .lock()
2777            .unwrap_or_else(|poisoned| poisoned.into_inner())
2778            .get(module_id)
2779            .copied()
2780            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2781    }
2782
2783    /// Retire a reserved-id gate only after its module has left supervision.
2784    ///
2785    /// A retained gate has no live nonce (`None`), so releasing any other entry
2786    /// would weaken a currently configured or otherwise active reservation.
2787    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2788        if self.get(module_id).is_some() {
2789            return false;
2790        }
2791        let mut reserved_nonces = self
2792            .reserved_nonces
2793            .lock()
2794            .unwrap_or_else(|poisoned| poisoned.into_inner());
2795        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2796            return false;
2797        }
2798        reserved_nonces.remove(module_id);
2799        true
2800    }
2801
2802    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2803        Arc::clone(&self.operation_lock)
2804    }
2805}
2806
2807/// Process supervisor for subc-owned singleton modules.
2808#[derive(Debug, Clone)]
2809pub struct Supervisor {
2810    registry: Arc<Registry>,
2811    restart_policy: RestartPolicy,
2812    drain_timeout: Duration,
2813    connection_file_path: Option<PathBuf>,
2814    capture_logs_dir: Option<PathBuf>,
2815    forwarding: Option<Arc<ForwardingTable>>,
2816    process_liveness: Arc<SupervisorProcessLiveness>,
2817    supervisor_handle: Option<SupervisorHandle>,
2818    health: HealthConfig,
2819    daemon_start_clock: crate::clock::StartClock,
2820    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2821    spawn_events: SpawnEventFeed,
2822    provenance_probe: ExecutableIdentityProbe,
2823    /// Every process spawned through this supervisor (and its clones) and not
2824    /// yet reaped, so daemon shutdown can end them.
2825    child_roster: ChildRoster,
2826    #[cfg(target_os = "linux")]
2827    cgroup_placement: Option<subc_cgroup::Placement>,
2828    #[cfg(test)]
2829    test_after_first_spawn: AfterFirstSpawnHook,
2830}
2831
2832/// A roster entry whose monitor and first process have not started yet.
2833/// Keeping the command receiver here lets discovery serve status immediately
2834/// while start/stop requests wait until bootstrap has published the connection file.
2835pub(crate) struct RegisteredConfiguredModule {
2836    spec: ModuleSpec,
2837    runtime: SupervisorRuntimeConfig,
2838    module: SupervisedModule,
2839    commands: mpsc::Receiver<SupervisorCommand>,
2840}
2841
2842/// Test-only hook run on the path that takes on a new module, right after its
2843/// first `spawn_child` returns (the process exists and could already be
2844/// registering) and before that process is handed to the module's supervise
2845/// loop and put on the roster. Lets a test observe what a fast child would see
2846/// in that window without racing a real one.
2847#[cfg(test)]
2848#[derive(Clone, Default)]
2849struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2850
2851#[cfg(test)]
2852type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2853
2854#[cfg(test)]
2855impl fmt::Debug for AfterFirstSpawnHook {
2856    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2857        f.write_str("AfterFirstSpawnHook")
2858    }
2859}
2860
2861#[cfg(test)]
2862impl AfterFirstSpawnHook {
2863    fn run(&self, module_id: &str) {
2864        if let Some(hook) = &self.0 {
2865            hook(module_id);
2866        }
2867    }
2868}
2869
2870impl Supervisor {
2871    #[cfg(test)]
2872    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2873        let supervisor = Self::new(registry, policy);
2874        #[cfg(target_os = "macos")]
2875        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2876        supervisor
2877    }
2878    /// Verify the trampoline once when configured. Missing private OS support
2879    /// refuses every macOS launch by name but does not stop the daemon's control
2880    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2881    /// entry point; the library must not exec an arbitrary hosting program.
2882    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2883        let path = path.into();
2884        #[cfg(target_os = "macos")]
2885        {
2886            let result = probe_privacy_trampoline(&path).map(|()| path);
2887            if let Err(cause) = &result {
2888                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2889            }
2890            self.child_roster.set_privacy_trampoline(result);
2891        }
2892        #[cfg(not(target_os = "macos"))]
2893        let _ = path;
2894        self
2895    }
2896    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2897    /// capability-requirement log to read.
2898    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2899        self.child_roster.shutdown_flag()
2900    }
2901
2902    /// The first step of an announced daemon shutdown, before the notice and
2903    /// before any connection is closed.
2904    ///
2905    /// Sets the daemon-shutdown flag first: from here on no module is
2906    /// respawned (crash restart, operator restart, or swap), and every child
2907    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2908    /// the module exits on the EOF this shutdown gives it or is signalled by a
2909    /// service manager that kills the whole cgroup. Then writes the journal's
2910    /// shutdown marker, which records the instant and closes this daemon
2911    /// incarnation's stretch of the journal.
2912    #[cfg(unix)]
2913    pub(crate) fn begin_daemon_shutdown(&self) {
2914        self.child_roster.close();
2915        if let Some(journal) = &self.terminal_journal {
2916            journal.stamp_shutdown();
2917        }
2918    }
2919
2920    /// Announce a cut while established connections can still carry replies.
2921    /// These budgets bound the attempt to deliver notices and the wait for
2922    /// routes to drain; neither guarantees delivery or child completion, and
2923    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2924    #[cfg(unix)]
2925    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2926        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2927        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2928        let Some(forwarding) = &self.forwarding else {
2929            return Ok(());
2930        };
2931        let notice_started = Instant::now();
2932        let modules = forwarding
2933            .begin_daemon_drain()
2934            .map_err(SuperviseError::Forwarding)?;
2935        let deadline_ms =
2936            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2937        let shutdown_reason = self
2938            .supervisor_handle
2939            .as_ref()
2940            .map(SupervisorHandle::take_daemon_shutdown_reason)
2941            .unwrap_or(RouteCloseReason::Disable);
2942        let mut modules_told = 0;
2943        let mut notices = tokio::task::JoinSet::new();
2944        let mut drains = Vec::new();
2945        for connection in forwarding
2946            .module_connections()
2947            .map_err(SuperviseError::Forwarding)?
2948        {
2949            let target = ModuleDrainTarget {
2950                endpoint: connection.endpoint,
2951                sink: connection.sink,
2952                negotiated_ver: connection.negotiated_ver,
2953                abandoned_bindings: Vec::new(),
2954                excluded_subscriptions: 0,
2955            };
2956            notices.spawn(async move {
2957                (
2958                    true,
2959                    send_module_draining(
2960                        &connection.module_id,
2961                        shutdown_reason,
2962                        deadline_ms,
2963                        &target,
2964                        true,
2965                    )
2966                    .await,
2967                )
2968            });
2969        }
2970        for (module_id, endpoint) in modules {
2971            let Some(target) = forwarding
2972                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2973                .map_err(SuperviseError::Forwarding)?
2974            else {
2975                continue;
2976            };
2977            let routes = forwarding
2978                .endpoint_routes(target.endpoint)
2979                .map_err(SuperviseError::Forwarding)?;
2980            // Only an announced restart promises that modules are coming back.
2981            // Client route notices retain their existing reconnect semantics.
2982            let mut recipients = Vec::new();
2983            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2984            for route in routes {
2985                let client = route.goodbye_target;
2986                if let Some((_, channels)) = clients
2987                    .iter_mut()
2988                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2989                {
2990                    channels.push(client.channel);
2991                } else {
2992                    let channel = client.channel;
2993                    clients.push((client, vec![channel]));
2994                }
2995            }
2996            for (client, mut channels) in clients {
2997                channels.sort_unstable();
2998                channels.dedup();
2999                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3000                    module_id: module_id.clone(),
3001                    channels,
3002                    reason: RouteCloseReason::Restart,
3003                })
3004                .expect("route closing serializes");
3005                recipients.push((client.sink, client.negotiated_ver, closing));
3006            }
3007            for (sink, version, body) in recipients {
3008                notices.spawn(async move {
3009                    let frame = Frame::build_with_version(
3010                        version,
3011                        FrameType::Push,
3012                        control_flags(),
3013                        0,
3014                        0,
3015                        0,
3016                        body,
3017                    )
3018                    .expect("bounded lifecycle notice frame builds");
3019                    (false, sink.send_flushed(frame).await.is_ok())
3020                });
3021            }
3022            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3023            drains.push((module_id, target.endpoint, gauges));
3024        }
3025        // A quiet forwarding table is not proof that queued notices reached the
3026        // socket. Wait for writer flush acknowledgements before testing quiescence.
3027        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3028        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3029            match result {
3030                Ok((true, true)) => modules_told += 1,
3031                Ok((false, true)) => {}
3032                result => warn!(?result, "daemon shutdown notice delivery failed"),
3033            }
3034        }
3035        notices.abort_all();
3036        info!(
3037            ?shutdown_reason,
3038            modules_told, "daemon shutdown module notices delivered"
3039        );
3040        let notice_elapsed = notice_started.elapsed();
3041        let drain_started = Instant::now();
3042        let deadline = drain_started + DRAIN_BUDGET;
3043        let mut waits = tokio::task::JoinSet::new();
3044        for (module_id, endpoint, gauges) in drains {
3045            let forwarding = Arc::clone(forwarding);
3046            let mut runtime = self.runtime_config();
3047            runtime.health.cadence = Duration::from_millis(100);
3048            waits.spawn(async move {
3049                let wait_result = wait_for_forwarding_quiescence(
3050                    &forwarding,
3051                    &module_id,
3052                    &runtime,
3053                    endpoint,
3054                    deadline,
3055                    &gauges,
3056                    DrainScope::Endpoint(endpoint),
3057                    &mut DrainGaugeTrace::new(&gauges),
3058                )
3059                .await;
3060                // Store the measurement before the task returns: quiescence
3061                // may be reached just as the deadline stops the join loop from
3062                // collecting results. Unfinished or failed waits report false.
3063                forwarding
3064                    .record_daemon_drain_result(
3065                        endpoint,
3066                        drained_after_quiescence_wait(&wait_result),
3067                    )
3068                    .map_err(SuperviseError::Forwarding)?;
3069                wait_result
3070            });
3071        }
3072        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3073            if !matches!(result, Ok(Ok(true))) {
3074                warn!(?result, "daemon shutdown drain did not reach quiescence");
3075            }
3076        }
3077        // Measure only the forwarding drain, not child teardown or the caller's
3078        // signal delivery and process-reaping delays.
3079        info!(
3080            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3081            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3082            "daemon shutdown forwarding drain complete"
3083        );
3084        Ok(())
3085    }
3086
3087    /// The last step of a graceful daemon shutdown, after the notice and the
3088    /// drain: send every registered module a module GOODBYE, the same planned
3089    /// stop signal `ck module stop` gives, then close every connection so each
3090    /// subc module sees EOF and starts its own teardown, then end every
3091    /// supervised child that has not exited
3092    /// by its own deadline (its drain budget, capped). Modules lead their own
3093    /// process groups, so a
3094    /// service manager's group kill no longer reaches them; without this a
3095    /// child that does not stop on EOF (every `protocol: "none"` child, which
3096    /// has no connection) would outlive the daemon. Every wait is bounded (see
3097    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3098    #[cfg(unix)]
3099    pub(crate) async fn end_children_for_daemon_shutdown(
3100        &self,
3101        already_escalated: bool,
3102        escalate: impl std::future::Future<Output = ()>,
3103    ) {
3104        tokio::pin!(escalate);
3105        let mut escalated = already_escalated;
3106        if let Some(forwarding) = &self.forwarding {
3107            let reason = CloseReason::new(
3108                "daemon_shutdown",
3109                "the daemon is exiting after its shutdown notice and drain",
3110            );
3111            if escalated {
3112                // The operator asked to stop waiting: queue the GOODBYEs but
3113                // do not wait for them to be written.
3114                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3115            } else {
3116                tokio::select! {
3117                    biased;
3118                    _ = escalate.as_mut() => {
3119                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3120                        escalated = true;
3121                    }
3122                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3123                }
3124            }
3125            let closed = forwarding.close_all_connections(&reason);
3126            debug!(closed, "closed established connections for daemon shutdown");
3127        }
3128        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3129        // already completed and must not be polled again; the child shutdown
3130        // wait is told it is escalated and gets a future that never fires.
3131        let escalated_here = escalated && !already_escalated;
3132        let remaining_escalate = async move {
3133            if escalated_here {
3134                std::future::pending::<()>().await;
3135            } else {
3136                escalate.await;
3137            }
3138        };
3139        crate::child_roster::end_children_for_daemon_shutdown(
3140            &self.child_roster,
3141            escalated,
3142            remaining_escalate,
3143        )
3144        .await;
3145    }
3146
3147    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3148        Self {
3149            registry,
3150            restart_policy,
3151            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3152            connection_file_path: None,
3153            capture_logs_dir: None,
3154            forwarding: None,
3155            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3156            supervisor_handle: None,
3157            health: HealthConfig::default(),
3158            daemon_start_clock: crate::clock::StartClock::capture(),
3159            terminal_journal: None,
3160            spawn_events: SpawnEventFeed::default(),
3161            provenance_probe: ExecutableIdentityProbe::default(),
3162            child_roster: ChildRoster::default(),
3163            #[cfg(target_os = "linux")]
3164            cgroup_placement: None,
3165            #[cfg(test)]
3166            test_after_first_spawn: AfterFirstSpawnHook::default(),
3167        }
3168    }
3169
3170    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3171        self.drain_timeout = drain_timeout;
3172        self
3173    }
3174
3175    /// The drain budget a module gets when its config entry sets none.
3176    pub(crate) fn default_drain_timeout(&self) -> Duration {
3177        self.drain_timeout
3178    }
3179
3180    pub fn with_process_liveness(
3181        mut self,
3182        process_liveness: Arc<SupervisorProcessLiveness>,
3183    ) -> Self {
3184        self.process_liveness = process_liveness;
3185        self
3186    }
3187
3188    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3189        self.connection_file_path = Some(connection_file_path.into());
3190        self
3191    }
3192
3193    /// Enables daemon-owned capture files for supervised stdout and stderr.
3194    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3195        self.capture_logs_dir = Some(logs_dir.into());
3196        self
3197    }
3198
3199    /// Names this daemon lifetime in spawn events, independently of whether a
3200    /// terminal journal is configured.
3201    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3202        // A millisecond start stamp can repeat after clock rollback or a rapid
3203        // restart. Use the connection file's random daemon_id instead: it already
3204        // identifies this daemon lifetime independently of the wall clock.
3205        self.spawn_events.configure_incarnation(daemon_incarnation);
3206        self
3207    }
3208
3209    /// Enables best-effort history shared by every supervised module. Without
3210    /// it, terminal history is kept only in each module's in-memory ring.
3211    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3212        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3213        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3214            path,
3215            daemon_incarnation,
3216        )));
3217        this
3218    }
3219
3220    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3221        self.child_roster.counters = forwarding.counters();
3222        self.forwarding = Some(forwarding);
3223        self
3224    }
3225
3226    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3227        self.spawn_events = supervisor_handle.spawn_events.clone();
3228        self.supervisor_handle = Some(supervisor_handle);
3229        self
3230    }
3231
3232    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3233        self.health = health;
3234        self
3235    }
3236
3237    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3238    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3239    /// record is kept.
3240    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3241        self.child_roster.record_to(path.into());
3242        self
3243    }
3244
3245    #[cfg(target_os = "linux")]
3246    pub fn with_cgroup_placement(
3247        mut self,
3248        cgroup_placement: Option<subc_cgroup::Placement>,
3249    ) -> Self {
3250        self.child_roster
3251            .set_cgroup_placement(cgroup_placement.clone());
3252        self.cgroup_placement = cgroup_placement;
3253        self
3254    }
3255
3256    /// Spawn `spec.program` and start monitoring it.
3257    ///
3258    /// The child is expected to parse `--subc <connection-file-path>`, read the
3259    /// TCP+key connection file, authenticate to the already-running listener, and
3260    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3261    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3262        validate_spec(&spec)?;
3263        self.establish_identity(&spec);
3264
3265        let runtime = self.runtime_config();
3266        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3267        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3268        let registration_watch = watch_registration(&self.registry, &spec);
3269        let spawned = spawn_child(
3270            &spec,
3271            runtime.connection_file_path.as_deref(),
3272            self.supervisor_handle.as_ref(),
3273            &runtime.stderr_ring,
3274            runtime.capture_logs_dir.as_deref(),
3275            &runtime.child_roster,
3276            #[cfg(target_os = "linux")]
3277            runtime.cgroup_placement.as_ref(),
3278        );
3279        #[cfg(test)]
3280        self.test_after_first_spawn.run(&spec.module_id);
3281        let mut child = match spawned {
3282            Ok(child) => child,
3283            Err(err) => {
3284                // Unlike the configured paths, a failed `spawn` leaves nothing
3285                // on the roster, so the module must not stay marked configured.
3286                self.abandon_unrostered(&spec.module_id);
3287                return Err(err);
3288            }
3289        };
3290        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3291        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3292        if let Some(spawn) = activity_spawn {
3293            spawn.complete(&snapshot);
3294        }
3295
3296        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3297    }
3298
3299    /// Make `spec`'s module count as configured, with its identity gates
3300    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3301    /// exists.
3302    ///
3303    /// Every path that takes on a new module calls this before `spawn_child`.
3304    /// The order is the point: the child can connect, register, sync its
3305    /// scopes and ask about them as soon as it is spawned, and the module is
3306    /// only put on the roster after `spawn_child` returns. Were the mark set
3307    /// with the roster entry, a fast child would see its own owner reported
3308    /// as not configured, and a scoped `route.open` in that window would be
3309    /// refused as terminal `scope_not_live` ("will never sync") instead of
3310    /// retryable `scope_not_synced`.
3311    fn establish_identity(&self, spec: &ModuleSpec) {
3312        if let Some(supervisor_handle) = &self.supervisor_handle {
3313            supervisor_handle.apply_identity_configuration(spec);
3314            supervisor_handle.mark_configured(&spec.module_id);
3315        }
3316    }
3317
3318    /// Take back [`Self::establish_identity`]'s configured mark when the
3319    /// module will not be put on the roster after all.
3320    fn abandon_unrostered(&self, module_id: &str) {
3321        if let Some(supervisor_handle) = &self.supervisor_handle {
3322            supervisor_handle.unmark_configured_unless_rostered(module_id);
3323        }
3324    }
3325
3326    /// Record a freshly spawned first process as running. On failure the
3327    /// module never reaches the roster, so its configured mark is taken back.
3328    fn mark_first_process_running(
3329        &self,
3330        spec: &ModuleSpec,
3331        runtime: &SupervisorRuntimeConfig,
3332        snapshot: &SharedSnapshot,
3333        child: &mut SupervisedChild,
3334    ) -> Result<(), SuperviseError> {
3335        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3336            containment::terminate_blocking(&runtime.child_roster.containment);
3337            self.abandon_unrostered(&spec.module_id);
3338            return Err(err);
3339        }
3340        self.process_liveness
3341            .track(spec.module_id.clone(), Arc::clone(snapshot));
3342        Ok(())
3343    }
3344
3345    /// Start supervising a module declared in daemon configuration.
3346    ///
3347    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3348    /// failures in the supervisor handle so operator-facing `supervisor.list`
3349    /// reflects every configured module while daemon startup continues.
3350    pub fn supervise_configured(
3351        &self,
3352        spec: ModuleSpec,
3353        enabled: bool,
3354    ) -> Result<SupervisedModule, SuperviseError> {
3355        self.supervise_configured_requested(
3356            spec,
3357            enabled,
3358            self.health.clone(),
3359            None,
3360            self.restart_policy,
3361            DEFAULT_REGISTER_TIMEOUT,
3362            None,
3363        )
3364    }
3365
3366    /// Supervise a configured module with its own health, drain, and crash
3367    /// budget. The restart policy is per-module because the config file is:
3368    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3369    /// module that is expensive to restart should not be forced onto the same
3370    /// budget as one that is cheap.
3371    pub fn supervise_configured_with_health(
3372        &self,
3373        spec: ModuleSpec,
3374        enabled: bool,
3375        health: HealthConfig,
3376        drain_timeout_ms: Option<u64>,
3377        restart_policy: RestartPolicy,
3378    ) -> Result<SupervisedModule, SuperviseError> {
3379        self.supervise_configured_requested(
3380            spec,
3381            enabled,
3382            health,
3383            drain_timeout_ms,
3384            restart_policy,
3385            DEFAULT_REGISTER_TIMEOUT,
3386            None,
3387        )
3388    }
3389
3390    #[allow(clippy::too_many_arguments)]
3391    pub(crate) fn supervise_configured_requested(
3392        &self,
3393        spec: ModuleSpec,
3394        enabled: bool,
3395        health: HealthConfig,
3396        drain_timeout_ms: Option<u64>,
3397        restart_policy: RestartPolicy,
3398        register_timeout: Duration,
3399        requester: Option<Principal>,
3400    ) -> Result<SupervisedModule, SuperviseError> {
3401        let registered = self.register_configured_with_health(
3402            spec,
3403            enabled,
3404            health,
3405            drain_timeout_ms,
3406            restart_policy,
3407            register_timeout,
3408        )?;
3409        let daemon_initiated = requester.is_none();
3410        self.start_registered_configured(registered, requester, daemon_initiated)
3411    }
3412
3413    /// Record a configured module as starting or disabled, without launching a
3414    /// process or running its monitor. Bootstrap registers the entire roster
3415    /// before publishing discovery, then activates it once children can read
3416    /// the connection file passed in their `--subc` argument.
3417    pub(crate) fn register_configured_with_health(
3418        &self,
3419        spec: ModuleSpec,
3420        enabled: bool,
3421        health: HealthConfig,
3422        drain_timeout_ms: Option<u64>,
3423        restart_policy: RestartPolicy,
3424        register_timeout: Duration,
3425    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3426        validate_spec(&spec)?;
3427        self.establish_identity(&spec);
3428
3429        let mut runtime = self.runtime_config();
3430        runtime.health = health;
3431        runtime.restart_policy = restart_policy;
3432        runtime.register_timeout = register_timeout;
3433        if let Some(ms) = drain_timeout_ms {
3434            runtime.drain_timeout = Duration::from_millis(ms);
3435            *runtime
3436                .effective_drain_timeout
3437                .lock()
3438                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3439        }
3440        let snapshot = if enabled {
3441            SupervisorSnapshot::starting()
3442        } else {
3443            SupervisorSnapshot::disabled()
3444        };
3445        let snapshot = {
3446            let mut snapshot = snapshot;
3447            if enabled {
3448                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3449                {
3450                    snapshot.waiting_for_previous_pid = Some(pid);
3451                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3452                }
3453            }
3454            snapshot
3455        };
3456        let (module, commands) =
3457            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3458        Ok(RegisteredConfiguredModule {
3459            spec,
3460            runtime,
3461            module,
3462            commands,
3463        })
3464    }
3465
3466    /// Activate a previously registered module. Queued control requests are
3467    /// served by the monitor after the initial configured spawn has settled.
3468    pub(crate) fn start_registered_configured(
3469        &self,
3470        registered: RegisteredConfiguredModule,
3471        requester: Option<Principal>,
3472        daemon_initiated: bool,
3473    ) -> Result<SupervisedModule, SuperviseError> {
3474        let RegisteredConfiguredModule {
3475            spec,
3476            mut runtime,
3477            module,
3478            commands,
3479        } = registered;
3480        runtime.start_requester = requester;
3481        runtime.recovery_is_daemon = daemon_initiated;
3482        let snapshot = &module.inner.snapshot;
3483        let should_spawn = {
3484            let state = lock_snapshot(snapshot)?;
3485            state.enabled && state.waiting_for_previous_pid.is_none()
3486        };
3487        let child = if should_spawn {
3488            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3489            let registration_watch = watch_registration(&self.registry, &spec);
3490            let spawned = spawn_child(
3491                &spec,
3492                runtime.connection_file_path.as_deref(),
3493                self.supervisor_handle.as_ref(),
3494                &runtime.stderr_ring,
3495                runtime.capture_logs_dir.as_deref(),
3496                &runtime.child_roster,
3497                #[cfg(target_os = "linux")]
3498                runtime.cgroup_placement.as_ref(),
3499            );
3500            #[cfg(test)]
3501            self.test_after_first_spawn.run(&spec.module_id);
3502            match spawned {
3503                Ok(mut child) => {
3504                    arm_registration_deadline(
3505                        &mut child,
3506                        runtime.register_timeout,
3507                        registration_watch,
3508                    );
3509                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3510                    if let Some(spawn) = activity_spawn {
3511                        spawn.complete(snapshot);
3512                    }
3513                    Some(child)
3514                }
3515                Err(err) => {
3516                    if runtime.health.critical {
3517                        error!(
3518                            module_id = %spec.module_id,
3519                            program = %spec.program.display(),
3520                            error = %err,
3521                            "critical configured module failed to spawn; marking failed and alerting"
3522                        );
3523                    } else {
3524                        error!(
3525                            module_id = %spec.module_id,
3526                            program = %spec.program.display(),
3527                            error = %err,
3528                            "configured module failed to spawn; marking failed and continuing"
3529                        );
3530                    }
3531                    fail_snapshot(
3532                        snapshot,
3533                        Some(&spec.module_id),
3534                        None,
3535                        &runtime.failure_context(daemon_initiated),
3536                    );
3537                    None
3538                }
3539            }
3540        } else {
3541            None
3542        };
3543        self.start_monitor(spec, runtime, &module, child, commands);
3544        Ok(module)
3545    }
3546
3547    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3548        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3549        SupervisorRuntimeConfig {
3550            registry: self.registry.clone(),
3551            recovery_is_daemon: true,
3552            start_requester: None,
3553            pending_activity_restart: false,
3554            provenance_probe: self.provenance_probe.clone(),
3555            scheduled_respawn: Arc::default(),
3556            deferred_reload_reply: Arc::default(),
3557            restart_policy: self.restart_policy,
3558            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3559            drain_timeout: self.drain_timeout,
3560            // Shared with this module's roster copy: daemon shutdown waits on
3561            // each child for the module's own drain budget, as resolved now.
3562            child_roster: self
3563                .child_roster
3564                .for_module(Arc::clone(&effective_drain_timeout)),
3565            effective_drain_timeout,
3566            default_drain_timeout: self.drain_timeout,
3567            health: self.health.clone(),
3568            connection_file_path: self.connection_file_path.clone(),
3569            capture_logs_dir: self.capture_logs_dir.clone(),
3570            forwarding: self.forwarding.clone(),
3571            supervisor_handle: self.supervisor_handle.clone(),
3572            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3573            terminal_ring: Arc::new(Mutex::new(
3574                TerminalRing::new(
3575                    TerminalRingConfig::default(),
3576                    self.daemon_start_clock.started_at_ms(),
3577                )
3578                .with_start_clock(self.daemon_start_clock)
3579                .with_journal(self.terminal_journal.clone())
3580                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3581            )),
3582            spawn_events: self.spawn_events.clone(),
3583            #[cfg(target_os = "linux")]
3584            cgroup_placement: self.cgroup_placement.clone(),
3585            #[cfg(test)]
3586            test_seed_stale_facts_before_enable_spawn: false,
3587            #[cfg(test)]
3588            test_reload_exit_record_gate: None,
3589        }
3590    }
3591
3592    fn supervised_module(
3593        &self,
3594        spec: ModuleSpec,
3595        runtime: SupervisorRuntimeConfig,
3596        snapshot: SharedSnapshot,
3597        child: Option<SupervisedChild>,
3598    ) -> SupervisedModule {
3599        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3600        self.start_monitor(spec, runtime, &module, child, commands);
3601        module
3602    }
3603
3604    fn register_module(
3605        &self,
3606        spec: &ModuleSpec,
3607        runtime: &SupervisorRuntimeConfig,
3608        snapshot: SharedSnapshot,
3609    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3610        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3611            spec: spec.clone(),
3612            health: runtime.health.clone(),
3613        }));
3614        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3615        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3616        // The module's OWN policy, which may be its per-module config rather than
3617        // the supervisor-wide one; status must report the budget the supervise
3618        // loop actually enforces.
3619        let restart_policy = runtime.restart_policy;
3620        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3621        let (tx, rx) = mpsc::channel(4);
3622        let module_id = spec.module_id.clone();
3623        let module = SupervisedModule {
3624            inner: Arc::new(SupervisedModuleInner {
3625                module_id: module_id.clone(),
3626                registry: Arc::clone(&self.registry),
3627                snapshot,
3628                configuration,
3629                stderr_ring,
3630                terminal_ring,
3631                commands: tx,
3632                monitor: Mutex::new(None),
3633                restart_policy,
3634                effective_drain_timeout,
3635                provenance_probe: self.provenance_probe.clone(),
3636                retained: runtime.child_roster.retained_view(),
3637            }),
3638        };
3639        // Identity gates and the configured mark must precede the roster entry,
3640        // so a client never sees a module without its admission policy.
3641        if let Some(supervisor_handle) = &self.supervisor_handle {
3642            supervisor_handle.insert(module.clone());
3643        }
3644        (module, rx)
3645    }
3646
3647    fn start_monitor(
3648        &self,
3649        spec: ModuleSpec,
3650        runtime: SupervisorRuntimeConfig,
3651        module: &SupervisedModule,
3652        child: Option<SupervisedChild>,
3653        commands: mpsc::Receiver<SupervisorCommand>,
3654    ) {
3655        let monitor = tokio::spawn(supervise_loop(
3656            spec,
3657            runtime,
3658            Arc::clone(&self.registry),
3659            Arc::clone(&self.process_liveness),
3660            Arc::clone(&module.inner.snapshot),
3661            child,
3662            commands,
3663        ));
3664        *module
3665            .inner
3666            .monitor
3667            .lock()
3668            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3669    }
3670}
3671
3672impl Default for Supervisor {
3673    fn default() -> Self {
3674        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3675    }
3676}
3677
3678/// Handle to one supervised child process.
3679#[derive(Clone)]
3680pub struct SupervisedModule {
3681    inner: Arc<SupervisedModuleInner>,
3682}
3683
3684struct SupervisedModuleInner {
3685    retained: crate::child_roster::RetainedView,
3686    module_id: String,
3687    registry: Arc<Registry>,
3688    snapshot: SharedSnapshot,
3689    configuration: Arc<Mutex<SupervisedConfiguration>>,
3690    stderr_ring: Arc<Mutex<StderrRing>>,
3691    terminal_ring: Arc<Mutex<TerminalRing>>,
3692    commands: mpsc::Sender<SupervisorCommand>,
3693    monitor: Mutex<Option<JoinHandle<()>>>,
3694    /// Copied from the supervisor's runtime config at spawn so `status()` can
3695    /// report the restart budget without reaching back into the supervisor. The
3696    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3697    restart_policy: RestartPolicy,
3698    effective_drain_timeout: Arc<Mutex<Duration>>,
3699    provenance_probe: ExecutableIdentityProbe,
3700}
3701
3702impl fmt::Debug for SupervisedModule {
3703    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3704        f.debug_struct("SupervisedModule")
3705            .field("module_id", &self.inner.module_id)
3706            .field("status", &self.status())
3707            .finish_non_exhaustive()
3708    }
3709}
3710
3711impl SupervisedModule {
3712    pub fn module_id(&self) -> &str {
3713        &self.inner.module_id
3714    }
3715
3716    /// Test-only: put one probe miss on the streak, the way
3717    /// `handle_health_probe_failure` does, so tests can assert what a later
3718    /// event does to the streak without driving the whole probe loop.
3719    #[cfg(test)]
3720    pub(crate) fn record_health_probe_failure_for_test(
3721        &self,
3722        detail: &str,
3723    ) -> Result<(), SuperviseError> {
3724        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3725            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3726            state.health.detail = Some(detail.to_string());
3727        })
3728    }
3729
3730    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3731        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3732    }
3733
3734    pub(crate) fn registration_end_reason(
3735        &self,
3736    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3737        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3738        Ok(match snapshot.state {
3739            ModuleState::Draining if snapshot.draining_to_replace => {
3740                Some(RegistrationEndReason::SupervisorRestart)
3741            }
3742            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3743            ModuleState::Restarting if snapshot.draining_to_replace => {
3744                Some(RegistrationEndReason::SupervisorRestart)
3745            }
3746            _ => None,
3747        })
3748    }
3749
3750    /// The module's retained stderr, newest lines last.
3751    ///
3752    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3753    /// module, `supervisor.list` renders every module, and putting it in the
3754    /// shared snapshot would make each status read carry a payload almost nobody
3755    /// asked for. Callers that want the text ask for it.
3756    pub fn stderr_tail(
3757        &self,
3758        max_lines: Option<usize>,
3759        max_bytes: Option<usize>,
3760    ) -> StderrTailSnapshot {
3761        self.inner
3762            .stderr_ring
3763            .lock()
3764            .unwrap_or_else(|poisoned| poisoned.into_inner())
3765            .snapshot(max_lines, max_bytes)
3766    }
3767
3768    /// The module's bounded terminal history, oldest retained exit first.
3769    ///
3770    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3771    /// daemon whose in-memory history was necessarily reset.
3772    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3773        self.inner
3774            .terminal_ring
3775            .lock()
3776            .unwrap_or_else(|poisoned| poisoned.into_inner())
3777            .snapshot()
3778    }
3779
3780    /// Retained observations from the current ring and all journal generations.
3781    ///
3782    /// Blocking: this reads the journal files. Async callers use
3783    /// `Self::read_durable_terminal_history`.
3784    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3785        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3786    }
3787
3788    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3789    /// read (up to every retained generation) never occupies a runtime worker.
3790    /// Fails only if the blocking task could not finish (runtime shutdown or a
3791    /// panic in the read).
3792    pub(crate) async fn read_durable_terminal_history(
3793        &self,
3794    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3795        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3796        let module_id = self.inner.module_id.clone();
3797        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3798            .await
3799    }
3800
3801    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3802        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3803            .map(|(status, _)| status)
3804    }
3805
3806    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3807        let state = lock_snapshot(&self.inner.snapshot)?;
3808        Ok(state
3809            .failed_retry
3810            .pending
3811            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3812            .map(|retry| {
3813                unix_ms_now().saturating_add(
3814                    u64::try_from(
3815                        retry
3816                            .deadline
3817                            .saturating_duration_since(Instant::now())
3818                            .as_millis(),
3819                    )
3820                    .unwrap_or(u64::MAX),
3821                )
3822            }))
3823    }
3824
3825    pub(crate) fn record_deliberate_severance(
3826        &self,
3827        identity: ProcessIdentity,
3828    ) -> Result<bool, SuperviseError> {
3829        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3830        if snapshot.pid != Some(identity.pid)
3831            || snapshot.process_start_time != Some(identity.start_time)
3832        {
3833            return Ok(false);
3834        }
3835        snapshot.deliberate_severance = Some(identity);
3836        Ok(true)
3837    }
3838
3839    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3840    ///
3841    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3842    /// does not produce reader-observability logs.
3843    pub(crate) fn status_for_control(
3844        &self,
3845        caller: &'static str,
3846    ) -> Result<ModuleStatus, SuperviseError> {
3847        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3848            .map(|(status, _)| status)
3849    }
3850
3851    fn status_with_snapshot_lock(
3852        &self,
3853        snapshot: &SharedSnapshot,
3854        caller: Option<&'static str>,
3855    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3856        let mut guard = match caller {
3857            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3858            None => lock_snapshot(snapshot)?,
3859        };
3860        // Read the budget through the pruning path so a reader sees the same
3861        // in-window count the restart decision would use, not a stale total.
3862        let restart_count =
3863            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3864        let snapshot = guard.clone();
3865        drop(guard);
3866        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3867            SuperviseError::StatePoisoned {
3868                module_id: Some(self.inner.module_id.clone()),
3869            }
3870        })?;
3871        let registration_active = self
3872            .inner
3873            .registry
3874            .get_module(&self.inner.module_id)
3875            .map_err(SuperviseError::Registry)?
3876            .is_some();
3877        let protocol = snapshot
3878            .spawned_protocol
3879            .unwrap_or(self.declared_protocol()?);
3880        let running_process =
3881            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3882        // Registration is the difference between the two protocols and the only
3883        // one: a subc module that has not registered cannot serve a request even
3884        // though its process is up, and a `none` module never registers at all,
3885        // so requiring it there would pin `live` to false for the whole life of
3886        // a perfectly healthy process.
3887        let live = match protocol {
3888            ModuleProtocol::Subc => running_process && registration_active,
3889            ModuleProtocol::None => running_process,
3890        };
3891        // Read only the retained cache: a status read must not resend signals,
3892        // rewrite the record, or keep the containment resource alive.
3893        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3894        let carry_configured = self
3895            .inner
3896            .configuration
3897            .lock()
3898            .map_err(|_| SuperviseError::StatePoisoned {
3899                module_id: Some(self.inner.module_id.clone()),
3900            })?
3901            .spec
3902            .carry_containment;
3903        let recovery_block = carries_containment(
3904            carry_configured,
3905            self.inner
3906                .retained
3907                .has_carried_container(&self.inner.module_id),
3908        )
3909        .then(|| self.inner.retained.recovery_block())
3910        .flatten();
3911        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3912        let pid = snapshot.reported_pid();
3913        let mut health = snapshot.health;
3914        if let Some(detail) = recovery_block
3915            .as_ref()
3916            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3917        {
3918            health.detail = Some(match health.detail {
3919                Some(previous) => format!("{previous}; {detail}"),
3920                None => detail.clone(),
3921            });
3922        }
3923
3924        Ok((
3925            ModuleStatus {
3926                waiting_for_previous_pid: retained
3927                    .as_ref()
3928                    .map(|(pid, _)| *pid)
3929                    .filter(|pid| *pid > 0)
3930                    .or(snapshot.waiting_for_previous_pid),
3931                waiting_for_previous_detail: recovery_block
3932                    .or_else(|| retained.map(|(_, detail)| detail))
3933                    .or(snapshot.waiting_for_previous_detail),
3934                module_id: self.inner.module_id.clone(),
3935                launch_nonce_source,
3936                state: snapshot.state,
3937                enabled: snapshot.enabled,
3938                process_alive: snapshot.process_alive,
3939                registration_active,
3940                protocol,
3941                live,
3942                restart_count,
3943                lifetime_restarts: snapshot.lifetime_restarts,
3944                spawn_generation: snapshot.spawn_generation,
3945                max_restarts: self.inner.restart_policy.max_restarts,
3946                restart_window: self.inner.restart_policy.window,
3947                drain_timeout,
3948                restart_backoff: self.inner.restart_policy.backoff,
3949                restart_max_backoff: self.inner.restart_policy.max_backoff,
3950                pid,
3951                spawned_at_ms: snapshot.spawned_at_ms,
3952                spawned_from: snapshot.spawned_from,
3953                launch_spec_digest: snapshot.launch_spec_digest,
3954                process_start_time: snapshot.process_start_time,
3955                last_exit: snapshot.last_exit,
3956                health,
3957            },
3958            snapshot.spawned_file_identity.clone(),
3959        ))
3960    }
3961
3962    #[cfg(test)]
3963    pub(crate) fn hold_snapshot_for_test(
3964        &self,
3965        acquired: std::sync::mpsc::Sender<()>,
3966        hold: Duration,
3967    ) -> std::thread::JoinHandle<()> {
3968        let snapshot = Arc::clone(&self.inner.snapshot);
3969        std::thread::spawn(move || {
3970            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3971            acquired
3972                .send(())
3973                .expect("test receiver waits for snapshot lock");
3974            std::thread::sleep(hold);
3975        })
3976    }
3977
3978    /// The status and the running-image check for `supervisor.provenance`,
3979    /// taken from one status read. The exec acknowledgement can land between
3980    /// two separate reads, and the reply would then pair "no pid yet" with an
3981    /// image observed after the module started, which describes no single
3982    /// moment.
3983    pub(crate) async fn status_and_running_image_agreement(
3984        &self,
3985    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3986        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3987        let image = self
3988            .inner
3989            .provenance_probe
3990            .observe(
3991                status.pid,
3992                status.spawned_from.as_deref(),
3993                identity,
3994                status.process_start_time,
3995            )
3996            .await;
3997        Ok((status, image))
3998    }
3999
4000    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4001        let snapshot = match lock_snapshot(&self.inner.snapshot) {
4002            Ok(snapshot) => snapshot.clone(),
4003            Err(_) => {
4004                return subc_control::RunningImageAgreement::Unavailable {
4005                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
4006                };
4007            }
4008        };
4009        self.inner
4010            .provenance_probe
4011            .observe(
4012                snapshot.reported_pid(),
4013                snapshot.spawned_from.as_deref(),
4014                snapshot.spawned_file_identity.clone(),
4015                snapshot.process_start_time,
4016            )
4017            .await
4018    }
4019
4020    /// Memory and CPU time of the module's current process, read now. Only the
4021    /// process the supervisor spawned is read, not processes it has started.
4022    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4023        #[cfg(windows)]
4024        {
4025            let image = lock_snapshot(&self.inner.snapshot)
4026                .ok()
4027                .and_then(|snapshot| {
4028                    snapshot
4029                        .spawned_file_identity
4030                        .as_ref()
4031                        .and_then(|identity| identity.windows_image.clone())
4032                });
4033            match image {
4034                Some(image) => {
4035                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
4036                }
4037                None => subc_control::ChildResourceUsage::Unavailable {
4038                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4039                },
4040            }
4041        }
4042        #[cfg(not(windows))]
4043        {
4044            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4045                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4046                Err(_) => {
4047                    return subc_control::ChildResourceUsage::Unavailable {
4048                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4049                    }
4050                }
4051            };
4052            crate::child_resources::read(pid, start_time)
4053        }
4054    }
4055
4056    #[cfg(all(test, any(target_os = "linux", windows)))]
4057    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4058        self.inner
4059            .snapshot
4060            .try_lock()
4061            .map(|snapshot| snapshot.subtree_target.clone())
4062            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4063    }
4064
4065    #[cfg(test)]
4066    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4067        self.inner.snapshot.lock().unwrap().subtree_target = target;
4068    }
4069
4070    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4071        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4072        Ok(match snapshot.state {
4073            ModuleState::Restarting => true,
4074            ModuleState::Failed | ModuleState::Disabled => false,
4075            _ if snapshot.enabled && snapshot.respawn_pending => true,
4076            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4077        })
4078    }
4079
4080    #[cfg(test)]
4081    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4082        self.is_warming_with_snapshot_lock(None)
4083    }
4084
4085    pub(crate) fn is_warming_for_control(
4086        &self,
4087        caller: &'static str,
4088    ) -> Result<bool, SuperviseError> {
4089        self.is_warming_with_snapshot_lock(Some(caller))
4090    }
4091
4092    fn is_warming_with_snapshot_lock(
4093        &self,
4094        caller: Option<&'static str>,
4095    ) -> Result<bool, SuperviseError> {
4096        let snapshot = match caller {
4097            Some(caller) => {
4098                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4099            }
4100            None => lock_snapshot(&self.inner.snapshot)?,
4101        }
4102        .clone();
4103        Ok(matches!(
4104            snapshot.state,
4105            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4106        ))
4107    }
4108
4109    /// Drain the module and stop monitoring it.
4110    pub async fn drain(&self) -> Result<(), SuperviseError> {
4111        self.stop().await
4112    }
4113
4114    pub(crate) async fn retire_requested(
4115        &self,
4116        requester: Option<Principal>,
4117    ) -> Result<(), SuperviseError> {
4118        match self.state()? {
4119            ModuleState::Stopped => return Ok(()),
4120            ModuleState::Failed
4121                if lock_snapshot(&self.inner.snapshot)?
4122                    .failed_retry
4123                    .pending
4124                    .is_none() =>
4125            {
4126                return Ok(())
4127            }
4128            ModuleState::Failed
4129            | ModuleState::Starting
4130            | ModuleState::Running
4131            | ModuleState::Unresponsive
4132            | ModuleState::Restarting
4133            | ModuleState::Draining
4134            | ModuleState::Disabled => {}
4135        }
4136
4137        let (reply_tx, reply_rx) = oneshot::channel();
4138        self.inner
4139            .commands
4140            .send(SupervisorCommand::Retire {
4141                requester,
4142                reply: reply_tx,
4143            })
4144            .await
4145            .map_err(|_| SuperviseError::CommandClosed {
4146                module_id: self.inner.module_id.clone(),
4147            })?;
4148        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4149            module_id: self.inner.module_id.clone(),
4150        })?
4151    }
4152
4153    pub async fn stop(&self) -> Result<(), SuperviseError> {
4154        self.stop_requested(None).await
4155    }
4156
4157    pub(crate) async fn stop_requested(
4158        &self,
4159        requester: Option<Principal>,
4160    ) -> Result<(), SuperviseError> {
4161        match self.state()? {
4162            ModuleState::Stopped => return Ok(()),
4163            ModuleState::Failed
4164                if lock_snapshot(&self.inner.snapshot)?
4165                    .failed_retry
4166                    .pending
4167                    .is_none() =>
4168            {
4169                return Ok(())
4170            }
4171            ModuleState::Failed
4172            | ModuleState::Starting
4173            | ModuleState::Running
4174            | ModuleState::Unresponsive
4175            | ModuleState::Restarting
4176            | ModuleState::Draining
4177            | ModuleState::Disabled => {}
4178        }
4179
4180        let (reply_tx, reply_rx) = oneshot::channel();
4181        self.inner
4182            .commands
4183            .send(SupervisorCommand::Drain {
4184                requester,
4185                reply: reply_tx,
4186            })
4187            .await
4188            .map_err(|_| SuperviseError::CommandClosed {
4189                module_id: self.inner.module_id.clone(),
4190            })?;
4191        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4192            module_id: self.inner.module_id.clone(),
4193        })?
4194    }
4195
4196    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4197        self.restart_requested(drain_timeout_ms, None).await
4198    }
4199
4200    pub(crate) async fn restart_requested(
4201        &self,
4202        drain_timeout_ms: Option<u64>,
4203        requester: Option<Principal>,
4204    ) -> Result<(), SuperviseError> {
4205        let queued_at = Instant::now();
4206        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4207        let (reply_tx, reply_rx) = oneshot::channel();
4208        self.inner
4209            .commands
4210            .send(SupervisorCommand::Restart {
4211                requester,
4212                drain_timeout_ms,
4213                received_at_generation,
4214                queued_at,
4215                reply: reply_tx,
4216            })
4217            .await
4218            .map_err(|_| SuperviseError::CommandClosed {
4219                module_id: self.inner.module_id.clone(),
4220            })?;
4221        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4222            module_id: self.inner.module_id.clone(),
4223        })?
4224    }
4225
4226    /// Blue/green restart: see `SupervisorCommand::Swap` and the
4227    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4228    /// process then drains in the background of the supervise loop) or has
4229    /// failed, leaving the old process serving.
4230    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4231        self.swap_requested(ready_timeout, None).await
4232    }
4233
4234    pub(crate) async fn swap_requested(
4235        &self,
4236        ready_timeout: Option<Duration>,
4237        requester: Option<Principal>,
4238    ) -> Result<(), SuperviseError> {
4239        let (reply_tx, reply_rx) = oneshot::channel();
4240        self.inner
4241            .commands
4242            .send(SupervisorCommand::Swap {
4243                requester,
4244                ready_timeout,
4245                reply: reply_tx,
4246            })
4247            .await
4248            .map_err(|_| SuperviseError::CommandClosed {
4249                module_id: self.inner.module_id.clone(),
4250            })?;
4251        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4252            module_id: self.inner.module_id.clone(),
4253        })?
4254    }
4255
4256    pub async fn reload(&self) -> Result<(), SuperviseError> {
4257        self.reload_requested(None).await
4258    }
4259
4260    pub(crate) async fn reload_requested(
4261        &self,
4262        requester: Option<Principal>,
4263    ) -> Result<(), SuperviseError> {
4264        let (reply_tx, reply_rx) = oneshot::channel();
4265        self.inner
4266            .commands
4267            .send(SupervisorCommand::Reload {
4268                requester,
4269                reply: reply_tx,
4270            })
4271            .await
4272            .map_err(|_| SuperviseError::CommandClosed {
4273                module_id: self.inner.module_id.clone(),
4274            })?;
4275        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4276            module_id: self.inner.module_id.clone(),
4277        })?
4278    }
4279
4280    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4281        self.set_enabled_requested(enabled, None).await
4282    }
4283
4284    pub(crate) async fn set_enabled_requested(
4285        &self,
4286        enabled: bool,
4287        requester: Option<Principal>,
4288    ) -> Result<bool, SuperviseError> {
4289        let (reply_tx, reply_rx) = oneshot::channel();
4290        self.inner
4291            .commands
4292            .send(SupervisorCommand::SetEnabled {
4293                requester,
4294                enabled,
4295                reply: reply_tx,
4296            })
4297            .await
4298            .map_err(|_| SuperviseError::CommandClosed {
4299                module_id: self.inner.module_id.clone(),
4300            })?;
4301        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4302            module_id: self.inner.module_id.clone(),
4303        })?
4304    }
4305
4306    /// The current process's protocol, or the configured protocol when down.
4307    /// A rescan stores the next launch spec without changing how an existing
4308    /// process registers, serves routes, is probed, or exits.
4309    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4310        let configured = self
4311            .inner
4312            .configuration
4313            .lock()
4314            .map_err(|_| SuperviseError::StatePoisoned {
4315                module_id: Some(self.inner.module_id.clone()),
4316            })?
4317            .spec
4318            .protocol;
4319        let state = lock_snapshot(&self.inner.snapshot)?;
4320        Ok(state.spawned_protocol.unwrap_or(configured))
4321    }
4322
4323    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4324        let configuration =
4325            self.inner
4326                .configuration
4327                .lock()
4328                .map_err(|_| SuperviseError::StatePoisoned {
4329                    module_id: Some(self.inner.module_id.clone()),
4330                })?;
4331        Ok((configuration.spec.clone(), configuration.health.clone()))
4332    }
4333
4334    /// Replace this module's launch spec, keeping its health and drain policy,
4335    /// the way a rescan does for a changed config entry. The running process is
4336    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4337    #[cfg(any(test, feature = "test-support"))]
4338    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4339        let (_, health) = self.configuration()?;
4340        let drain_timeout_ms = u64::try_from(
4341            self.inner
4342                .effective_drain_timeout
4343                .lock()
4344                .unwrap_or_else(|poisoned| poisoned.into_inner())
4345                .as_millis(),
4346        )
4347        .ok();
4348        self.update_configuration(spec, health, drain_timeout_ms)
4349            .await
4350    }
4351
4352    pub(crate) async fn update_configuration(
4353        &self,
4354        spec: ModuleSpec,
4355        health: HealthConfig,
4356        drain_timeout_ms: Option<u64>,
4357    ) -> Result<(), SuperviseError> {
4358        if spec.module_id != self.inner.module_id {
4359            return Err(SuperviseError::InvalidSpec {
4360                reason: "a supervised module's module_id cannot be changed".to_string(),
4361            });
4362        }
4363        validate_spec(&spec)?;
4364        let (reply_tx, reply_rx) = oneshot::channel();
4365        self.inner
4366            .commands
4367            .send(SupervisorCommand::UpdateConfiguration {
4368                spec: spec.clone(),
4369                health: health.clone(),
4370                drain_timeout_ms,
4371                reply: reply_tx,
4372            })
4373            .await
4374            .map_err(|_| SuperviseError::CommandClosed {
4375                module_id: self.inner.module_id.clone(),
4376            })?;
4377        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4378            module_id: self.inner.module_id.clone(),
4379        })?;
4380        let mut configuration =
4381            self.inner
4382                .configuration
4383                .lock()
4384                .map_err(|_| SuperviseError::StatePoisoned {
4385                    module_id: Some(self.inner.module_id.clone()),
4386                })?;
4387        configuration.spec = spec;
4388        configuration.health = health;
4389        Ok(())
4390    }
4391}
4392
4393impl Drop for SupervisedModuleInner {
4394    fn drop(&mut self) {
4395        let Ok(mut monitor) = self.monitor.lock() else {
4396            return;
4397        };
4398        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4399            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4400                state.state = ModuleState::Stopped;
4401                clear_current_process_facts(state);
4402            });
4403            monitor.abort();
4404        }
4405        let _ = monitor.take();
4406    }
4407}
4408
4409#[derive(Debug)]
4410enum SupervisorCommand {
4411    Drain {
4412        requester: Option<Principal>,
4413        reply: oneshot::Sender<Result<(), SuperviseError>>,
4414    },
4415    Retire {
4416        requester: Option<Principal>,
4417        reply: oneshot::Sender<Result<(), SuperviseError>>,
4418    },
4419    Restart {
4420        requester: Option<Principal>,
4421        /// Operator override for this one restart's drain budget, in ms. `None`
4422        /// uses the module's configured/default budget; `Some(0)` cuts
4423        /// immediately (wedge bounce: a stuck request never settles, so
4424        /// waiting only delays recovery).
4425        drain_timeout_ms: Option<u64>,
4426        /// The module's `spawn_generation` when the request was received, before
4427        /// it waited in the command queue. A queued restart whose module has
4428        /// since spawned a newer process is already satisfied (see the handler).
4429        received_at_generation: u64,
4430        /// When the request entered the command queue, so the handler can log
4431        /// how long it waited behind the loop's other work.
4432        queued_at: Instant,
4433        reply: oneshot::Sender<Result<(), SuperviseError>>,
4434    },
4435    Reload {
4436        requester: Option<Principal>,
4437        reply: oneshot::Sender<Result<(), SuperviseError>>,
4438    },
4439    SetEnabled {
4440        requester: Option<Principal>,
4441        enabled: bool,
4442        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4443    },
4444    UpdateConfiguration {
4445        spec: ModuleSpec,
4446        health: HealthConfig,
4447        /// Per-module drain override from the new config; `None` re-resolves to
4448        /// the supervisor-wide default.
4449        drain_timeout_ms: Option<u64>,
4450        reply: oneshot::Sender<()>,
4451    },
4452    Swap {
4453        requester: Option<Principal>,
4454        /// How long the candidate may take to register and declare itself
4455        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4456        ready_timeout: Option<Duration>,
4457        /// Answered at cutover or failure; the incumbent's drain follows.
4458        reply: oneshot::Sender<Result<(), SuperviseError>>,
4459    },
4460}
4461
4462#[derive(Debug)]
4463pub enum SuperviseError {
4464    InvalidSpec {
4465        reason: String,
4466    },
4467    Spawn {
4468        program: PathBuf,
4469        source: io::Error,
4470        cgroup_path: Option<PathBuf>,
4471    },
4472    Cgroup {
4473        module_id: String,
4474        source: io::Error,
4475    },
4476    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4477    /// than spawn a reserved module without its identity binding.
4478    LaunchNonce {
4479        reason: String,
4480    },
4481    Wait {
4482        module_id: String,
4483        source: io::Error,
4484    },
4485    Kill {
4486        module_id: String,
4487        source: io::Error,
4488    },
4489    Forwarding(ForwardingError),
4490    Registry(RegistryError),
4491    ReloadUnavailable {
4492        module_id: String,
4493        reason: String,
4494    },
4495    /// An operator restart/reload was requested for a module that is currently
4496    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4497    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4498    /// by a restart, so these commands are rejected instead of re-enabling it.
4499    Disabled {
4500        module_id: String,
4501    },
4502    ReloadFailed {
4503        module_id: String,
4504        reason: String,
4505    },
4506    RegistrationStillActive {
4507        module_id: String,
4508        waited: Duration,
4509    },
4510    StatePoisoned {
4511        module_id: Option<String>,
4512    },
4513    CommandClosed {
4514        module_id: String,
4515    },
4516    /// A restart or reload arrived while a swap's candidate was warming. The
4517    /// swap owns the module until it cuts over or fails; a stop or disable
4518    /// would have aborted it instead.
4519    SwapInProgress {
4520        module_id: String,
4521    },
4522    /// A swap was refused before anything was spawned.
4523    SwapRefused {
4524        module_id: String,
4525        reason: SwapRefusal,
4526    },
4527    /// A swap spawned a candidate and gave up on it. The candidate has been
4528    /// killed and its slot freed; the incumbent was left serving and was never
4529    /// drained, except in the one `CutoverLost` case described on that arm.
4530    SwapFailed {
4531        module_id: String,
4532        arm: SwapFailureArm,
4533        detail: String,
4534        /// How the candidate exited, when it exited on its own before the
4535        /// supervisor gave up on it.
4536        candidate_exit: Option<ExitReport>,
4537    },
4538}
4539
4540/// Why a swap was refused before a candidate was spawned.
4541#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4542pub enum SwapRefusal {
4543    /// The module's config does not declare `overlap: "safe"`.
4544    OverlapExclusive,
4545    /// The module is not registered, so there is no incumbent to keep serving
4546    /// and nothing a swap would improve on; a plain restart is the tool.
4547    NotRegistered,
4548    /// The module does not speak the subc wire, so a candidate could never
4549    /// register or declare itself ready.
4550    ProtocolNone,
4551    /// The supervisor lacks the forwarding table (to cut routes over) or the
4552    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4553    NotConfigured,
4554    /// A swap is already open for this module.
4555    AlreadySwapping,
4556}
4557
4558impl SwapRefusal {
4559    pub fn as_str(self) -> &'static str {
4560        match self {
4561            Self::OverlapExclusive => "overlap_exclusive",
4562            Self::NotRegistered => "not_registered",
4563            Self::ProtocolNone => "protocol_none",
4564            Self::NotConfigured => "not_configured",
4565            Self::AlreadySwapping => "already_swapping",
4566        }
4567    }
4568}
4569
4570/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4571/// serving and undrained; see `CutoverLost`.
4572#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4573pub enum SwapFailureArm {
4574    /// The candidate process could not be started.
4575    SpawnFailed,
4576    /// The candidate did not register within the readiness budget.
4577    NeverRegistered,
4578    /// The candidate registered but did not declare itself ready in time.
4579    NeverReady,
4580    /// The candidate exited before cutover.
4581    CandidateExited,
4582    /// The candidate declared itself ready but failed its health probe.
4583    CandidateUnhealthy,
4584    /// An operator stop, disable or retire arrived while the candidate warmed.
4585    /// The candidate was killed and the operator's command then carried out on
4586    /// the incumbent.
4587    Interrupted,
4588    /// The candidate's connection closed at the moment of cutover. If it
4589    /// closed before forwarding moved, the incumbent is untouched. If it closed
4590    /// between the forwarding and registry halves of cutover, forwarding can no
4591    /// longer route to the incumbent, so the module is restarted plainly.
4592    CutoverLost,
4593}
4594
4595impl SwapFailureArm {
4596    pub fn as_str(self) -> &'static str {
4597        match self {
4598            Self::SpawnFailed => "spawn_failed",
4599            Self::NeverRegistered => "never_registered",
4600            Self::NeverReady => "never_ready",
4601            Self::CandidateExited => "candidate_exited",
4602            Self::CandidateUnhealthy => "candidate_unhealthy",
4603            Self::Interrupted => "interrupted",
4604            Self::CutoverLost => "cutover_lost",
4605        }
4606    }
4607}
4608
4609impl fmt::Display for SuperviseError {
4610    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4611        match self {
4612            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4613            Self::Spawn {
4614                program,
4615                source,
4616                cgroup_path: Some(cgroup_path),
4617            } => write!(
4618                f,
4619                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4620                cgroup_path.display(),
4621                program.display()
4622            ),
4623            Self::Spawn {
4624                program,
4625                source,
4626                cgroup_path: None,
4627            } => write!(
4628                f,
4629                "failed to spawn module '{}': {source}",
4630                program.display()
4631            ),
4632            Self::Cgroup { module_id, source } => {
4633                write!(
4634                    f,
4635                    "failed to prepare cgroup for module '{module_id}': {source}"
4636                )
4637            }
4638            Self::LaunchNonce { reason } => {
4639                write!(
4640                    f,
4641                    "failed to generate reserved-module launch nonce: {reason}"
4642                )
4643            }
4644            Self::Wait { module_id, source } => {
4645                write!(f, "failed to wait for module '{module_id}': {source}")
4646            }
4647            Self::Kill { module_id, source } => {
4648                write!(f, "failed to kill module '{module_id}': {source}")
4649            }
4650            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4651            Self::Registry(err) => write!(f, "registry error: {err}"),
4652            Self::ReloadUnavailable { module_id, reason } => {
4653                write!(f, "reload unavailable for module '{module_id}': {reason}")
4654            }
4655            Self::Disabled { module_id } => {
4656                write!(
4657                    f,
4658                    "module '{module_id}' is disabled; enable it before restart or reload"
4659                )
4660            }
4661            Self::ReloadFailed { module_id, reason } => {
4662                write!(f, "reload failed for module '{module_id}': {reason}")
4663            }
4664            Self::RegistrationStillActive { module_id, waited } => write!(
4665                f,
4666                "module '{module_id}' registration remained active after waiting {waited:?}"
4667            ),
4668            Self::StatePoisoned { module_id } => match module_id {
4669                Some(module_id) => {
4670                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4671                }
4672                None => write!(f, "supervisor state was poisoned"),
4673            },
4674            Self::CommandClosed { module_id } => {
4675                write!(
4676                    f,
4677                    "supervisor command channel for module '{module_id}' is closed"
4678                )
4679            }
4680            Self::SwapInProgress { module_id } => write!(
4681                f,
4682                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4683            ),
4684            Self::SwapRefused { module_id, reason } => match reason {
4685                SwapRefusal::OverlapExclusive => write!(
4686                    f,
4687                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4688                ),
4689                SwapRefusal::NotRegistered => write!(
4690                    f,
4691                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4692                ),
4693                SwapRefusal::ProtocolNone => write!(
4694                    f,
4695                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4696                ),
4697                SwapRefusal::NotConfigured => write!(
4698                    f,
4699                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4700                ),
4701                SwapRefusal::AlreadySwapping => {
4702                    write!(f, "module '{module_id}' is already being swapped")
4703                }
4704            },
4705            Self::SwapFailed {
4706                module_id,
4707                arm,
4708                detail,
4709                ..
4710            } => write!(
4711                f,
4712                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4713                arm.as_str()
4714            ),
4715        }
4716    }
4717}
4718
4719impl Error for SuperviseError {
4720    fn source(&self) -> Option<&(dyn Error + 'static)> {
4721        match self {
4722            Self::Spawn { source, .. }
4723            | Self::Cgroup { source, .. }
4724            | Self::Wait { source, .. }
4725            | Self::Kill { source, .. } => Some(source),
4726            Self::Forwarding(err) => Some(err),
4727            Self::Registry(err) => Some(err),
4728            Self::LaunchNonce { .. }
4729            | Self::InvalidSpec { .. }
4730            | Self::ReloadUnavailable { .. }
4731            | Self::Disabled { .. }
4732            | Self::ReloadFailed { .. }
4733            | Self::RegistrationStillActive { .. }
4734            | Self::StatePoisoned { .. }
4735            | Self::CommandClosed { .. }
4736            | Self::SwapInProgress { .. }
4737            | Self::SwapRefused { .. }
4738            | Self::SwapFailed { .. } => None,
4739        }
4740    }
4741}
4742
4743pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4744    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4745        return Err(SuperviseError::InvalidSpec {
4746            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4747        });
4748    }
4749    if spec.module_id.trim().is_empty() {
4750        return Err(SuperviseError::InvalidSpec {
4751            reason: "module_id must not be empty".to_string(),
4752        });
4753    }
4754
4755    Ok(())
4756}
4757
4758#[derive(Debug, Default)]
4759struct HealthProbeRuntime {
4760    spawn_generation: Option<u64>,
4761    configured_health: Option<HealthConfig>,
4762    registered_connection: Option<crate::ConnectionId>,
4763    advertised: bool,
4764    next_probe_at: Option<Instant>,
4765    probe_index: u64,
4766}
4767
4768fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4769    lock_snapshot(snapshot)
4770        .ok()
4771        .and_then(|state| state.spawned_protocol)
4772        .unwrap_or(spec.protocol)
4773}
4774
4775impl HealthProbeRuntime {
4776    fn refresh_registration(
4777        &mut self,
4778        spec: &ModuleSpec,
4779        runtime: &SupervisorRuntimeConfig,
4780        registry: &Registry,
4781        snapshot: &SharedSnapshot,
4782    ) {
4783        let generation = lock_snapshot(snapshot)
4784            .ok()
4785            .map(|state| state.spawn_generation);
4786        if self.spawn_generation != generation {
4787            // A replacement cannot reuse a deadline that is already due for its
4788            // predecessor. HTTP children get a full cadence to start; wire
4789            // children still wait for their own health-advertising HELLO.
4790            *self = Self {
4791                spawn_generation: generation,
4792                ..Self::default()
4793            };
4794        }
4795        if self.configured_health.as_ref() != Some(&runtime.health) {
4796            self.configured_health = Some(runtime.health.clone());
4797            self.next_probe_at = None;
4798            self.registered_connection = None;
4799            self.probe_index = 0;
4800        }
4801        // A non-wire process never registers. Only an explicitly configured
4802        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4803        // health failure for that kind of process.
4804        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4805            self.registered_connection = None;
4806            self.advertised = runtime.health.http.is_some();
4807            if !self.advertised {
4808                self.next_probe_at = None;
4809                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4810                    let unknown = ModuleHealthStatus::default();
4811                    if state.health != unknown {
4812                        state.health = unknown;
4813                    }
4814                });
4815            } else if self.next_probe_at.is_none() {
4816                self.next_probe_at = Some(
4817                    Instant::now()
4818                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4819                );
4820            }
4821            return;
4822        }
4823
4824        let registration = match registry.get_module(&spec.module_id) {
4825            Ok(registration) => registration,
4826            Err(err) => {
4827                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4828                self.advertised = false;
4829                self.next_probe_at = None;
4830                return;
4831            }
4832        };
4833
4834        let Some(registration) = registration else {
4835            self.registered_connection = None;
4836            self.advertised = false;
4837            self.next_probe_at = None;
4838            return;
4839        };
4840
4841        let advertised = registration
4842            .control_ops
4843            .iter()
4844            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4845        if !advertised {
4846            self.registered_connection = Some(registration.connection_id);
4847            self.advertised = false;
4848            self.next_probe_at = None;
4849            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4850                if state.health.status != SupervisorHealthStatus::Unknown
4851                    || state.health.consecutive_failures != 0
4852                    || state.health.last_probe_ms.is_some()
4853                    || state.health.detail.is_some()
4854                    || state.health.metrics.is_some()
4855                {
4856                    state.health.status = SupervisorHealthStatus::Unknown;
4857                    state.health.consecutive_failures = 0;
4858                    state.health.last_probe_ms = None;
4859                    state.health.detail = None;
4860                    state.health.metrics = None;
4861                }
4862            });
4863            return;
4864        }
4865
4866        let reregistered = self.registered_connection != Some(registration.connection_id);
4867        self.registered_connection = Some(registration.connection_id);
4868        self.advertised = true;
4869        if reregistered || self.next_probe_at.is_none() {
4870            self.probe_index = 0;
4871            self.next_probe_at = Some(
4872                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4873            );
4874            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4875                state.health.status = SupervisorHealthStatus::Unknown;
4876                state.health.consecutive_failures = 0;
4877                state.health.detail = None;
4878                state.health.metrics = None;
4879            });
4880        }
4881    }
4882
4883    fn wake_at(&self) -> Option<Instant> {
4884        if !self.advertised {
4885            return None;
4886        }
4887        self.next_probe_at
4888    }
4889
4890    fn due(&self) -> bool {
4891        self.advertised
4892            && self
4893                .next_probe_at
4894                .is_some_and(|next| Instant::now() >= next)
4895    }
4896
4897    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4898        self.probe_index = self.probe_index.wrapping_add(1);
4899        self.next_probe_at = Some(
4900            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4901        );
4902    }
4903}
4904
4905/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4906///
4907/// This was a struct with a single `message: String`, and every one of the
4908/// fifteen construction sites collapsed into it. Each site knows exactly what it
4909/// saw -- the lane is gone, the module did not answer in time, the module
4910/// answered with the wrong thing -- and `handle_health_probe_failure` then
4911/// treated all of them identically: increment a counter, compare to a threshold,
4912/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4913/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4914///
4915/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4916///
4917/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4918///   answer on it again.
4919/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4920///   AND with a perfectly healthy one that lost a CPU race -- which is what
4921///   happens under machine load, and is how this supervisor killed a healthy
4922///   module three times in one day.
4923/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4924///   Restarting on it is defensible, but it is not the silence case and should
4925///   never be counted as one.
4926/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4927///   anything, so it cannot be evidence about the module at all.
4928///
4929/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4930/// one that fires most often, and while every variant collapsed into one string
4931/// it carried the same weight as the strongest.
4932///
4933/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4934/// DESIGN and a reader stopping at it gets the build backwards: the restart
4935/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4936/// probes still increment the failure streak and drive escalation at the
4937/// threshold (see `is_proof_of_death` below for why that is deliberate and
4938/// what gates the change). Absence of evidence restarts modules today.
4939#[derive(Debug)]
4940enum HealthProbeEvidence {
4941    /// The module's control lane is gone. Proof of death.
4942    LaneDead,
4943    /// No reply within the deadline. Proves nothing about the module's state.
4944    NoAnswer,
4945    /// The module replied, but not with a usable health report. Proves it is alive.
4946    BadAnswer,
4947    /// The daemon could not ask. Says nothing about the module.
4948    Misconfigured,
4949}
4950
4951#[derive(Debug)]
4952struct HealthProbeError {
4953    evidence: HealthProbeEvidence,
4954    message: String,
4955}
4956
4957impl HealthProbeError {
4958    fn lane_dead(message: impl Into<String>) -> Self {
4959        Self::with(HealthProbeEvidence::LaneDead, message)
4960    }
4961
4962    fn no_answer(message: impl Into<String>) -> Self {
4963        Self::with(HealthProbeEvidence::NoAnswer, message)
4964    }
4965
4966    fn bad_answer(message: impl Into<String>) -> Self {
4967        Self::with(HealthProbeEvidence::BadAnswer, message)
4968    }
4969
4970    fn misconfigured(message: impl Into<String>) -> Self {
4971        Self::with(HealthProbeEvidence::Misconfigured, message)
4972    }
4973
4974    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4975        Self {
4976            evidence,
4977            message: message.into(),
4978        }
4979    }
4980
4981    /// Whether this observation is proof the module cannot serve.
4982    ///
4983    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4984    /// variant that fires under CPU starvation, and treating it as proof is the
4985    /// defect this enum exists to make impossible to reintroduce silently.
4986    ///
4987    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4988    /// to restart also needs a bound for the case it excludes -- a genuinely
4989    /// wedged module, alive but never answering -- and that bound must come from
4990    /// the distribution of real late-answer latencies, which nothing measures
4991    /// yet. Landing the classification first makes the later change a one-line
4992    /// decision against evidence that already exists, rather than two unproven
4993    /// changes at once.
4994    #[allow(dead_code)]
4995    fn is_proof_of_death(&self) -> bool {
4996        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4997    }
4998
4999    /// Short stable label for logs and the health snapshot.
5000    ///
5001    /// An operator reading `ck health` currently cannot tell "the module is gone"
5002    /// from "the module did not answer in five seconds", because both render as
5003    /// prose in the same field. These labels are what make the two
5004    /// distinguishable at a glance, and they are what a later restart-policy
5005    /// change will be argued from.
5006    fn label(&self) -> &'static str {
5007        match self.evidence {
5008            HealthProbeEvidence::LaneDead => "lane-dead",
5009            HealthProbeEvidence::NoAnswer => "no-answer",
5010            HealthProbeEvidence::BadAnswer => "bad-answer",
5011            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5012        }
5013    }
5014}
5015
5016impl fmt::Display for HealthProbeError {
5017    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5018        f.write_str(&self.message)
5019    }
5020}
5021
5022async fn run_health_probe_cycle(
5023    spec: &ModuleSpec,
5024    runtime: &SupervisorRuntimeConfig,
5025    registry: &Registry,
5026    process_liveness: &SupervisorProcessLiveness,
5027    snapshot: &SharedSnapshot,
5028    child: &mut Option<SupervisedChild>,
5029) {
5030    let now_ms = unix_ms_now();
5031    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5032        .then_some(runtime.health.http.as_deref())
5033        .flatten();
5034    let result = match http {
5035        Some(url) => probe_http_health(url, runtime.health.deadline).await,
5036        None => probe_module_health(&spec.module_id, runtime, None).await,
5037    };
5038    match result {
5039        Ok(report) => {
5040            handle_health_report(
5041                spec,
5042                runtime,
5043                registry,
5044                process_liveness,
5045                snapshot,
5046                child,
5047                report,
5048                now_ms,
5049            )
5050            .await;
5051        }
5052        Err(err) => {
5053            if http.is_some() {
5054                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5055                    state.health.status = SupervisorHealthStatus::Failing;
5056                });
5057            }
5058            handle_health_probe_failure(
5059                spec,
5060                runtime,
5061                registry,
5062                process_liveness,
5063                snapshot,
5064                child,
5065                err,
5066                now_ms,
5067            )
5068            .await;
5069        }
5070    }
5071}
5072
5073pub(crate) struct HttpProbeTarget<'a> {
5074    address: std::net::SocketAddr,
5075    localhost: bool,
5076    authority: &'a str,
5077    path: String,
5078}
5079
5080/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
5081/// or TLS. A URL cannot turn a local health check into an outbound connection.
5082pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5083    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5084        return Err("must not contain whitespace, controls, or a fragment".into());
5085    }
5086    let rest = url
5087        .strip_prefix("http://")
5088        .ok_or("must use plain http://")?;
5089    let split = rest.find(['/', '?']).unwrap_or(rest.len());
5090    let (authority, suffix) = rest.split_at(split);
5091    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5092        ("::1", rest)
5093    } else {
5094        let split = authority.find(':').unwrap_or(authority.len());
5095        authority.split_at(split)
5096    };
5097    let ip: std::net::IpAddr = match host {
5098        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5099        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5100        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5101    };
5102    let port = if port.is_empty() {
5103        80
5104    } else {
5105        port.strip_prefix(':')
5106            .and_then(|p| p.parse::<u16>().ok())
5107            .filter(|p| *p > 0)
5108            .ok_or("must have a valid nonzero TCP port")?
5109    };
5110    let path = if suffix.is_empty() {
5111        "/".into()
5112    } else if suffix.starts_with('?') {
5113        format!("/{suffix}")
5114    } else {
5115        suffix.into()
5116    };
5117    Ok(HttpProbeTarget {
5118        address: std::net::SocketAddr::new(ip, port),
5119        localhost: host == "localhost",
5120        authority,
5121        path,
5122    })
5123}
5124
5125async fn probe_http_health(
5126    url: &str,
5127    deadline: Duration,
5128) -> Result<HealthReport, HealthProbeError> {
5129    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5130    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5131    // Keep partial diagnostics outside the timed future so cancellation does
5132    // not discard a status line or body bytes already received.
5133    let mut response_status = String::new();
5134    let mut body = Vec::new();
5135    let probe = async {
5136        // Resolve localhost ourselves so a hosts-file override cannot turn
5137        // this into an outbound request, while IPv6-only local servers work.
5138        let connection = match tokio::net::TcpStream::connect(target.address).await {
5139            Err(_) if target.localhost => {
5140                tokio::net::TcpStream::connect((
5141                    std::net::Ipv6Addr::LOCALHOST,
5142                    target.address.port(),
5143                ))
5144                .await
5145            }
5146            result => result,
5147        };
5148        let mut stream = connection.map_err(|error| {
5149            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5150        })?;
5151        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5152            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5153        let mut reader = BufReader::new(stream);
5154        let mut budget = 16 * 1024;
5155        let status = http_line(&mut reader, &mut budget).await?;
5156        let mut words = status.split_ascii_whitespace();
5157        let version = words.next();
5158        let code = words
5159            .next()
5160            .filter(|word| word.len() == 3)
5161            .and_then(|word| word.parse::<u16>().ok());
5162        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5163            || !code.is_some_and(|code| (100..600).contains(&code))
5164        {
5165            return Err(HealthProbeError::bad_answer(format!(
5166                "invalid HTTP status: {status}"
5167            )));
5168        }
5169        let code = code.expect("validated status code");
5170        response_status = status.clone();
5171        let mut length = None;
5172        let mut chunked = false;
5173        loop {
5174            let line = http_line(&mut reader, &mut budget).await?;
5175            if line.is_empty() {
5176                break;
5177            }
5178            if let Some((name, value)) = line.split_once(':') {
5179                if name.eq_ignore_ascii_case("content-length") {
5180                    length = Some(value.trim().parse::<u64>().map_err(|_| {
5181                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
5182                    })?);
5183                } else if name.eq_ignore_ascii_case("transfer-encoding") {
5184                    chunked = value.trim().eq_ignore_ascii_case("chunked");
5185                }
5186            }
5187        }
5188        if chunked {
5189            while body.len() < 200 {
5190                let line = http_line(&mut reader, &mut budget).await?;
5191                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5192                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5193                if size == 0 {
5194                    break;
5195                }
5196                let count = size.min((200 - body.len()) as u64) as usize;
5197                let start = body.len();
5198                (&mut reader)
5199                    .take(count as u64)
5200                    .read_to_end(&mut body)
5201                    .await
5202                    .map_err(|error| {
5203                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5204                    })?;
5205                if body.len() - start != count {
5206                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5207                }
5208                if size > count as u64 || body.len() == 200 {
5209                    break;
5210                }
5211                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5212                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5213                }
5214            }
5215        } else {
5216            reader
5217                .take(length.unwrap_or(200).min(200))
5218                .read_to_end(&mut body)
5219                .await
5220                .map_err(|error| {
5221                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5222                })?;
5223        }
5224        if (200..300).contains(&code) {
5225            Ok(HealthReport::ok())
5226        } else {
5227            Err(HealthProbeError::bad_answer(
5228                "HTTP health endpoint returned non-2xx",
5229            ))
5230        }
5231    };
5232    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5233        Err(HealthProbeError::no_answer(format!(
5234            "HTTP probe timed out after {deadline:?}"
5235        )))
5236    });
5237    if let Err(error) = &mut result {
5238        if !response_status.is_empty() {
5239            error.message = format!(
5240                "{}; {response_status}: {}",
5241                error.message,
5242                String::from_utf8_lossy(&body)
5243            );
5244        }
5245    }
5246    result
5247}
5248
5249async fn http_line(
5250    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5251    remaining: &mut usize,
5252) -> Result<String, HealthProbeError> {
5253    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5254    let mut line = Vec::new();
5255    (&mut *reader)
5256        .take(*remaining as u64)
5257        .read_until(b'\n', &mut line)
5258        .await
5259        .map_err(|error| {
5260            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5261        })?;
5262    *remaining -= line.len();
5263    if !line.ends_with(b"\r\n") {
5264        return Err(HealthProbeError::bad_answer(
5265            "HTTP headers are incomplete or exceed 16 KiB",
5266        ));
5267    }
5268    line.truncate(line.len() - 2);
5269    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5270}
5271
5272async fn probe_module_health(
5273    module_id: &str,
5274    runtime: &SupervisorRuntimeConfig,
5275    drain_deadline: Option<Instant>,
5276) -> Result<HealthReport, HealthProbeError> {
5277    let Some(forwarding) = runtime.forwarding.as_ref() else {
5278        return Err(HealthProbeError::misconfigured(
5279            "supervisor was not configured with a forwarding table",
5280        ));
5281    };
5282    let probe_started_at = Instant::now();
5283    let mut deadline = probe_started_at + runtime.health.deadline;
5284    if let Some(drain_deadline) = drain_deadline {
5285        deadline = deadline.min(drain_deadline);
5286    }
5287    let pending = if drain_deadline.is_some() {
5288        forwarding.begin_drain_health_probe_rpc_for(
5289            module_id,
5290            MODULE_CONTROL_OP_HEALTH_CHECK,
5291            probe_started_at,
5292            deadline,
5293        )
5294    } else {
5295        forwarding.begin_health_probe_rpc_for(
5296            module_id,
5297            MODULE_CONTROL_OP_HEALTH_CHECK,
5298            probe_started_at,
5299            deadline,
5300        )
5301    }
5302    .map_err(|err| {
5303        // The endpoint is not registered, so there is no live control lane to
5304        // ask. That is the module being absent, not slow.
5305        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5306    })?;
5307    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5308}
5309
5310/// [`probe_module_health`] for one endpoint rather than the id's active one.
5311///
5312/// A swap probes two processes that no by-id lookup reaches: its candidate
5313/// before cutover, and its superseded incumbent (for busy gauges) while the
5314/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5315/// bounds the by-id drain probe.
5316async fn probe_endpoint_health(
5317    endpoint: crate::ModuleEndpointId,
5318    runtime: &SupervisorRuntimeConfig,
5319    deadline_cap: Option<Instant>,
5320) -> Result<HealthReport, HealthProbeError> {
5321    let Some(forwarding) = runtime.forwarding.as_ref() else {
5322        return Err(HealthProbeError::misconfigured(
5323            "supervisor was not configured with a forwarding table",
5324        ));
5325    };
5326    let probe_started_at = Instant::now();
5327    let mut deadline = probe_started_at + runtime.health.deadline;
5328    if let Some(cap) = deadline_cap {
5329        deadline = deadline.min(cap);
5330    }
5331    let pending = forwarding
5332        .begin_endpoint_health_probe_rpc_for(
5333            endpoint,
5334            MODULE_CONTROL_OP_HEALTH_CHECK,
5335            probe_started_at,
5336            deadline,
5337        )
5338        .map_err(|err| {
5339            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5340        })?;
5341    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5342}
5343
5344/// Send a begun health probe and classify its answer.
5345async fn await_health_probe(
5346    forwarding: &ForwardingTable,
5347    pending: PendingModuleControlRpc,
5348    deadline: Instant,
5349    probe_budget: Duration,
5350) -> Result<HealthReport, HealthProbeError> {
5351    let PendingModuleControlRpc {
5352        endpoint,
5353        module_sink,
5354        negotiated_ver,
5355        corr,
5356        receiver,
5357    } = pending;
5358    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5359        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5360    })?;
5361    let frame = Frame::build_with_version(
5362        negotiated_ver,
5363        FrameType::Request,
5364        control_flags(),
5365        0,
5366        0,
5367        corr,
5368        body,
5369    )
5370    .map_err(|err| {
5371        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5372    })?;
5373
5374    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5375    // blocks waiting for capacity when the module's egress queue is full, and an
5376    // unbounded await here freezes the whole supervision actor (it stops polling
5377    // Child::wait and supervisor commands), making the module unrecoverable
5378    // in-band. On timeout the probe fails like any transport failure.
5379    match timeout_at(deadline, module_sink.send(frame)).await {
5380        Ok(Ok(())) => {}
5381        Ok(Err(err)) => {
5382            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5383            // A closed sink means the module's egress channel is gone -- the
5384            // receiving half is dropped when its connection tears down. Proof.
5385            return Err(HealthProbeError::lane_dead(format!(
5386                "failed to send health.check: {err}"
5387            )));
5388        }
5389        Err(_elapsed) => {
5390            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5391            // A full egress queue means the module is not draining its socket, which
5392            // is consistent with a wedged module AND with one whose reader is merely
5393            // starved. Silence, not proof.
5394            return Err(HealthProbeError::no_answer(
5395                "health.check send timed out before enqueue (module egress full)",
5396            ));
5397        }
5398    }
5399
5400    match timeout_at(deadline, receiver).await {
5401        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5402        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5403        // and those prove it is alive even though the probe failed.
5404        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5405            response.health_report().ok_or_else(|| {
5406                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5407            })
5408        }
5409        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5410            format!("health.check rejected: {}", body.message),
5411        )),
5412        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5413            Err(HealthProbeError::lane_dead(message))
5414        }
5415        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5416            Err(HealthProbeError::bad_answer(message))
5417        }
5418        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5419            Err(HealthProbeError::bad_answer(format!(
5420                "expected module-control op '{expected}', got '{actual}'"
5421            )))
5422        }
5423        // A reply that crosses the deadline before this waiter observes it is
5424        // still proof of life. The forwarding path records its end-to-end latency
5425        // before delivering this classification.
5426        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5427            "module answered health.check after its daemon deadline",
5428        )),
5429        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5430            "health.check waiter was canceled before the module responded",
5431        )),
5432        Err(_) => {
5433            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5434            Err(HealthProbeError::no_answer(format!(
5435                "module did not answer health.check within {probe_budget:?}"
5436            )))
5437        }
5438    }
5439}
5440
5441#[allow(clippy::too_many_arguments)]
5442async fn handle_health_report(
5443    spec: &ModuleSpec,
5444    runtime: &SupervisorRuntimeConfig,
5445    registry: &Registry,
5446    process_liveness: &SupervisorProcessLiveness,
5447    snapshot: &SharedSnapshot,
5448    child: &mut Option<SupervisedChild>,
5449    report: HealthReport,
5450    now_ms: u64,
5451) {
5452    let status = supervisor_health_status(report.status);
5453    let detail = report.detail.clone();
5454    let metrics = truncate_health_metrics(report.metrics);
5455    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5456        state.health.status = status;
5457        state.health.last_probe_ms = Some(now_ms);
5458        state.health.detail = detail.clone();
5459        state.health.metrics = metrics.clone();
5460        state.health.consecutive_failures = 0;
5461        state.failed_retry.observe_health(
5462            report.status == HealthStatus::Ok,
5463            runtime.restart_policy.window,
5464        );
5465    });
5466
5467    let action = match report.status {
5468        HealthStatus::Ok => return,
5469        HealthStatus::Degraded => runtime.health.on_degraded,
5470        HealthStatus::Failing => runtime.health.on_failing,
5471    };
5472    apply_l3_health_action(
5473        spec,
5474        runtime,
5475        registry,
5476        process_liveness,
5477        snapshot,
5478        child,
5479        status,
5480        detail.as_deref(),
5481        action,
5482        now_ms,
5483    )
5484    .await;
5485}
5486
5487#[allow(clippy::too_many_arguments)]
5488async fn handle_health_probe_failure(
5489    spec: &ModuleSpec,
5490    runtime: &SupervisorRuntimeConfig,
5491    registry: &Registry,
5492    process_liveness: &SupervisorProcessLiveness,
5493    snapshot: &SharedSnapshot,
5494    child: &mut Option<SupervisedChild>,
5495    err: HealthProbeError,
5496    now_ms: u64,
5497) {
5498    let threshold = runtime.health.failure_threshold.max(1);
5499    let mut failures = 0;
5500    // Carry the evidence class into the operator-visible detail. Without it,
5501    // "module did not answer within 5s" and "the control lane is gone" are two
5502    // prose strings in the same field, and the reader has to know the codebase to
5503    // tell which one is proof of anything.
5504    let detail = format!("[{}] {err}", err.label());
5505    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506        state
5507            .failed_retry
5508            .observe_health(false, runtime.restart_policy.window);
5509        // A failed wire probe invalidates the last report, even before the
5510        // restart threshold. HTTP probes already mark failures as Failing.
5511        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5512            state.health.status = SupervisorHealthStatus::Unknown;
5513        }
5514        state.health.last_probe_ms = Some(now_ms);
5515        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5516        state.health.detail = Some(detail.clone());
5517        state.health.metrics = None;
5518        failures = state.health.consecutive_failures;
5519    });
5520
5521    if failures < threshold {
5522        warn!(
5523            module_id = %spec.module_id,
5524            consecutive_failures = failures,
5525            threshold,
5526            evidence = err.label(),
5527            detail = %detail,
5528            "health.check probe failed"
5529        );
5530        return;
5531    }
5532
5533    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5534        state.state = ModuleState::Unresponsive;
5535        state.health.status = SupervisorHealthStatus::Unresponsive;
5536    });
5537    // The evidence class is logged at the kill site because this is the line an
5538    // operator reads after an unexplained restart. A streak of `no-answer` under
5539    // machine load is the known false-positive shape; a `lane-dead` is not.
5540    if runtime.health.critical {
5541        error!(
5542            module_id = %spec.module_id,
5543            status = "unresponsive",
5544            evidence = err.label(),
5545            detail = %detail,
5546            "critical module health alert"
5547        );
5548    } else {
5549        warn!(
5550            module_id = %spec.module_id,
5551            status = "unresponsive",
5552            evidence = err.label(),
5553            detail = %detail,
5554            "module health threshold breached"
5555        );
5556    }
5557    if let Err(err) = health_restart_child(
5558        spec,
5559        runtime,
5560        registry,
5561        process_liveness,
5562        snapshot,
5563        child,
5564        SupervisorHealthStatus::Unresponsive,
5565        Some(&detail),
5566        now_ms,
5567    )
5568    .await
5569    {
5570        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5571    }
5572}
5573
5574#[allow(clippy::too_many_arguments)]
5575async fn apply_l3_health_action(
5576    spec: &ModuleSpec,
5577    runtime: &SupervisorRuntimeConfig,
5578    registry: &Registry,
5579    process_liveness: &SupervisorProcessLiveness,
5580    snapshot: &SharedSnapshot,
5581    child: &mut Option<SupervisedChild>,
5582    status: SupervisorHealthStatus,
5583    detail: Option<&str>,
5584    action: HealthAction,
5585    now_ms: u64,
5586) {
5587    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5588    match action {
5589        HealthAction::Report => {
5590            info!(
5591                module_id = %spec.module_id,
5592                status = ?status,
5593                detail,
5594                "module reported non-ok health"
5595            );
5596        }
5597        HealthAction::Alert => {
5598            error!(
5599                module_id = %spec.module_id,
5600                status = ?status,
5601                detail,
5602                "module health alert"
5603            );
5604        }
5605        HealthAction::Restart => {
5606            if let Err(err) = health_restart_child(
5607                spec,
5608                runtime,
5609                registry,
5610                process_liveness,
5611                snapshot,
5612                child,
5613                status,
5614                detail,
5615                now_ms,
5616            )
5617            .await
5618            {
5619                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5620            }
5621        }
5622    }
5623}
5624
5625#[allow(clippy::too_many_arguments)]
5626async fn health_restart_child(
5627    spec: &ModuleSpec,
5628    runtime: &SupervisorRuntimeConfig,
5629    registry: &Registry,
5630    process_liveness: &SupervisorProcessLiveness,
5631    snapshot: &SharedSnapshot,
5632    child: &mut Option<SupervisedChild>,
5633    status: SupervisorHealthStatus,
5634    detail: Option<&str>,
5635    now_ms: u64,
5636) -> Result<(), SuperviseError> {
5637    let (enabled, schedule) = {
5638        let mut state = lock_snapshot(snapshot)?;
5639        let enabled = state.enabled;
5640        let schedule = if enabled {
5641            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5642        } else {
5643            None
5644        };
5645        (enabled, schedule)
5646    };
5647
5648    if !enabled {
5649        return Err(SuperviseError::Disabled {
5650            module_id: spec.module_id.clone(),
5651        });
5652    }
5653
5654    if schedule.is_none() {
5655        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5656        error!(
5657            module_id = %spec.module_id,
5658            status = ?status,
5659            detail,
5660            max_restarts = runtime.restart_policy.max_restarts,
5661            window_secs = runtime.restart_policy.window.as_secs(),
5662            reason = %runtime.restart_policy.budget_exhausted_detail(),
5663            "health restart budget exhausted; marking module failed"
5664        );
5665        let stop_notice = begin_forwarding_drain_if_configured(
5666            spec,
5667            runtime,
5668            registry,
5669            snapshot,
5670            Some(true),
5671            RouteCloseReason::Disable,
5672        )
5673        .await?;
5674        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5675            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5676        })?;
5677        let result = drain_optional_child(
5678            &spec.module_id,
5679            spec.protocol,
5680            stop_notice,
5681            registry,
5682            runtime.forwarding.as_deref(),
5683            snapshot,
5684            &runtime.terminal_ring,
5685            &runtime.spawn_events,
5686            child,
5687            runtime.drain_timeout,
5688            ModuleState::Failed,
5689            Some(true),
5690            &runtime.failure_context(true),
5691        )
5692        .await;
5693        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5694            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5695                state.failed_retry.schedule(&spec.module_id);
5696            }
5697        })?;
5698        result?;
5699        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5700        return Ok(());
5701    }
5702
5703    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5704    let mut restart_count = 0;
5705    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5706        restart_count = state.crash_restarts.len();
5707        state.state = ModuleState::Unresponsive;
5708        state.health.status = status;
5709        state.health.last_action = Some(HealthAction::Restart.to_string());
5710        state.health.last_action_ms = Some(now_ms);
5711    })?;
5712    warn!(
5713        module_id = %spec.module_id,
5714        status = ?status,
5715        detail,
5716        restart_count,
5717        restart_in_window = schedule.restart_in_window,
5718        delay_ms = schedule.delay.as_millis() as u64,
5719        "health-triggered module restart"
5720    );
5721
5722    let stop_notice = begin_forwarding_drain_if_configured(
5723        spec,
5724        runtime,
5725        registry,
5726        snapshot,
5727        Some(true),
5728        RouteCloseReason::Restart,
5729    )
5730    .await?;
5731    drain_optional_child(
5732        &spec.module_id,
5733        spec.protocol,
5734        stop_notice,
5735        registry,
5736        runtime.forwarding.as_deref(),
5737        snapshot,
5738        &runtime.terminal_ring,
5739        &runtime.spawn_events,
5740        child,
5741        runtime.drain_timeout,
5742        ModuleState::Restarting,
5743        Some(true),
5744        &runtime.failure_context(true),
5745    )
5746    .await?;
5747    schedule_respawn(
5748        runtime,
5749        snapshot,
5750        &spec.module_id,
5751        schedule.delay,
5752        RespawnKind::Spawn,
5753        true,
5754    )
5755}
5756
5757fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5758    if let Some(reply) = runtime
5759        .deferred_reload_reply
5760        .lock()
5761        .unwrap_or_else(|p| p.into_inner())
5762        .take()
5763    {
5764        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5765            module_id: module_id.to_string(),
5766            reason: reason.to_string(),
5767        }));
5768    }
5769}
5770
5771fn schedule_respawn(
5772    runtime: &SupervisorRuntimeConfig,
5773    snapshot: &SharedSnapshot,
5774    module_id: &str,
5775    delay: Duration,
5776    kind: RespawnKind,
5777    daemon_initiated: bool,
5778) -> Result<(), SuperviseError> {
5779    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5780    update_snapshot(snapshot, Some(module_id), |state| {
5781        state.respawn_pending = true
5782    })?;
5783    *runtime
5784        .scheduled_respawn
5785        .lock()
5786        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5787        deadline: Instant::now() + delay,
5788        kind,
5789        daemon_initiated,
5790    });
5791    Ok(())
5792}
5793
5794fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5795    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5796        state.health.last_action = Some(action);
5797        state.health.last_action_ms = Some(now_ms);
5798    });
5799}
5800
5801fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5802    match status {
5803        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5804        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5805        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5806    }
5807}
5808
5809/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5810/// returned to every `supervisor.list` and `supervisor.health` caller.
5811///
5812/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5813/// path: that request exists to return a module's complete metrics object, and
5814/// `ck health <module-id>` documents it as the way to see what the cached view
5815/// truncates. The asymmetry is the feature.
5816///
5817/// So a new caller must decide which side it is on rather than assume the cap is
5818/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5819/// the truncation that path exists to avoid.
5820fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5821    let metrics = metrics?;
5822    match serde_json::to_vec(&metrics) {
5823        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5824            "truncated": true,
5825            "original_bytes": encoded.len(),
5826        })),
5827        Ok(_) | Err(_) => Some(metrics),
5828    }
5829}
5830
5831/// Spread health probes so a fleet-wide restart does not converge them.
5832///
5833/// The delay is derived from the module id and probe index rather than a random
5834/// source, so it is deterministic per module: a module keeps its own offset
5835/// across daemon restarts instead of re-rolling into a collision.
5836fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5837    if cadence.is_zero() {
5838        return Duration::ZERO;
5839    }
5840    let cadence_ms = cadence.as_millis() as u64;
5841    // This early return is REDUNDANT, deliberately, and a mutation run will show
5842    // it surviving removal. Recording why here so the next person to notice does
5843    // not have to re-derive it:
5844    //
5845    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5846    //   a zero cadence and builds the Duration from whole milliseconds, so a
5847    //   sub-millisecond cadence cannot come from config.
5848    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5849    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5850    //   -- exactly what this returns.
5851    //
5852    // Kept as a guard against a future widening of the config parser (accepting
5853    // microseconds, say), which would make the sub-millisecond case reachable.
5854    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5855    // divides by zero. Remove this and nothing changes.
5856    if cadence_ms == 0 {
5857        return cadence;
5858    }
5859    // Note that this never returns less than one cadence, including for the FIRST
5860    // probe. So a freshly registered module reports health `unknown` for a full
5861    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5862    // ready to answer.
5863    //
5864    // That is a property of the supervisor's schedule, not of any module: an
5865    // operator watching a restart sees `unknown` and cannot tell it from a module
5866    // that is slow to warm. Measured on two unrelated modules, both flipping to
5867    // `ok` between 22s and 32s after restart.
5868    //
5869    // Left as-is because spreading the first probe is what keeps a fleet-wide
5870    // restart from firing fourteen simultaneous probes into a cold machine. The
5871    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5872    // that thundering herd for a faster first reading.
5873    let jitter_span = (cadence_ms / 10).max(1);
5874    let hash = module_id.as_bytes().iter().fold(
5875        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5876        |acc, byte| {
5877            acc.wrapping_mul(1099511628211)
5878                .wrapping_add(u64::from(*byte))
5879        },
5880    );
5881    cadence + Duration::from_millis(hash % jitter_span)
5882}
5883
5884#[cfg(test)]
5885mod tests {
5886    use super::*;
5887
5888    #[test]
5889    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5890        let handle = SupervisorHandle::new();
5891        let module_id = "readded-tombstone";
5892        handle.record_rescan_removal(module_id);
5893        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5894
5895        handle.apply_identity_configuration(&ModuleSpec {
5896            module_id: module_id.to_string(),
5897            program: PathBuf::from("/test/module"),
5898            args: Vec::new(),
5899            env: Vec::new(),
5900            reserved: false,
5901            reserved_prefixes: Vec::new(),
5902            protocol: ModuleProtocol::Subc,
5903            overlap: Default::default(),
5904            carry_containment: false,
5905        });
5906
5907        assert!(
5908            handle.removal_tombstone_age_ms(module_id).is_none(),
5909            "a re-added module must not retain a stale removal tombstone"
5910        );
5911    }
5912
5913    /// What one module's owner looked like from the control plane at the
5914    /// instant after its first process was spawned.
5915    #[derive(Debug, PartialEq, Eq)]
5916    struct OwnerInSpawnWindow {
5917        module_id: String,
5918        configured: bool,
5919        on_roster: bool,
5920        admission_refusal: Option<&'static str>,
5921    }
5922
5923    /// A supervised module's process can connect, register, sync its scopes
5924    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5925    /// the module on the roster until afterward; configured supervision does
5926    /// so before spawning. In both cases the owner must
5927    /// already read as configured, so a scoped `route.open` against it is
5928    /// refused as retryable `scope_not_synced` and not as terminal
5929    /// `scope_not_live` ("will never sync").
5930    ///
5931    /// The hook runs just after each initial spawn attempt, before returning to
5932    /// the caller, so no race with a real child's registration is needed.
5933    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5934    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5935        use crate::scopes::ScopeTable;
5936        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5937
5938        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5939        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5940            module_id: module_id.to_string(),
5941            program,
5942            args: Vec::new(),
5943            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5944                .into_iter()
5945                .map(|key| (key.to_string(), dir.path().display().to_string()))
5946                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5947                .collect(),
5948            reserved: false,
5949            reserved_prefixes: Vec::new(),
5950            protocol: ModuleProtocol::Subc,
5951            overlap: Default::default(),
5952            carry_containment: false,
5953        };
5954        let live = super::terminal_history_tests::fake_aft_stub_path();
5955        let missing = dir.path().join("definitely-missing-module");
5956
5957        let handle = SupervisorHandle::new();
5958        let mut supervisor =
5959            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5960                .with_handle(handle.clone());
5961        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5962        let hook_handle = handle.clone();
5963        let hook_observed = Arc::clone(&observed);
5964        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5965            // Exactly what the control plane computes for a scoped route.open
5966            // naming this module as the owner of a scope it has not synced.
5967            let configured = hook_handle.is_configured(module_id);
5968            let selector = ScopeSelector::new(
5969                Principal::Reserved {
5970                    module_id: module_id.to_string(),
5971                },
5972                "s".to_string(),
5973                Some(1),
5974            );
5975            let carrier = Principal::Reserved {
5976                module_id: "carrier".to_string(),
5977            };
5978            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5979                .admit(&carrier, module_id, &selector, configured)
5980            {
5981                Ok(_) => None,
5982                Err(refusal) => Some(refusal.code),
5983            };
5984            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5985                module_id: module_id.to_string(),
5986                configured,
5987                on_roster: hook_handle.get(module_id).is_some(),
5988                admission_refusal,
5989            });
5990        })));
5991
5992        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5993        let configured = supervisor
5994            .supervise_configured(stub("configured", live.clone()), true)
5995            .unwrap();
5996        let with_health = supervisor
5997            .supervise_configured_with_health(
5998                stub("with-health", live.clone()),
5999                true,
6000                HealthConfig::default(),
6001                None,
6002                RestartPolicy::default(),
6003            )
6004            .unwrap();
6005        // The failed-spawn path still puts the module on the roster (as
6006        // failed), so it is configured throughout.
6007        let failed = supervisor
6008            .supervise_configured_with_health(
6009                stub("failed-spawn", missing.clone()),
6010                true,
6011                HealthConfig::default(),
6012                None,
6013                RestartPolicy::default(),
6014            )
6015            .unwrap();
6016        // A failed plain `spawn` puts nothing on the roster, so its mark is
6017        // taken back once the spawn has failed.
6018        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6019
6020        let expected = [
6021            "plain",
6022            "configured",
6023            "with-health",
6024            "failed-spawn",
6025            "spawn-error",
6026        ]
6027        .into_iter()
6028        .map(|module_id| OwnerInSpawnWindow {
6029            module_id: module_id.to_string(),
6030            configured: true,
6031            on_roster: !matches!(module_id, "plain" | "spawn-error"),
6032            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6033        })
6034        .collect::<Vec<_>>();
6035        assert_eq!(*observed.lock().unwrap(), expected);
6036
6037        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6038            assert!(
6039                handle.get(module_id).is_some(),
6040                "{module_id} is on the roster"
6041            );
6042            assert!(
6043                handle.is_configured(module_id),
6044                "{module_id} stays configured"
6045            );
6046        }
6047        assert!(handle.get("spawn-error").is_none());
6048        assert!(
6049            !handle.is_configured("spawn-error"),
6050            "a plain spawn that failed must not leave its module marked configured"
6051        );
6052
6053        // Leaving the roster clears the mark with it.
6054        handle.retire("failed-spawn");
6055        assert!(!handle.is_configured("failed-spawn"));
6056
6057        for module in [plain, configured, with_health] {
6058            module.stop().await.unwrap();
6059        }
6060        drop(failed);
6061    }
6062
6063    /// A configured module reads as starting until the supervisor has admitted
6064    /// its first process (exec confirmed, identity durably recorded), even
6065    /// though that process is already running by then and can register over
6066    /// HELLO, which is what lists it in catalog.list. So a client that needs
6067    /// the supervisor state must wait for that state; seeing the module in
6068    /// the catalog does not mean supervisor.list already says running.
6069    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6070    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6071        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6072        let spec = ModuleSpec {
6073            module_id: "spawned-not-admitted".to_string(),
6074            program: super::terminal_history_tests::fake_aft_stub_path(),
6075            args: Vec::new(),
6076            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6077                .into_iter()
6078                .map(|key| (key.to_string(), dir.path().display().to_string()))
6079                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6080                .collect(),
6081            reserved: false,
6082            reserved_prefixes: Vec::new(),
6083            protocol: ModuleProtocol::Subc,
6084            overlap: Default::default(),
6085            carry_containment: false,
6086        };
6087        let handle = SupervisorHandle::new();
6088        let mut supervisor =
6089            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6090                .with_handle(handle.clone());
6091        let observed = Arc::new(Mutex::new(None));
6092        let hook_handle = handle.clone();
6093        let hook_observed = Arc::clone(&observed);
6094        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6095            let state = hook_handle
6096                .get(module_id)
6097                .expect("a configured module is on the roster before it is spawned")
6098                .state()
6099                .unwrap();
6100            *hook_observed.lock().unwrap() = Some(state);
6101        })));
6102
6103        let module = supervisor.supervise_configured(spec, true).unwrap();
6104        assert_eq!(
6105            *observed.lock().unwrap(),
6106            Some(ModuleState::Starting),
6107            "between the spawn and its admission the supervisor still reports starting"
6108        );
6109        assert_eq!(module.state().unwrap(), ModuleState::Running);
6110        module.stop().await.unwrap();
6111    }
6112
6113    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6114        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6115        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6116            snapshot.process_alive = true;
6117            snapshot.pid = Some(41);
6118            snapshot.spawned_at_ms = Some(42);
6119            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6120            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6121            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6122                device: 43,
6123                inode: 44,
6124                #[cfg(windows)]
6125                windows_image: None,
6126            });
6127        })
6128        .unwrap();
6129        snapshot
6130    }
6131
6132    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6133        let snapshot = lock_snapshot(snapshot).unwrap();
6134        assert!(!snapshot.process_alive);
6135        assert_eq!(snapshot.pid, None);
6136        assert_eq!(snapshot.spawned_at_ms, None);
6137        assert_eq!(snapshot.spawned_from, None);
6138        assert_eq!(snapshot.launch_spec_digest, None);
6139        assert_eq!(snapshot.spawned_file_identity, None);
6140    }
6141
6142    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6143    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6144        let supervisor =
6145            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6146        let mut runtime = supervisor.runtime_config();
6147        runtime.test_seed_stale_facts_before_enable_spawn = true;
6148        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6149        let mut child = None;
6150        let spec = ModuleSpec {
6151            module_id: "failed-enable-clears-facts".to_string(),
6152            program: PathBuf::from("/definitely/missing/failed-enable-module"),
6153            args: Vec::new(),
6154            env: Vec::new(),
6155            reserved: false,
6156            reserved_prefixes: Vec::new(),
6157            protocol: ModuleProtocol::Subc,
6158            overlap: Default::default(),
6159            carry_containment: false,
6160        };
6161
6162        let result = set_child_enabled(
6163            &spec,
6164            &runtime,
6165            &supervisor.registry,
6166            &supervisor.process_liveness,
6167            &snapshot,
6168            &mut child,
6169            true,
6170            &runtime.failure_context(false),
6171        )
6172        .await;
6173
6174        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6175        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6176        assert_snapshot_process_facts_cleared(&snapshot);
6177    }
6178
6179    #[tokio::test]
6180    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6181        let supervisor =
6182            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6183        let runtime = supervisor.runtime_config();
6184        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6185            ModuleState::Restarting,
6186            true,
6187        )));
6188        let spec = ModuleSpec {
6189            module_id: "start-stranded-restarting".to_string(),
6190            program: super::terminal_history_tests::fake_aft_stub_path(),
6191            args: Vec::new(),
6192            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6193            reserved: false,
6194            reserved_prefixes: Vec::new(),
6195            protocol: ModuleProtocol::None,
6196            overlap: Default::default(),
6197            carry_containment: false,
6198        };
6199        let mut child = None;
6200        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6201        assert!(!super::set_child_enabled(
6202            &spec,
6203            &runtime,
6204            &Registry::default(),
6205            &supervisor.process_liveness,
6206            &snapshot,
6207            &mut child,
6208            true,
6209            &runtime.failure_context(false),
6210        )
6211        .await
6212        .unwrap());
6213        assert!(child.is_none());
6214        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6215        assert!(super::set_child_enabled(
6216            &spec,
6217            &runtime,
6218            &Registry::default(),
6219            &supervisor.process_liveness,
6220            &snapshot,
6221            &mut child,
6222            true,
6223            &runtime.failure_context(false),
6224        )
6225        .await
6226        .unwrap());
6227        assert_eq!(
6228            lock_snapshot(&snapshot).unwrap().state,
6229            ModuleState::Running
6230        );
6231        let mut child = child.unwrap();
6232        child.start_kill().unwrap();
6233        child.wait().await.unwrap();
6234    }
6235
6236    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6237    async fn failed_reload_spawn_clears_current_process_facts() {
6238        let supervisor =
6239            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6240        let mut runtime = supervisor.runtime_config();
6241        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6242        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6243        let mut child = None;
6244        let spec = ModuleSpec {
6245            module_id: "failed-reload-clears-facts".to_string(),
6246            program: PathBuf::from("/unused/failed-reload-module"),
6247            args: Vec::new(),
6248            env: Vec::new(),
6249            reserved: false,
6250            reserved_prefixes: Vec::new(),
6251            protocol: ModuleProtocol::Subc,
6252            overlap: Default::default(),
6253            carry_containment: false,
6254        };
6255
6256        let result = handle_reload_spawn_failure(
6257            &spec,
6258            &runtime,
6259            &supervisor.process_liveness,
6260            &snapshot,
6261            &mut child,
6262            "forced reload spawn failure".to_string(),
6263        )
6264        .await;
6265
6266        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6267        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6268        assert_snapshot_process_facts_cleared(&snapshot);
6269    }
6270
6271    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6272    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6273        let supervisor =
6274            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6275        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6276        let module = supervisor.supervised_module(
6277            ModuleSpec {
6278                module_id: "drop-clears-facts".to_string(),
6279                program: PathBuf::from("/unused/drop-module"),
6280                args: Vec::new(),
6281                env: Vec::new(),
6282                reserved: false,
6283                reserved_prefixes: Vec::new(),
6284                protocol: ModuleProtocol::Subc,
6285                overlap: Default::default(),
6286                carry_containment: false,
6287            },
6288            supervisor.runtime_config(),
6289            Arc::clone(&snapshot),
6290            None,
6291        );
6292        assert!(!module
6293            .inner
6294            .monitor
6295            .lock()
6296            .unwrap()
6297            .as_ref()
6298            .unwrap()
6299            .is_finished());
6300
6301        drop(module);
6302
6303        assert_eq!(
6304            lock_snapshot(&snapshot).unwrap().state,
6305            ModuleState::Stopped
6306        );
6307        assert_snapshot_process_facts_cleared(&snapshot);
6308    }
6309
6310    #[cfg(unix)]
6311    #[tokio::test]
6312    async fn rescan_preserves_running_protocol_until_respawn() {
6313        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6314        let initial = ModuleSpec {
6315            module_id: "rescan-protocol".into(),
6316            program: PathBuf::from("/bin/sleep"),
6317            args: vec!["60".into()],
6318            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6319                .into_iter()
6320                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6321                .collect(),
6322            reserved: false,
6323            reserved_prefixes: vec![],
6324            protocol: ModuleProtocol::None,
6325            overlap: Default::default(),
6326            carry_containment: false,
6327        };
6328        let supervisor =
6329            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6330        let module = supervisor.spawn(initial.clone()).unwrap();
6331        assert!(module.status().unwrap().live);
6332        let mut next = initial;
6333        next.protocol = ModuleProtocol::Subc;
6334        module
6335            .update_configuration(next.clone(), HealthConfig::default(), None)
6336            .await
6337            .unwrap();
6338        assert!(
6339            module.status().unwrap().live,
6340            "rescan must not require HELLO from the old non-wire process"
6341        );
6342        let runtime = supervisor.runtime_config();
6343        let action = on_child_exit(
6344            &next,
6345            RestartPolicy::default(),
6346            &supervisor.registry,
6347            &module.inner.snapshot,
6348            &runtime.terminal_ring,
6349            &runtime.spawn_events,
6350            &runtime.child_roster,
6351            ExitReport {
6352                kind: ExitKind::Clean,
6353                code: Some(0),
6354                signal: None,
6355                at_ms: unix_ms_now(),
6356            },
6357            true,
6358        )
6359        .await;
6360        assert!(
6361            matches!(action, NextAction::Restart { .. }),
6362            "the old non-wire process's clean exit must restart"
6363        );
6364        module.drain().await.unwrap();
6365    }
6366
6367    #[cfg(unix)]
6368    #[tokio::test]
6369    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6370        use std::os::unix::fs::PermissionsExt;
6371        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6372        let script = dir.join("module.sh");
6373        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6374        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6375        let record_path = dir.join("live-children.json");
6376        let supervisor =
6377            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6378                .with_live_children_record(&record_path);
6379        for (program, args) in [
6380            (PathBuf::from("sleep"), vec!["60".into()]),
6381            (script, vec![]),
6382        ] {
6383            let spec = ModuleSpec {
6384                module_id: "image-identity".into(),
6385                program: program.clone(),
6386                args,
6387                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6388                    .into_iter()
6389                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6390                    .collect(),
6391                reserved: false,
6392                reserved_prefixes: vec![],
6393                protocol: ModuleProtocol::None,
6394                overlap: Default::default(),
6395                carry_containment: false,
6396            };
6397            let module = supervisor.spawn(spec).unwrap();
6398            #[cfg(target_os = "macos")]
6399            {
6400                // SETEXEC confirmation is asynchronous; the orphan record must
6401                // identify the final image, never the intermediate trampoline.
6402                let deadline = Instant::now() + Duration::from_secs(5);
6403                while crate::live_children::read_record(&record_path)
6404                    .unwrap()
6405                    .iter()
6406                    .all(|entry| entry.executable.is_none())
6407                {
6408                    assert!(Instant::now() < deadline, "module image was not confirmed");
6409                    tokio::time::sleep(Duration::from_millis(5)).await;
6410                }
6411            }
6412            let entry = crate::live_children::read_record(&record_path)
6413                .unwrap()
6414                .pop()
6415                .unwrap();
6416            let observed = subc_os::Process::open(entry.pid)
6417                .unwrap()
6418                .unwrap()
6419                .observe()
6420                .unwrap();
6421            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6422            module.drain().await.unwrap();
6423            assert_eq!(
6424                verdict,
6425                crate::live_children::IdentityVerdict::Matches,
6426                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6427            );
6428        }
6429    }
6430
6431    #[cfg(unix)]
6432    fn http_fixture(
6433        dir: &std::path::Path,
6434        url: &str,
6435        threshold: u32,
6436    ) -> crate::daemon_config::ConfiguredModule {
6437        let path = dir.join("subc.jsonc");
6438        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6439            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6440            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6441            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6442            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6443        }}}).to_string()).unwrap();
6444        crate::daemon_config::load(&path)
6445            .unwrap()
6446            .unwrap()
6447            .modules
6448            .pop()
6449            .unwrap()
6450    }
6451
6452    #[cfg(unix)]
6453    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6454        timeout(Duration::from_secs(5), async {
6455            loop {
6456                if module.status().unwrap().health.status == status {
6457                    break;
6458                }
6459                sleep(Duration::from_millis(5)).await;
6460            }
6461        })
6462        .await
6463        .unwrap_or_else(|_| {
6464            panic!(
6465                "expected {status:?}, got {:?}",
6466                module.status().unwrap().health
6467            )
6468        });
6469    }
6470
6471    #[cfg(unix)]
6472    #[tokio::test]
6473    async fn http_health_status_flips_ok_failing_ok() {
6474        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6475        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6476        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6477        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6478        let serving_status = status.clone();
6479        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6480        let server = tokio::spawn(async move {
6481            loop {
6482                let (mut stream, _) = listener.accept().await.unwrap();
6483                let mut request = [0u8; 2048];
6484                let count = stream.read(&mut request).await.unwrap();
6485                assert!(count > 0, "a probe must send an HTTP request");
6486                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6487                let body = if code == 200 {
6488                    "ready"
6489                } else {
6490                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6491                };
6492                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6493                let _ = stream.write_all(response.as_bytes()).await;
6494            }
6495        });
6496        let configured = http_fixture(&dir, &url, 1000);
6497        let module =
6498            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6499                .supervise_configured_with_health(
6500                    configured.module_spec(),
6501                    true,
6502                    configured.health,
6503                    configured.drain_timeout_ms,
6504                    configured.restart,
6505                )
6506                .unwrap();
6507        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6508        status.store(503, std::sync::atomic::Ordering::SeqCst);
6509        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6510        assert!(module
6511            .status()
6512            .unwrap()
6513            .health
6514            .detail
6515            .unwrap()
6516            .contains("scratch failure"));
6517        status.store(200, std::sync::atomic::Ordering::SeqCst);
6518        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6519        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6520        let before = module.status().unwrap();
6521        let (spec, mut health) = module.configuration().unwrap();
6522        health.http = None;
6523        module
6524            .update_configuration(spec.clone(), health.clone(), Some(10))
6525            .await
6526            .unwrap();
6527        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6528        health.http = Some(url);
6529        module
6530            .update_configuration(spec, health, Some(10))
6531            .await
6532            .unwrap();
6533        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6534        assert_eq!(
6535            module.status().unwrap().pid,
6536            before.pid,
6537            "changing a probe must apply live, not restart its process"
6538        );
6539        let (spec, mut health) = module.configuration().unwrap();
6540        health.failure_threshold = 2;
6541        module
6542            .update_configuration(spec, health, Some(10))
6543            .await
6544            .unwrap();
6545        status.store(503, std::sync::atomic::Ordering::SeqCst);
6546        timeout(Duration::from_secs(5), async {
6547            while module.status().unwrap().spawn_generation == before.spawn_generation {
6548                sleep(Duration::from_millis(5)).await;
6549            }
6550        })
6551        .await
6552        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6553        module.drain().await.unwrap();
6554        server.abort();
6555    }
6556
6557    #[cfg(unix)]
6558    #[tokio::test]
6559    async fn http_health_replacement_recovers_after_one_restart() {
6560        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6561        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6562        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6563        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6564        let mut configured = http_fixture(&dir, &url, 2);
6565        configured.restart.max_restarts = 2;
6566        let module =
6567            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6568                .supervise_configured_with_health(
6569                    configured.module_spec(),
6570                    true,
6571                    configured.health,
6572                    configured.drain_timeout_ms,
6573                    configured.restart,
6574                )
6575                .unwrap();
6576        let first_generation = module.status().unwrap().spawn_generation;
6577        let observed = module.clone();
6578        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6579        let server = tokio::spawn(async move {
6580            let mut generation = first_generation;
6581            let mut probes = 0;
6582            loop {
6583                let (mut stream, _) = listener.accept().await.unwrap();
6584                let mut request = [0; 2048];
6585                assert!(stream.read(&mut request).await.unwrap() > 0);
6586                let current = observed.status().unwrap().spawn_generation;
6587                if current != generation {
6588                    generation = current;
6589                    probes = 0;
6590                }
6591                probes += 1;
6592                // The original process never recovers. Each replacement has one
6593                // transient miss, then answers successfully for its remaining life.
6594                let healthy = generation != first_generation && probes > 1;
6595                let code = if healthy { 200 } else { 503 };
6596                let response = format!(
6597                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6598                );
6599                let _ = stream.write_all(response.as_bytes()).await;
6600                if healthy {
6601                    healthy_tx.send(()).await.unwrap();
6602                }
6603            }
6604        });
6605        timeout(Duration::from_secs(5), async {
6606            let mut healthy_probes = 0;
6607            loop {
6608                if module.state().unwrap() == ModuleState::Failed {
6609                    break;
6610                }
6611                while healthy_rx.try_recv().is_ok() {
6612                    healthy_probes += 1;
6613                }
6614                if healthy_probes >= 3
6615                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6616                {
6617                    break;
6618                }
6619                sleep(Duration::from_millis(5)).await;
6620            }
6621        })
6622        .await
6623        .expect("the replacement must settle healthy without exhausting its restart budget");
6624        let status = module.status().unwrap();
6625        module.drain().await.unwrap();
6626        server.abort();
6627        assert_eq!(status.state, ModuleState::Running);
6628        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6629        assert_eq!(status.health.consecutive_failures, 0);
6630        assert_eq!(status.lifetime_restarts, 1);
6631        assert_eq!(status.restart_count, 1);
6632    }
6633
6634    #[cfg(unix)]
6635    #[tokio::test]
6636    async fn http_health_sustained_failure_exhausts_restart_budget() {
6637        use std::collections::BTreeMap;
6638        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6639        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6640        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6641        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6642        let mut configured = http_fixture(&dir, &url, 2);
6643        configured.restart.max_restarts = 2;
6644        let module =
6645            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6646                .supervise_configured_with_health(
6647                    configured.module_spec(),
6648                    true,
6649                    configured.health,
6650                    configured.drain_timeout_ms,
6651                    configured.restart,
6652                )
6653                .unwrap();
6654        let observed = module.clone();
6655        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6656        let served = probes.clone();
6657        let server = tokio::spawn(async move {
6658            loop {
6659                let (mut stream, _) = listener.accept().await.unwrap();
6660                let mut request = [0; 2048];
6661                assert!(stream.read(&mut request).await.unwrap() > 0);
6662                *served
6663                    .lock()
6664                    .unwrap()
6665                    .entry(observed.status().unwrap().spawn_generation)
6666                    .or_default() += 1;
6667                let _ = stream
6668                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6669                    .await;
6670            }
6671        });
6672        timeout(Duration::from_secs(5), async {
6673            while module.state().unwrap() != ModuleState::Failed {
6674                sleep(Duration::from_millis(5)).await;
6675            }
6676        })
6677        .await
6678        .expect("sustained failures must still exhaust the health restart budget");
6679        let status = module.status().unwrap();
6680        server.abort();
6681        assert_eq!(status.restart_count, 2);
6682        assert_eq!(status.lifetime_restarts, 2);
6683        assert!(!status.process_alive);
6684        assert_eq!(
6685            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6686            vec![2, 2, 2]
6687        );
6688    }
6689
6690    #[cfg(unix)]
6691    #[tokio::test]
6692    async fn spawn_clears_the_previous_process_health() {
6693        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6694        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6695        configured.health.cadence = Duration::from_secs(3600);
6696        let module =
6697            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6698                .supervise_configured_with_health(
6699                    configured.module_spec(),
6700                    false,
6701                    configured.health,
6702                    configured.drain_timeout_ms,
6703                    configured.restart,
6704                )
6705                .unwrap();
6706        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6707            state.health.status = SupervisorHealthStatus::Unresponsive;
6708            state.health.consecutive_failures = 3;
6709            state.health.detail = Some("old process refused connections".into());
6710            state.health.metrics = Some(serde_json::json!({"old": true}));
6711            state.health.last_probe_ms = Some(123);
6712            state.health.last_action = Some("restart".into());
6713            state.health.last_action_ms = Some(124);
6714        })
6715        .unwrap();
6716        module.set_enabled(true).await.unwrap();
6717        let health = module.status().unwrap().health;
6718        module.drain().await.unwrap();
6719        assert_eq!(health.consecutive_failures, 0);
6720        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6721        assert_eq!(health.detail, None);
6722        assert_eq!(health.metrics, None);
6723        assert_eq!(health.last_probe_ms, None);
6724        assert_eq!(health.last_action.as_deref(), Some("restart"));
6725        assert_eq!(health.last_action_ms, Some(124));
6726    }
6727
6728    #[cfg(unix)]
6729    #[tokio::test]
6730    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6731        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6732        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6733        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6734        drop(unused);
6735        let configured = http_fixture(&dir, &url, 2);
6736        let module =
6737            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6738                .supervise_configured_with_health(
6739                    configured.module_spec(),
6740                    true,
6741                    configured.health,
6742                    configured.drain_timeout_ms,
6743                    configured.restart,
6744                )
6745                .unwrap();
6746        let before = module.status().unwrap().spawn_generation;
6747        timeout(Duration::from_secs(5), async {
6748            loop {
6749                let status = module.status().unwrap();
6750                if status.spawn_generation > before {
6751                    assert!(status.lifetime_restarts > 0);
6752                    break;
6753                }
6754                sleep(Duration::from_millis(5)).await;
6755            }
6756        })
6757        .await
6758        .expect("sustained HTTP refusal must trigger the health restart policy");
6759        module.drain().await.unwrap();
6760    }
6761
6762    #[cfg(unix)]
6763    #[tokio::test]
6764    async fn http_health_timeout_honours_deadline() {
6765        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6766        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6767        let server = tokio::spawn(async move {
6768            let _held = listener.accept().await.unwrap();
6769            std::future::pending::<()>().await;
6770        });
6771        let error = timeout(
6772            Duration::from_secs(1),
6773            probe_http_health(&url, Duration::from_millis(10)),
6774        )
6775        .await
6776        .expect("the probe must enforce its own deadline")
6777        .unwrap_err();
6778        server.abort();
6779        assert!(error.to_string().contains("timed out"));
6780    }
6781
6782    #[cfg(unix)]
6783    #[tokio::test]
6784    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6785        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6786        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6787        let url = format!(
6788            "http://localhost:{}/healthz",
6789            listener.local_addr().unwrap().port()
6790        );
6791        let server = tokio::spawn(async move {
6792            let (mut stream, _) = listener.accept().await.unwrap();
6793            let mut request = [0u8; 2048];
6794            assert!(stream.read(&mut request).await.unwrap() > 0);
6795            stream
6796                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6797                .await
6798                .unwrap();
6799        });
6800        // The deadline only bounds a hang. A probe that never tried the IPv6
6801        // address would be refused on 127.0.0.1 and fail at once, so a longer
6802        // deadline does not weaken the assertion; one second timed out under a
6803        // loaded parallel test run.
6804        assert_eq!(
6805            probe_http_health(&url, Duration::from_secs(10))
6806                .await
6807                .unwrap()
6808                .status,
6809            HealthStatus::Ok
6810        );
6811        server.await.unwrap();
6812    }
6813
6814    #[cfg(unix)]
6815    #[tokio::test]
6816    async fn http_health_timeout_keeps_partial_status_and_body() {
6817        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6818        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6819        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6820        let server = tokio::spawn(async move {
6821            let (mut stream, _) = listener.accept().await.unwrap();
6822            let mut request = [0u8; 2048];
6823            assert!(stream.read(&mut request).await.unwrap() > 0);
6824            stream
6825                .write_all(
6826                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6827                )
6828                .await
6829                .unwrap();
6830            std::future::pending::<()>().await;
6831        });
6832        let error = probe_http_health(&url, Duration::from_secs(1))
6833            .await
6834            .unwrap_err()
6835            .to_string();
6836        server.abort();
6837        assert!(
6838            error.contains("timed out")
6839                && error.contains("503 Unavailable")
6840                && error.contains("partial diagnostic"),
6841            "{error}"
6842        );
6843    }
6844
6845    #[cfg(unix)]
6846    #[tokio::test]
6847    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6848        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6849        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6850        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6851        let server = tokio::spawn(async move {
6852            let (mut stream, _) = listener.accept().await.unwrap();
6853            let mut request = [0u8; 2048];
6854            assert!(stream.read(&mut request).await.unwrap() > 0);
6855            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6856            let response = format!(
6857                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6858                body.len()
6859            );
6860            stream.write_all(response.as_bytes()).await.unwrap();
6861        });
6862        let error = probe_http_health(&url, Duration::from_secs(1))
6863            .await
6864            .unwrap_err()
6865            .to_string();
6866        server.await.unwrap();
6867        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6868        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6869        assert!(!error.contains("not-in-diagnostic"));
6870    }
6871
6872    #[cfg(unix)]
6873    #[tokio::test]
6874    async fn http_health_real_nats_server_monitoring() {
6875        if std::process::Command::new("nats-server")
6876            .arg("--version")
6877            .env("XDG_DATA_HOME", std::env::temp_dir())
6878            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6879            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6880            .output()
6881            .is_err()
6882        {
6883            eprintln!(
6884                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6885            );
6886            return;
6887        }
6888        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6889        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6890        let port = monitor.local_addr().unwrap().port();
6891        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6892        let client_port = client.local_addr().unwrap().port();
6893        let config = dir.join("server.conf");
6894        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6895        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6896        configured.program = PathBuf::from("nats-server");
6897        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6898        drop(monitor);
6899        drop(client);
6900        let module =
6901            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6902                .supervise_configured_with_health(
6903                    configured.module_spec(),
6904                    true,
6905                    configured.health,
6906                    configured.drain_timeout_ms,
6907                    configured.restart,
6908                )
6909                .unwrap();
6910        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6911        module.drain().await.unwrap();
6912    }
6913
6914    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6915    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6916        let supervisor =
6917            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6918        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6919        let initial = ModuleSpec {
6920            module_id: "rescan-preserves-spawn-facts".to_string(),
6921            program: PathBuf::from("/spawned/module"),
6922            args: Vec::new(),
6923            env: Vec::new(),
6924            reserved: false,
6925            reserved_prefixes: Vec::new(),
6926            protocol: ModuleProtocol::Subc,
6927            overlap: Default::default(),
6928            carry_containment: false,
6929        };
6930        let module = supervisor.supervised_module(
6931            initial.clone(),
6932            supervisor.runtime_config(),
6933            snapshot,
6934            None,
6935        );
6936        let before = module.status().unwrap();
6937        let mut replacement = initial;
6938        replacement.program = PathBuf::from("/rescanned/replacement-module");
6939
6940        module
6941            .update_configuration(replacement, HealthConfig::default(), None)
6942            .await
6943            .unwrap();
6944
6945        let after = module.status().unwrap();
6946        assert_eq!(after.pid, before.pid);
6947        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6948        assert_eq!(after.spawned_from, before.spawned_from);
6949        drop(module);
6950    }
6951}
6952
6953fn unix_ms_now() -> u64 {
6954    SystemTime::now()
6955        .duration_since(UNIX_EPOCH)
6956        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6957        .unwrap_or(0)
6958}
6959
6960async fn supervise_loop(
6961    mut spec: ModuleSpec,
6962    mut runtime: SupervisorRuntimeConfig,
6963    registry: Arc<Registry>,
6964    process_liveness: Arc<SupervisorProcessLiveness>,
6965    snapshot: SharedSnapshot,
6966    mut child: Option<SupervisedChild>,
6967    mut commands: mpsc::Receiver<SupervisorCommand>,
6968) {
6969    let mut health_probe = HealthProbeRuntime::default();
6970    // Registry writes (including embedded callers) notify this module only.
6971    // Subscribe before the first refresh; watch retains changes that arrive
6972    // while commands or probes are running. No polling fallback is needed.
6973    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6974        Ok(changes) => Some(changes),
6975        Err(err) => {
6976            // A poisoned registry cannot accept further writes, so it cannot
6977            // recover via a timer. Keep exit and command handling alive.
6978            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6979            None
6980        }
6981    };
6982    // All restart backoffs run here, including health and operator requests.
6983    // While one is pending the loop serves commands, so disable or drain can
6984    // cancel the replacement without spawning a process just to stop it.
6985    let mut pending_respawn: Option<PendingRespawn> = None;
6986    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6987    // before anything else so a stop that interrupted a swap runs at once.
6988    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6989    let mut previous_instance_recheck = tokio::time::interval_at(
6990        Instant::now() + Duration::from_secs(5),
6991        Duration::from_secs(5),
6992    );
6993    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6994    loop {
6995        #[cfg(test)]
6996        {
6997            lock_snapshot(&snapshot).unwrap().actor_select = None;
6998        }
6999        #[cfg(target_os = "macos")]
7000        if let Some(active) = child.as_mut() {
7001            active.confirm_privacy_exec().await;
7002        }
7003        if let Some(scheduled) = runtime
7004            .scheduled_respawn
7005            .lock()
7006            .unwrap_or_else(|p| p.into_inner())
7007            .take()
7008        {
7009            runtime.recovery_is_daemon = scheduled.daemon_initiated;
7010            pending_respawn = Some(scheduled);
7011        }
7012        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7013            pending_respawn = None;
7014            cancel_deferred_reload(
7015                &runtime,
7016                &spec.module_id,
7017                "respawn cancelled by a supervisor command",
7018            );
7019        }
7020        if child.is_none()
7021            && pending_respawn.is_none()
7022            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7023        {
7024            cancel_deferred_reload(
7025                &runtime,
7026                &spec.module_id,
7027                "respawn cancelled before a replacement was spawned",
7028            );
7029            let mut stranded = false;
7030            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7031                state.respawn_pending = false;
7032                state.coalesced_restart_pending = false;
7033                if matches!(
7034                    state.state,
7035                    ModuleState::Restarting
7036                        | ModuleState::Starting
7037                        | ModuleState::Draining
7038                        | ModuleState::Unresponsive
7039                ) {
7040                    error!(module_id = %spec.module_id, state = ?state.state,
7041                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7042                    stranded = true;
7043                }
7044            });
7045            if stranded {
7046                fail_snapshot(
7047                    &snapshot,
7048                    Some(&spec.module_id),
7049                    None,
7050                    &runtime.failure_context(runtime.recovery_is_daemon),
7051                );
7052            }
7053        }
7054        if child.is_none()
7055            && (runtime.child_roster.is_closed()
7056                || pending_respawn.is_none()
7057                || !lock_snapshot(&snapshot)
7058                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7059        {
7060            containment::terminate(&runtime.child_roster.containment).await;
7061        }
7062        if let Some(command) = requeued.pop_front() {
7063            if !handle_supervisor_command(
7064                command,
7065                &mut spec,
7066                &mut runtime,
7067                &registry,
7068                &process_liveness,
7069                &snapshot,
7070                &mut child,
7071                &mut commands,
7072                &mut requeued,
7073            )
7074            .await
7075            {
7076                containment::terminate(&runtime.child_roster.containment).await;
7077                return;
7078            }
7079            if child.is_some() || !respawn_still_pending(&snapshot) {
7080                pending_respawn = None;
7081                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7082                    state.respawn_pending = false
7083                });
7084            }
7085            continue;
7086        }
7087        if child.is_some() {
7088            runtime.recovery_is_daemon = true;
7089            #[cfg(test)]
7090            {
7091                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7092            }
7093            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
7094            let active = child.as_mut().expect("child checked above");
7095            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7096                active.registration_deadline = None;
7097            }
7098            let registration_deadline = active.registration_deadline;
7099            let no_checks = runtime.health.http.is_none()
7100                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7101                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7102            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7103                state
7104                    .failed_retry
7105                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
7106            });
7107            let wake_at = match (health_probe.wake_at(), uptime_reset) {
7108                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7109                (probe, None) => probe,
7110            };
7111            let wake_at = match (wake_at, registration_deadline) {
7112                (Some(wake), Some(register)) => Some(wake.min(register)),
7113                (wake, register) => wake.or(register),
7114            };
7115            #[cfg(test)]
7116            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7117            let probe_sleep = supervisor_sleep(wake_at);
7118            tokio::pin!(probe_sleep);
7119            // A `protocol: "none"` child (a plain process that never registers
7120            // over the subc wire, such as nats-server) with no HTTP health
7121            // check has nothing to probe, so it waits only for its exit or a
7122            // supervisor command. A rescan that adds an HTTP check is a command.
7123            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7124            #[cfg(test)]
7125            {
7126                let registration_pending = wire_child
7127                    && registration_changes
7128                        .as_ref()
7129                        .is_some_and(|changes| changes.has_changed().unwrap());
7130                let mut state = lock_snapshot(&snapshot).unwrap();
7131                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7132                    generation: state.spawn_generation,
7133                    turn: state.actor_turns,
7134                    registered_connection: health_probe.registered_connection,
7135                    next_probe_at: health_probe.next_probe_at,
7136                    wake_after,
7137                });
7138            }
7139            let active_child = child.as_mut().expect("child checked above");
7140            tokio::select! {
7141                wait_result = active_child.wait() => {
7142                    if let Ok(mut state) = lock_snapshot(&snapshot) {
7143                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7144                    }
7145                    #[cfg(test)]
7146                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7147                    // Every arm below that gives up on the CHILD must keep the
7148                    // supervision task itself alive (child = None, loop
7149                    // continues into command-serving mode). Returning here
7150                    // closes the command channel, which makes the module
7151                    // permanently unrestartable in-band: a clean child exit
7152                    // of an enabled module once wedged the fleet this way
7153                    // ('supervisor command channel is closed') and required a
7154                    // full daemon restart to recover.
7155                    let exit_report = match wait_result {
7156                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7157                        Err(err) => {
7158                            active_child.drain_stderr(&spec.module_id).await;
7159                            fail_after_wait_error(
7160                                &spec.module_id,
7161                                &snapshot,
7162                                &runtime.terminal_ring,
7163                                &runtime.spawn_events,
7164                                format!("failed to wait for supervised module: {err}"),
7165                            &runtime.failure_context(true),
7166                            );
7167                            untrack_if_registration_released(
7168                                &process_liveness,
7169                                &registry,
7170                                &spec.module_id,
7171                                &snapshot,
7172                            );
7173                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7174                            child = None;
7175                            continue;
7176                        }
7177                    };
7178                    active_child.drain_stderr(&spec.module_id).await;
7179
7180                    let next = on_child_exit(
7181                        &spec,
7182                        runtime.restart_policy,
7183                        &registry,
7184                        &snapshot,
7185                        &runtime.terminal_ring,
7186                        &runtime.spawn_events,
7187                        &runtime.child_roster,
7188                        exit_report,
7189                        true,
7190                    ).await;
7191                    runtime.start_requester = None;
7192                    runtime.pending_activity_restart = false;
7193                    // The exit is recorded, so a daemon shutdown may stop
7194                    // waiting for this child (see `SupervisedChild::wait`).
7195                    let carry = matches!(next, NextAction::Restart { .. })
7196                        && active_child.spawn_failure.is_none()
7197                        && !runtime.child_roster.is_closed();
7198                    if let Err(error) = active_child.finish_exit(carry).await {
7199                        containment::terminate(&runtime.child_roster.containment).await;
7200                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7201                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7202                            &runtime.failure_context(true));
7203                        child = None;
7204                        continue;
7205                    }
7206                    match next {
7207                        NextAction::Stop { registration_released } => {
7208                            if registration_released {
7209                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7210                            }
7211                            child = None;
7212                        }
7213                        NextAction::Restart { schedule } => {
7214                            let delay = schedule.map_or(
7215                                runtime.restart_policy.delay_for_restart(0),
7216                                |schedule| schedule.delay,
7217                            );
7218                            if let Some(schedule) = schedule {
7219                                log_crash_respawn(&spec.module_id, schedule);
7220                            }
7221                            // The exited child is fully recorded at this point,
7222                            // so release it and count the backoff down in the
7223                            // command-serving branch below rather than sleeping
7224                            // here: commands cannot be received from inside this
7225                            // select arm, and an operator disable or drain that
7226                            // arrives during the backoff must cancel the pending
7227                            // respawn instead of waiting for it to spawn first.
7228                            child = None;
7229                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7230                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7231                        }
7232                    }
7233                }
7234                command = commands.recv() => {
7235                    #[cfg(test)]
7236                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7237                    let Some(command) = command else {
7238                        containment::terminate(&runtime.child_roster.containment).await;
7239                        return;
7240                    };
7241                    if !handle_supervisor_command(
7242                        command,
7243                        &mut spec,
7244                        &mut runtime,
7245                        &registry,
7246                        &process_liveness,
7247                        &snapshot,
7248                        &mut child,
7249                        &mut commands,
7250                        &mut requeued,
7251                    ).await {
7252                        containment::terminate(&runtime.child_roster.containment).await;
7253                        return;
7254                    }
7255                }
7256                _ = async {
7257                    match registration_changes.as_mut() {
7258                        Some(changes) => { let _ = changes.changed().await; }
7259                        None => std::future::pending().await,
7260                    }
7261                }, if wire_child => {
7262                    #[cfg(test)]
7263                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7264                }
7265                _ = &mut probe_sleep => {
7266                    #[cfg(test)]
7267                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7268                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7269                        // Recheck after the wake: HELLO, exit or shutdown may
7270                        // have become ready in the same executor tick.
7271                        active_child.registration_deadline = None;
7272                        if !runtime.child_roster.is_closed()
7273                            && !active_child.registered_since_spawn()
7274                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7275                        {
7276                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7277                                wait_secs = runtime.register_timeout.as_secs(),
7278                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7279                            active_child.spawn_failure = Some("registration_timeout".into());
7280                            if let Err(error) = stop_unregistered_child(active_child).await {
7281                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7282                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7283                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7284                                    &runtime.failure_context(true));
7285                                child = None;
7286                            }
7287                            // The ordinary exit handler records the child's exit,
7288                            // spends crash budget, and schedules recovery.
7289                            continue;
7290                        }
7291                    }
7292                    if health_probe.due() {
7293                        run_health_probe_cycle(
7294                            &spec,
7295                            &runtime,
7296                            &registry,
7297                            &process_liveness,
7298                            &snapshot,
7299                            &mut child,
7300                        ).await;
7301                        if child.is_none() {
7302                            runtime.start_requester = None;
7303                            runtime.pending_activity_restart = false;
7304                        }
7305                        if child.is_some() {
7306                            health_probe.schedule_next(&spec, runtime.health.cadence);
7307                        }
7308                    }
7309                }
7310            }
7311        } else if let Some(pending) = pending_respawn {
7312            tokio::select! {
7313                _ = runtime.child_roster.closed() => {
7314                    pending_respawn = None;
7315                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7316                        state.state = ModuleState::Stopped;
7317                        state.respawn_pending = false;
7318                    });
7319                    containment::terminate(&runtime.child_roster.containment).await;
7320                }
7321                _ = sleep_until(pending.deadline) => {
7322                    pending_respawn = None;
7323                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7324                    // A command handled below while the backoff elapsed may
7325                    // have stopped the module; never respawn past an operator's
7326                    // disable or drain.
7327                    if !respawn_still_pending(&snapshot) {
7328                        continue;
7329                    }
7330                    // The daemon began shutting down during the backoff: the
7331                    // spawn would be refused anyway, and refusing it here
7332                    // leaves the module stopped instead of reporting a
7333                    // failed restart.
7334                    if runtime.child_roster.is_closed() {
7335                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7336                            state.state = ModuleState::Stopped;
7337                        });
7338                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7339                        continue;
7340                    }
7341                    if let Err(err) = release_dead_registration(
7342                        &registry,
7343                        runtime.forwarding.as_deref(),
7344                        &snapshot,
7345                        &spec.module_id,
7346                        &runtime.failure_context(runtime.recovery_is_daemon),
7347                    ).await {
7348                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7349                        continue;
7350                    }
7351
7352                    if matches!(pending.kind, RespawnKind::Reload) {
7353                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7354                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7355                        if result.is_ok() && runtime.pending_activity_restart {
7356                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7357                        }
7358                        runtime.pending_activity_restart = false;
7359                        if let Some(reply) = reply { let _ = reply.send(result); }
7360                        continue;
7361                    }
7362                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7363                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7364                        Ok(next_child) => {
7365                            child = Some(next_child);
7366                            if runtime.pending_activity_restart {
7367                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7368                                runtime.pending_activity_restart = false;
7369                            }
7370                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7371                        }
7372                        Err(err) => {
7373                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7374                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7375                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7376                        }
7377                    }
7378                }
7379                command = commands.recv() => {
7380                    let Some(command) = command else {
7381                        containment::terminate(&runtime.child_roster.containment).await;
7382                        return;
7383                    };
7384                    if !handle_supervisor_command(
7385                        command,
7386                        &mut spec,
7387                        &mut runtime,
7388                        &registry,
7389                        &process_liveness,
7390                        &snapshot,
7391                        &mut child,
7392                        &mut commands,
7393                        &mut requeued,
7394                    ).await {
7395                        containment::terminate(&runtime.child_roster.containment).await;
7396                        return;
7397                    }
7398                    // Reconcile the pending respawn with what the command did:
7399                    // a start may already have spawned a fresh child,
7400                    // while a disable or drain moved the snapshot out of the
7401                    // state the respawn was counting down from.
7402                    if child.is_some() || !respawn_still_pending(&snapshot) {
7403                        pending_respawn = None;
7404                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7405                    }
7406                }
7407            }
7408        } else {
7409            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7410                (state.enabled && state.state == ModuleState::Failed)
7411                    .then_some(state.failed_retry.pending)
7412                    .flatten()
7413            });
7414            let command = if lock_snapshot(&snapshot)
7415                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7416            {
7417                tokio::select! {
7418                    command = commands.recv() => command,
7419                    _ = previous_instance_recheck.tick() => {
7420                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7421                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7422                            if enabled && !runtime.child_roster.is_closed() {
7423                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7424                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7425                                    Ok(next_child) => child = Some(next_child),
7426                                    Err(error) => {
7427                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7428                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7429                                    }
7430                                }
7431                            } else {
7432                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7433                                    state.waiting_for_previous_pid = None;
7434                                    state.waiting_for_previous_detail = None;
7435                                });
7436                            }
7437                        }
7438                        continue;
7439                    }
7440                }
7441            } else {
7442                tokio::select! {
7443                    biased;
7444                    command = commands.recv() => command,
7445                    _ = async {
7446                        match slow_retry {
7447                            Some(retry) => sleep_until(retry.deadline).await,
7448                            None => std::future::pending().await,
7449                        }
7450                    } => {
7451                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7452                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7453                            state.failed_retry.pending = None;
7454                        });
7455                        if runtime.child_roster.is_closed() {
7456                            continue;
7457                        }
7458                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7459                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7460                        if let Err(error) = set_child_enabled(
7461                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7462                            &runtime.failure_context(true),
7463                        ).await {
7464                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7465                        }
7466                        continue;
7467                    }
7468                }
7469            };
7470            let Some(command) = command else {
7471                containment::terminate(&runtime.child_roster.containment).await;
7472                return;
7473            };
7474            if !handle_supervisor_command(
7475                command,
7476                &mut spec,
7477                &mut runtime,
7478                &registry,
7479                &process_liveness,
7480                &snapshot,
7481                &mut child,
7482                &mut commands,
7483                &mut requeued,
7484            )
7485            .await
7486            {
7487                containment::terminate(&runtime.child_roster.containment).await;
7488                return;
7489            }
7490        }
7491    }
7492}
7493
7494// Preserve the deadline until the future is polled. Starting a relative sleep
7495// after the executor yields would restart part of the countdown.
7496async fn supervisor_sleep(wake_at: Option<Instant>) {
7497    match wake_at {
7498        Some(deadline) => sleep_until(deadline).await,
7499        None => std::future::pending().await,
7500    }
7501}
7502
7503fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7504    info!(
7505        module_id,
7506        restart_in_window = schedule.restart_in_window,
7507        delay_ms = schedule.delay.as_millis() as u64,
7508        "respawning after crash"
7509    );
7510}
7511
7512/// Whether the respawn a backoff was counting down to is still wanted. A
7513/// disable or drain handled while the backoff elapsed moves the snapshot out
7514/// of `Restarting`, and the operator's stop must win over the pending respawn,
7515/// so every sleep-then-spawn path re-validates against the live snapshot
7516/// instead of assuming the state it left behind still holds.
7517fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7518    matches!(
7519        lock_snapshot(snapshot),
7520        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7521    )
7522}
7523
7524enum NextAction {
7525    Stop {
7526        registration_released: bool,
7527    },
7528    Restart {
7529        schedule: Option<CrashRestartSchedule>,
7530    },
7531}
7532
7533#[allow(clippy::too_many_arguments)]
7534async fn handle_supervisor_command(
7535    command: SupervisorCommand,
7536    spec: &mut ModuleSpec,
7537    runtime: &mut SupervisorRuntimeConfig,
7538    registry: &Arc<Registry>,
7539    process_liveness: &SupervisorProcessLiveness,
7540    snapshot: &SharedSnapshot,
7541    child: &mut Option<SupervisedChild>,
7542    commands: &mut mpsc::Receiver<SupervisorCommand>,
7543    requeued: &mut VecDeque<SupervisorCommand>,
7544) -> bool {
7545    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7546        runtime.recovery_is_daemon = false;
7547    }
7548    if matches!(
7549        command,
7550        SupervisorCommand::Drain { .. }
7551            | SupervisorCommand::Retire { .. }
7552            | SupervisorCommand::SetEnabled { .. }
7553            | SupervisorCommand::Restart { .. }
7554    ) {
7555        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7556            state.failed_retry.cancel()
7557        });
7558    }
7559    let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7560        runtime
7561            .child_roster
7562            .previous_instance(&spec.module_id)
7563            .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7564    });
7565    let command = if let Some(error) = blocked {
7566        match command {
7567            SupervisorCommand::Restart { reply, .. }
7568            | SupervisorCommand::Reload { reply, .. }
7569            | SupervisorCommand::Swap { reply, .. } => {
7570                let _ = reply.send(Err(error));
7571                return true;
7572            }
7573            SupervisorCommand::SetEnabled {
7574                enabled: true,
7575                reply,
7576                ..
7577            } => {
7578                let _ = reply.send(Err(error));
7579                return true;
7580            }
7581            other => other,
7582        }
7583    } else {
7584        command
7585    };
7586    match command {
7587        SupervisorCommand::Drain { requester, reply } => {
7588            // A plain stop runs no forwarding drain, so nothing reaches the
7589            // module over its connection before the wait: ask by signal.
7590            let result = drain_optional_child(
7591                &spec.module_id,
7592                spec.protocol,
7593                StopNotice::NotSent,
7594                registry,
7595                runtime.forwarding.as_deref(),
7596                snapshot,
7597                &runtime.terminal_ring,
7598                &runtime.spawn_events,
7599                child,
7600                runtime.drain_timeout,
7601                ModuleState::Stopped,
7602                None,
7603                &FailureContext::operator(),
7604            )
7605            .await;
7606            containment::terminate(&runtime.child_roster.containment).await;
7607            let registration_released = result.is_ok();
7608            if registration_released {
7609                runtime.record_activity_transition(
7610                    "subc.module_stopped",
7611                    &spec.module_id,
7612                    requester,
7613                );
7614            }
7615            let _ = reply.send(result);
7616            if registration_released {
7617                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7618            }
7619            false
7620        }
7621        SupervisorCommand::Retire { requester, reply } => {
7622            let result = async {
7623                let stop_notice = begin_forwarding_drain_if_configured(
7624                    spec,
7625                    runtime,
7626                    registry,
7627                    snapshot,
7628                    None,
7629                    RouteCloseReason::Disable,
7630                )
7631                .await?;
7632                drain_optional_child(
7633                    &spec.module_id,
7634                    spec.protocol,
7635                    stop_notice,
7636                    registry,
7637                    runtime.forwarding.as_deref(),
7638                    snapshot,
7639                    &runtime.terminal_ring,
7640                    &runtime.spawn_events,
7641                    child,
7642                    runtime.drain_timeout,
7643                    ModuleState::Stopped,
7644                    None,
7645                    &FailureContext::operator(),
7646                )
7647                .await
7648            }
7649            .await;
7650            containment::terminate(&runtime.child_roster.containment).await;
7651            let registration_released = result.is_ok();
7652            if registration_released {
7653                runtime.record_activity_transition(
7654                    "subc.module_stopped",
7655                    &spec.module_id,
7656                    requester,
7657                );
7658            }
7659            let _ = reply.send(result);
7660            if registration_released {
7661                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7662            }
7663            false
7664        }
7665        SupervisorCommand::Restart {
7666            requester,
7667            drain_timeout_ms,
7668            received_at_generation,
7669            queued_at,
7670            reply,
7671        } => {
7672            // Without this line a restart that waited in the queue (behind a
7673            // health probe cycle or another command) was invisible: the log
7674            // showed only the drain timing out, minutes after the operator's call.
7675            info!(
7676                module_id = %spec.module_id,
7677                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7678                "restart command dequeued"
7679            );
7680            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7681            // caller whose own request lane rides the module being restarted: the
7682            // caller's in-flight request keeps the drain from quiescing, the drain
7683            // keeps the restart from completing, and the completion keeps the reply
7684            // from releasing the caller — so the drain always timed out and cut the
7685            // initiator with a GOODBYE, even on a healthy module. Replying once the
7686            // restart is validated lets a self-lane caller settle, which is exactly
7687            // what makes the drain succeed. Completion is observable via
7688            // supervisor.list / module status; a post-ack failure lands the module
7689            // in a visible terminal state below rather than in a reply nobody can
7690            // receive.
7691            let validation = match lock_snapshot(snapshot) {
7692                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7693                    module_id: spec.module_id.clone(),
7694                }),
7695                Ok(_) => Ok(()),
7696                Err(err) => Err(err),
7697            };
7698            let initiated = validation.is_ok();
7699            let _ = reply.send(validation);
7700            // A restart asks for a fresh process. Commands run one at a time,
7701            // so a restart queued behind another restart (two operator calls
7702            // in quick succession) is dequeued the moment the first one has
7703            // spawned its replacement -- before that process has sent HELLO.
7704            // Running it would drain and kill the process the first restart
7705            // just produced, which is the opposite of what both callers asked
7706            // for. If a process spawned after this request was received is
7707            // still supervised, the request is already satisfied. Not when the
7708            // configuration changed since that spawn: then the newer process
7709            // predates the spec this restart may exist to apply.
7710            // Generation publication can lag a child that is already serving
7711            // while its identity is recorded. Only process creation after request
7712            // receipt satisfies a restart, not publication after receipt.
7713            let satisfied_by_generation = if initiated
7714                && child
7715                    .as_ref()
7716                    .is_some_and(|child| child.spawned_at > queued_at)
7717            {
7718                lock_snapshot(snapshot).ok().and_then(|state| {
7719                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7720                })
7721            } else {
7722                None
7723            };
7724            let satisfied_by_pending = initiated
7725                && child.is_none()
7726                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7727                    // No child exists yet, so this replacement will necessarily
7728                    // be created after the queued request, unlike a published
7729                    // generation whose process may already have been serving.
7730                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7731                    if pending {
7732                        state.coalesced_restart_pending = true;
7733                    }
7734                    pending
7735                });
7736            if satisfied_by_pending {
7737                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7738            } else if let Some(generation) = satisfied_by_generation {
7739                info!(
7740                    module_id = %spec.module_id,
7741                    received_at_generation,
7742                    "restart already satisfied by generation {generation}; not restarting again"
7743                );
7744            } else if initiated {
7745                // Precedence: this restart's operator override, else the module's
7746                // configured budget (already resolved into the runtime).
7747                let drain_timeout = drain_timeout_ms
7748                    .map(Duration::from_millis)
7749                    .unwrap_or(runtime.drain_timeout);
7750                if let Err(err) = restart_child(
7751                    spec,
7752                    runtime,
7753                    registry,
7754                    process_liveness,
7755                    snapshot,
7756                    child,
7757                    drain_timeout,
7758                )
7759                .await
7760                {
7761                    warn!(
7762                        module_id = %spec.module_id,
7763                        error = %err,
7764                        "operator restart failed after initiation ack; module state carries the outcome"
7765                    );
7766                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7767                        state.state = ModuleState::Failed;
7768                        clear_current_process_facts(state);
7769                    });
7770                } else {
7771                    runtime.start_requester = requester;
7772                    runtime.pending_activity_restart = true;
7773                }
7774            }
7775            true
7776        }
7777        SupervisorCommand::Reload { requester, reply } => {
7778            let result =
7779                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7780            if result.is_ok() {
7781                runtime.start_requester = requester;
7782                runtime.pending_activity_restart = true;
7783            }
7784            if result.is_ok()
7785                && runtime
7786                    .scheduled_respawn
7787                    .lock()
7788                    .unwrap_or_else(|p| p.into_inner())
7789                    .as_ref()
7790                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7791            {
7792                *runtime
7793                    .deferred_reload_reply
7794                    .lock()
7795                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7796            } else {
7797                let _ = reply.send(result);
7798            }
7799            true
7800        }
7801        SupervisorCommand::SetEnabled {
7802            enabled,
7803            requester,
7804            reply,
7805        } => {
7806            let previous_requester = runtime.start_requester.clone();
7807            if enabled {
7808                runtime.start_requester = requester.clone();
7809            }
7810            let result = set_child_enabled(
7811                spec,
7812                runtime,
7813                registry,
7814                process_liveness,
7815                snapshot,
7816                child,
7817                enabled,
7818                &runtime.failure_context(false),
7819            )
7820            .await;
7821            if !enabled {
7822                containment::terminate(&runtime.child_roster.containment).await;
7823            }
7824            if matches!(result, Ok(true)) {
7825                runtime.pending_activity_restart = false;
7826                if enabled {
7827                    runtime.start_requester = requester;
7828                } else {
7829                    runtime.start_requester = None;
7830                    runtime.record_activity_transition(
7831                        "subc.module_disabled",
7832                        &spec.module_id,
7833                        requester,
7834                    );
7835                }
7836            } else {
7837                runtime.start_requester = previous_requester;
7838            }
7839            let _ = reply.send(result);
7840            true
7841        }
7842        SupervisorCommand::UpdateConfiguration {
7843            spec: next_spec,
7844            health,
7845            drain_timeout_ms,
7846            reply,
7847        } => {
7848            if let Some(handle) = &runtime.supervisor_handle {
7849                handle.apply_identity_configuration(&next_spec);
7850            }
7851            *spec = next_spec;
7852            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7853                state.configuration_updated_since_spawn = true;
7854            });
7855            let health_changed = runtime.health != health;
7856            runtime.health = health;
7857            // Reset the cadence and old endpoint's failure streak on a live
7858            // health-policy change rather than waiting for its old deadline.
7859            if health_changed {
7860                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7861                    state.health = ModuleHealthStatus::default();
7862                });
7863            }
7864            runtime.drain_timeout = drain_timeout_ms
7865                .map(Duration::from_millis)
7866                .unwrap_or(runtime.default_drain_timeout);
7867            *runtime
7868                .effective_drain_timeout
7869                .lock()
7870                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7871            let _ = reply.send(());
7872            true
7873        }
7874        SupervisorCommand::Swap {
7875            requester,
7876            ready_timeout,
7877            reply,
7878        } => {
7879            if spec.carry_containment
7880                || containment::current(&runtime.child_roster.containment).is_some()
7881            {
7882                let _ = reply.send(Err(SuperviseError::InvalidSpec {
7883                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
7884                }));
7885                return true;
7886            }
7887            let mut swap_runtime = runtime.clone();
7888            swap_runtime.start_requester = requester;
7889            let before = lock_snapshot(snapshot)
7890                .ok()
7891                .map(|state| state.spawn_generation);
7892            let end = swap::run_swap(
7893                spec,
7894                &swap_runtime,
7895                registry,
7896                process_liveness,
7897                snapshot,
7898                child,
7899                commands,
7900                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7901                reply,
7902            )
7903            .await;
7904            if before
7905                != lock_snapshot(snapshot)
7906                    .ok()
7907                    .map(|state| state.spawn_generation)
7908            {
7909                runtime.start_requester = swap_runtime.start_requester;
7910            }
7911            requeued.extend(end.requeue);
7912            true
7913        }
7914    }
7915}
7916
7917async fn restart_child(
7918    spec: &ModuleSpec,
7919    runtime: &SupervisorRuntimeConfig,
7920    registry: &Registry,
7921    process_liveness: &SupervisorProcessLiveness,
7922    snapshot: &SharedSnapshot,
7923    child: &mut Option<SupervisedChild>,
7924    drain_timeout: Duration,
7925) -> Result<(), SuperviseError> {
7926    // Restart cycles a running module; it must not silently start a disabled one.
7927    if !lock_snapshot(snapshot)?.enabled {
7928        return Err(SuperviseError::Disabled {
7929            module_id: spec.module_id.clone(),
7930        });
7931    }
7932    let stop_notice = begin_forwarding_drain_with_timeout(
7933        spec,
7934        runtime,
7935        registry,
7936        snapshot,
7937        None,
7938        RouteCloseReason::Restart,
7939        drain_timeout,
7940    )
7941    .await?;
7942
7943    if child.is_some() {
7944        drain_optional_child(
7945            &spec.module_id,
7946            spec.protocol,
7947            stop_notice,
7948            registry,
7949            runtime.forwarding.as_deref(),
7950            snapshot,
7951            &runtime.terminal_ring,
7952            &runtime.spawn_events,
7953            child,
7954            drain_timeout,
7955            ModuleState::Restarting,
7956            Some(true),
7957            &FailureContext::operator(),
7958        )
7959        .await?;
7960    } else {
7961        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7962            state.enabled = true;
7963            state.state = ModuleState::Restarting;
7964            clear_current_process_facts(state);
7965        })?;
7966        release_dead_registration(
7967            registry,
7968            runtime.forwarding.as_deref(),
7969            snapshot,
7970            &spec.module_id,
7971            &runtime.failure_context(false),
7972        )
7973        .await?;
7974    }
7975
7976    reset_restart_count(snapshot, &spec.module_id)?;
7977    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7978    schedule_respawn(
7979        runtime,
7980        snapshot,
7981        &spec.module_id,
7982        runtime.restart_policy.backoff,
7983        RespawnKind::Spawn,
7984        false,
7985    )
7986}
7987
7988async fn reload_child(
7989    spec: &ModuleSpec,
7990    runtime: &SupervisorRuntimeConfig,
7991    registry: &Registry,
7992    process_liveness: &SupervisorProcessLiveness,
7993    snapshot: &SharedSnapshot,
7994    child: &mut Option<SupervisedChild>,
7995) -> Result<(), SuperviseError> {
7996    // Reload cycles a running module; it must not silently start a disabled one.
7997    if !lock_snapshot(snapshot)?.enabled {
7998        return Err(SuperviseError::Disabled {
7999            module_id: spec.module_id.clone(),
8000        });
8001    }
8002    let stop_notice = begin_forwarding_drain(
8003        spec,
8004        runtime,
8005        registry,
8006        snapshot,
8007        Some(true),
8008        RouteCloseReason::Reload,
8009    )
8010    .await?;
8011
8012    if child.is_some() {
8013        drain_optional_child(
8014            &spec.module_id,
8015            spec.protocol,
8016            stop_notice,
8017            registry,
8018            runtime.forwarding.as_deref(),
8019            snapshot,
8020            &runtime.terminal_ring,
8021            &runtime.spawn_events,
8022            child,
8023            runtime.drain_timeout,
8024            ModuleState::Restarting,
8025            Some(true),
8026            &FailureContext::operator(),
8027        )
8028        .await?;
8029    } else {
8030        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8031            state.enabled = true;
8032            state.state = ModuleState::Restarting;
8033            clear_current_process_facts(state);
8034        })?;
8035        release_dead_registration(
8036            registry,
8037            runtime.forwarding.as_deref(),
8038            snapshot,
8039            &spec.module_id,
8040            &runtime.failure_context(false),
8041        )
8042        .await?;
8043    }
8044
8045    reset_restart_count(snapshot, &spec.module_id)?;
8046    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8047    schedule_respawn(
8048        runtime,
8049        snapshot,
8050        &spec.module_id,
8051        runtime.restart_policy.backoff,
8052        RespawnKind::Reload,
8053        false,
8054    )
8055}
8056
8057async fn finish_reload_child(
8058    spec: &ModuleSpec,
8059    runtime: &SupervisorRuntimeConfig,
8060    registry: &Registry,
8061    process_liveness: &SupervisorProcessLiveness,
8062    snapshot: &SharedSnapshot,
8063    child: &mut Option<SupervisedChild>,
8064) -> Result<(), SuperviseError> {
8065    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8066    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8067        Ok(next_child) => next_child,
8068        Err(err) => {
8069            return handle_reload_spawn_failure(
8070                spec,
8071                runtime,
8072                process_liveness,
8073                snapshot,
8074                child,
8075                format!("new child failed to spawn: {err}"),
8076            )
8077            .await;
8078        }
8079    };
8080    *child = Some(next_child);
8081
8082    let wait_outcome = {
8083        let active_child = child.as_mut().expect("new reload child was just stored");
8084        wait_for_registration_after_reload(
8085            registry,
8086            &spec.module_id,
8087            snapshot,
8088            active_child,
8089            REGISTRY_RELEASE_TIMEOUT,
8090        )
8091        .await?
8092    };
8093
8094    match wait_outcome {
8095        RegistrationWaitOutcome::Registered => {
8096            child
8097                .as_mut()
8098                .expect("registered reload child")
8099                .registration_deadline = None;
8100            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8101            Ok(())
8102        }
8103        RegistrationWaitOutcome::Exited(exit_report) => {
8104            if let Some(active_child) = child.as_mut() {
8105                active_child.drain_stderr(&spec.module_id).await;
8106            }
8107            // Keep the exited child's roster guard until its terminal is written.
8108            // Shutdown waits on that guard, not on the child Option used for respawn.
8109            let mut exited_child = child.take().expect("exited reload child is still stored");
8110            #[cfg(test)]
8111            if let Some(gate) = &runtime.test_reload_exit_record_gate {
8112                gate.reached.notify_one();
8113                gate.resume.notified().await;
8114            }
8115            let result = handle_reload_child_registration_failure(
8116                spec,
8117                runtime,
8118                registry,
8119                process_liveness,
8120                snapshot,
8121                child,
8122                ReloadRegistrationFailure {
8123                    exit_report: registration_failure_exit_report(exit_report),
8124                    reason: exited_child
8125                        .spawn_failure
8126                        .clone()
8127                        .unwrap_or_else(|| "new child exited before registering".to_string()),
8128                },
8129            )
8130            .await;
8131            exited_child
8132                .finish_exit(false)
8133                .await
8134                .map_err(|source| SuperviseError::Wait {
8135                    module_id: spec.module_id.clone(),
8136                    source,
8137                })?;
8138            result
8139        }
8140        RegistrationWaitOutcome::TimedOut => {
8141            let mut timed_out_child = child
8142                .take()
8143                .expect("timed-out reload child is still running");
8144            timed_out_child
8145                .start_kill()
8146                .map_err(|source| SuperviseError::Kill {
8147                    module_id: spec.module_id.clone(),
8148                    source,
8149                })?;
8150            let status = timed_out_child
8151                .wait()
8152                .await
8153                .map_err(|source| SuperviseError::Wait {
8154                    module_id: spec.module_id.clone(),
8155                    source,
8156                })?;
8157            timed_out_child.drain_stderr(&spec.module_id).await;
8158            let result = handle_reload_child_registration_failure(
8159                spec,
8160                runtime,
8161                registry,
8162                process_liveness,
8163                snapshot,
8164                child,
8165                ReloadRegistrationFailure {
8166                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8167                        snapshot,
8168                        &timed_out_child,
8169                        &status,
8170                    )),
8171                    reason: format!(
8172                        "new child did not register within {:?}",
8173                        REGISTRY_RELEASE_TIMEOUT
8174                    ),
8175                },
8176            )
8177            .await;
8178            timed_out_child
8179                .finish_exit(false)
8180                .await
8181                .map_err(|source| SuperviseError::Wait {
8182                    module_id: spec.module_id.clone(),
8183                    source,
8184                })?;
8185            result
8186        }
8187    }
8188}
8189
8190#[allow(clippy::too_many_arguments)]
8191async fn set_child_enabled(
8192    spec: &ModuleSpec,
8193    runtime: &SupervisorRuntimeConfig,
8194    registry: &Registry,
8195    process_liveness: &SupervisorProcessLiveness,
8196    snapshot: &SharedSnapshot,
8197    child: &mut Option<SupervisedChild>,
8198    enabled: bool,
8199    failure: &FailureContext,
8200) -> Result<bool, SuperviseError> {
8201    let (current_enabled, current_state, respawn_pending) = {
8202        let state = lock_snapshot(snapshot)?;
8203        (state.enabled, state.state, state.respawn_pending)
8204    };
8205    // `start` (enable on an already-enabled module) heals TERMINAL states instead
8206    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
8207    // clean (Stopped) has no live process and no other in-band recovery — the
8208    // operator's start is the explicit recovery act and resets the budget. Without
8209    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
8210    // which the 2026-07-14 aft outage proved is a trap when the failed module is
8211    // the one providing every agent's shell.
8212    let revive_terminal = enabled
8213        && current_enabled
8214        && child.is_none()
8215        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8216            || (current_state == ModuleState::Restarting && !respawn_pending));
8217    if current_enabled == enabled && !revive_terminal {
8218        return Ok(false);
8219    }
8220
8221    if enabled {
8222        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8223            state.enabled = true;
8224            state.state = ModuleState::Starting;
8225            clear_current_process_facts(state);
8226        })?;
8227        #[cfg(test)]
8228        if runtime.test_seed_stale_facts_before_enable_spawn {
8229            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8230                state.process_alive = true;
8231                state.pid = Some(41);
8232                state.spawned_at_ms = Some(42);
8233                state.spawned_from = Some(PathBuf::from("/spawned/module"));
8234                state.spawned_file_identity = Some(SpawnedFileIdentity {
8235                    device: 43,
8236                    inode: 44,
8237                    #[cfg(windows)]
8238                    windows_image: None,
8239                });
8240            })?;
8241        }
8242        release_dead_registration(
8243            registry,
8244            runtime.forwarding.as_deref(),
8245            snapshot,
8246            &spec.module_id,
8247            failure,
8248        )
8249        .await?;
8250        reset_restart_count(snapshot, &spec.module_id)?;
8251        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8252        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8253            Ok(next_child) => next_child,
8254            Err(err) => {
8255                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8256                process_liveness.untrack_if_current(&spec.module_id, snapshot);
8257                return Err(err);
8258            }
8259        };
8260        *child = Some(next_child);
8261        debug!(module_id = %spec.module_id, "supervised module enabled");
8262        Ok(true)
8263    } else {
8264        let stop_notice = begin_forwarding_drain_if_configured(
8265            spec,
8266            runtime,
8267            registry,
8268            snapshot,
8269            Some(false),
8270            RouteCloseReason::Disable,
8271        )
8272        .await?;
8273        drain_optional_child(
8274            &spec.module_id,
8275            spec.protocol,
8276            stop_notice,
8277            registry,
8278            runtime.forwarding.as_deref(),
8279            snapshot,
8280            &runtime.terminal_ring,
8281            &runtime.spawn_events,
8282            child,
8283            runtime.drain_timeout,
8284            ModuleState::Disabled,
8285            Some(false),
8286            failure,
8287        )
8288        .await?;
8289        debug!(module_id = %spec.module_id, "supervised module disabled");
8290        Ok(true)
8291    }
8292}
8293
8294#[allow(clippy::too_many_arguments)]
8295async fn on_child_exit(
8296    spec: &ModuleSpec,
8297    policy: RestartPolicy,
8298    registry: &Registry,
8299    snapshot: &SharedSnapshot,
8300    terminal_ring: &Arc<Mutex<TerminalRing>>,
8301    spawn_events: &SpawnEventFeed,
8302    roster: &ChildRoster,
8303    exit_report: ExitReport,
8304    daemon_initiated: bool,
8305) -> NextAction {
8306    // Once the daemon has begun shutting down, no exit is a crash to recover
8307    // from: the module is exiting because the daemon is going away (EOF on its
8308    // connection, or a service manager signalling the whole cgroup). Record it
8309    // as such and never schedule a respawn, which would only start a process
8310    // for the shutdown to end again.
8311    if roster.is_closed() {
8312        return on_child_exit_during_daemon_shutdown(
8313            spec,
8314            registry,
8315            snapshot,
8316            terminal_ring,
8317            spawn_events,
8318            exit_report,
8319        )
8320        .await;
8321    }
8322    // Deliberate stops the supervisor asks for (operator stop, disable,
8323    // restart, reload, swap, a health restart, a drain that runs out of budget)
8324    // takes the child out of the supervise loop and reaps it in
8325    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8326    // that reaches this point is either unrequested or a first-HELLO timeout,
8327    // which is deliberately classified as a crash even after a clean TERM exit.
8328    //
8329    // For a subc-wire module a clean exit is still a stop: those modules are
8330    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8331    // crash, and exiting 0 is a deliberate choice the module made. A
8332    // `protocol: "none"` module is a stock program we cannot change, and many
8333    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8334    // stop would leave the module down for good after any stray signal, so it
8335    // goes through the crash path instead: it spends restart budget, respawns
8336    // with the crash backoff, and ends `failed` when the budget runs out.
8337    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8338        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8339    match exit_report.kind {
8340        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8341            info!(
8342                module_id = %spec.module_id,
8343                exit_code = ?exit_report.code,
8344                exit_signal = ?exit_report.signal,
8345                "supervised module exited cleanly"
8346            );
8347            if let Err(err) = publish_after_terminal_record(
8348                &spec.module_id,
8349                || {
8350                    record_terminal(
8351                        &spec.module_id,
8352                        terminal_ring,
8353                        spawn_events,
8354                        &exit_report,
8355                        TerminalDisposition::Stopped,
8356                    )
8357                },
8358                || {
8359                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8360                        state.state = ModuleState::Stopped;
8361                        clear_current_process_facts(state);
8362                        state.last_exit = Some(exit_report.clone());
8363                    })
8364                },
8365            ) {
8366                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8367            }
8368            let registration_released = match wait_for_registration_release(
8369                registry,
8370                &spec.module_id,
8371                REGISTRY_RELEASE_TIMEOUT,
8372            )
8373            .await
8374            {
8375                Ok(()) => true,
8376                Err(err) => {
8377                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8378                    false
8379                }
8380            };
8381            NextAction::Stop {
8382                registration_released,
8383            }
8384        }
8385        ExitKind::Clean | ExitKind::Crash => {
8386            if unrequested_clean_exit_of_protocol_none {
8387                warn!(
8388                    module_id = %spec.module_id,
8389                    exit_code = ?exit_report.code,
8390                    exit_signal = ?exit_report.signal,
8391                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8392                );
8393            } else {
8394                warn!(
8395                    module_id = %spec.module_id,
8396                    exit_code = ?exit_report.code,
8397                    exit_signal = ?exit_report.signal,
8398                    "supervised module exited abnormally (crash)"
8399                );
8400            }
8401            let mut restart_schedule = None;
8402            let mut disposition = TerminalDisposition::Disabled;
8403            let mut final_state = ModuleState::Disabled;
8404            // Set only when the budget is what stopped the module, so the
8405            // terminal record says which limit was hit rather than leaving
8406            // `failed` to be read as "crashed once, badly".
8407            let mut disposition_detail = lock_snapshot(snapshot)
8408                .ok()
8409                .and_then(|mut state| state.spawn_failure.take());
8410            let now = Instant::now();
8411            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8412                if state.enabled {
8413                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8414                        // Reserving the last slot exhausts the budget for the
8415                        // replacement, not for this exit. Publish its scheduled
8416                        // recovery atomically with the reservation so connection
8417                        // cleanup sees it while terminal recording still holds
8418                        // the visible state at Running.
8419                        state.respawn_pending = true;
8420                        final_state = ModuleState::Restarting;
8421                        restart_schedule = Some(schedule);
8422                        disposition = TerminalDisposition::Restarting;
8423                    } else {
8424                        final_state = ModuleState::Failed;
8425                        disposition = TerminalDisposition::Failed;
8426                        let budget = policy.budget_exhausted_detail();
8427                        disposition_detail =
8428                            Some(disposition_detail.take().map_or_else(
8429                                || budget.clone(),
8430                                |cause| format!("{cause}; {budget}"),
8431                            ));
8432                    }
8433                } else {
8434                    disposition = TerminalDisposition::Disabled;
8435                }
8436            }) {
8437                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8438                return NextAction::Stop {
8439                    registration_released: false,
8440                };
8441            }
8442            if disposition == TerminalDisposition::Failed {
8443                // The window is in the message, not only in the fields: this line
8444                // is read in a scrollback where a bare `max_restarts=3` reads as a
8445                // lifetime cap and sends the operator looking for three crashes
8446                // that never happened together.
8447                error!(
8448                    module_id = %spec.module_id,
8449                    max_restarts = policy.max_restarts,
8450                    window_secs = policy.window.as_secs(),
8451                    "module stopped: {}",
8452                    policy.budget_exhausted_detail()
8453                );
8454            }
8455            if let Err(err) = publish_after_terminal_record(
8456                &spec.module_id,
8457                || {
8458                    record_terminal_with_detail(
8459                        &spec.module_id,
8460                        terminal_ring,
8461                        spawn_events,
8462                        &exit_report,
8463                        disposition,
8464                        disposition_detail,
8465                    )
8466                },
8467                || {
8468                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8469                        state.state = final_state;
8470                        clear_current_process_facts(state);
8471                        if final_state == ModuleState::Failed
8472                            && daemon_initiated
8473                            && !roster.is_closed()
8474                        {
8475                            state.failed_retry.schedule(&spec.module_id);
8476                        }
8477                        state.last_exit = Some(exit_report.clone());
8478                    })
8479                },
8480            ) {
8481                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8482            }
8483
8484            if let Some(schedule) = restart_schedule {
8485                NextAction::Restart {
8486                    schedule: Some(schedule),
8487                }
8488            } else {
8489                let registration_released = match wait_for_registration_release(
8490                    registry,
8491                    &spec.module_id,
8492                    REGISTRY_RELEASE_TIMEOUT,
8493                )
8494                .await
8495                {
8496                    Ok(()) => true,
8497                    Err(err) => {
8498                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8499                        false
8500                    }
8501                };
8502                NextAction::Stop {
8503                    registration_released,
8504                }
8505            }
8506        }
8507        ExitKind::DeliberateSeverance => {
8508            warn!(
8509                module_id = %spec.module_id,
8510                exit_code = ?exit_report.code,
8511                exit_signal = ?exit_report.signal,
8512                "supervised module exited after deliberate connection severance"
8513            );
8514            let should_restart = match lock_snapshot(snapshot) {
8515                Ok(state) => state.enabled,
8516                Err(err) => {
8517                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8518                    return NextAction::Stop {
8519                        registration_released: false,
8520                    };
8521                }
8522            };
8523            let final_state = if should_restart {
8524                ModuleState::Restarting
8525            } else {
8526                ModuleState::Disabled
8527            };
8528            if let Err(err) = publish_after_terminal_record(
8529                &spec.module_id,
8530                || {
8531                    record_terminal(
8532                        &spec.module_id,
8533                        terminal_ring,
8534                        spawn_events,
8535                        &exit_report,
8536                        terminal_disposition(final_state),
8537                    )
8538                },
8539                || {
8540                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8541                        state.state = final_state;
8542                        clear_current_process_facts(state);
8543                        state.last_exit = Some(exit_report.clone());
8544                        state.lifetime_restarts += 1;
8545                    })
8546                },
8547            ) {
8548                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8549                return NextAction::Stop {
8550                    registration_released: false,
8551                };
8552            }
8553
8554            if should_restart {
8555                NextAction::Restart { schedule: None }
8556            } else {
8557                let registration_released = match wait_for_registration_release(
8558                    registry,
8559                    &spec.module_id,
8560                    REGISTRY_RELEASE_TIMEOUT,
8561                )
8562                .await
8563                {
8564                    Ok(()) => true,
8565                    Err(err) => {
8566                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8567                        false
8568                    }
8569                };
8570                NextAction::Stop {
8571                    registration_released,
8572                }
8573            }
8574        }
8575    }
8576}
8577
8578async fn on_child_exit_during_daemon_shutdown(
8579    spec: &ModuleSpec,
8580    registry: &Registry,
8581    snapshot: &SharedSnapshot,
8582    terminal_ring: &Arc<Mutex<TerminalRing>>,
8583    spawn_events: &SpawnEventFeed,
8584    exit_report: ExitReport,
8585) -> NextAction {
8586    info!(
8587        module_id = %spec.module_id,
8588        exit_code = ?exit_report.code,
8589        exit_signal = ?exit_report.signal,
8590        exit_kind = ?exit_report.kind,
8591        "supervised module exited during daemon shutdown; not restarting it"
8592    );
8593    if let Err(err) = publish_after_terminal_record(
8594        &spec.module_id,
8595        || {
8596            record_terminal(
8597                &spec.module_id,
8598                terminal_ring,
8599                spawn_events,
8600                &exit_report,
8601                TerminalDisposition::DaemonShutdown,
8602            )
8603        },
8604        || {
8605            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8606                state.state = ModuleState::Stopped;
8607                clear_current_process_facts(state);
8608                state.last_exit = Some(exit_report.clone());
8609            })
8610        },
8611    ) {
8612        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8613    }
8614    let registration_released =
8615        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8616            .await
8617            .is_ok();
8618    NextAction::Stop {
8619        registration_released,
8620    }
8621}
8622
8623// Keep the status lock out of terminal I/O. Readers may observe the previous
8624// state while recording is in progress, but never a completed exit without its
8625// history. Journal errors are logged by the journal and retained in the ring;
8626// a recording panic still publishes the final state before resuming the unwind.
8627fn publish_after_terminal_record<T>(
8628    module_id: &str,
8629    record: impl FnOnce(),
8630    publish: impl FnOnce() -> T,
8631) -> T {
8632    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8633    let published = publish();
8634    if let Err(panic) = recorded {
8635        error!(
8636            module_id,
8637            "terminal recording panicked; published final state before resuming unwind"
8638        );
8639        std::panic::resume_unwind(panic);
8640    }
8641    published
8642}
8643
8644fn fail_after_wait_error(
8645    module_id: &str,
8646    snapshot: &SharedSnapshot,
8647    terminal_ring: &Arc<Mutex<TerminalRing>>,
8648    spawn_events: &SpawnEventFeed,
8649    detail: String,
8650    failure: &FailureContext,
8651) {
8652    let report = wait_error_exit_report();
8653    publish_after_terminal_record(
8654        module_id,
8655        || {
8656            record_terminal_with_detail(
8657                module_id,
8658                terminal_ring,
8659                spawn_events,
8660                &report,
8661                TerminalDisposition::Failed,
8662                Some(detail),
8663            )
8664        },
8665        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8666    );
8667}
8668
8669#[cfg(test)]
8670fn record_wait_error_terminal(
8671    module_id: &str,
8672    terminal_ring: &Arc<Mutex<TerminalRing>>,
8673    spawn_events: &SpawnEventFeed,
8674) {
8675    record_terminal(
8676        module_id,
8677        terminal_ring,
8678        spawn_events,
8679        &wait_error_exit_report(),
8680        TerminalDisposition::Failed,
8681    );
8682}
8683
8684fn record_terminal(
8685    module_id: &str,
8686    terminal_ring: &Arc<Mutex<TerminalRing>>,
8687    spawn_events: &SpawnEventFeed,
8688    exit_report: &ExitReport,
8689    disposition: TerminalDisposition,
8690) {
8691    record_terminal_with_detail(
8692        module_id,
8693        terminal_ring,
8694        spawn_events,
8695        exit_report,
8696        disposition,
8697        None,
8698    );
8699}
8700
8701/// The ring lock is held only to capture the read (see
8702/// `TerminalJournal::capture_read`), so this module's exits keep recording
8703/// while the journal files are read. Blocking: it reads files.
8704fn durable_terminal_history_of(
8705    terminal_ring: &Mutex<TerminalRing>,
8706    module_id: &str,
8707) -> subc_control::TerminalHistory {
8708    let read = terminal_ring
8709        .lock()
8710        .unwrap_or_else(|p| p.into_inner())
8711        .capture_durable_history();
8712    read.read(module_id)
8713}
8714
8715fn record_terminal_with_detail(
8716    module_id: &str,
8717    terminal_ring: &Arc<Mutex<TerminalRing>>,
8718    spawn_events: &SpawnEventFeed,
8719    exit_report: &ExitReport,
8720    disposition: TerminalDisposition,
8721    disposition_detail: Option<String>,
8722) {
8723    #[cfg(test)]
8724    terminal_history_tests::before_terminal_record(spawn_events);
8725    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8726    let record = TerminalRecord {
8727        exit_code: exit_report.code,
8728        exit_signal: exit_report.signal,
8729        at_ms: exit_report.at_ms,
8730        disposition,
8731        exit_kind: exit_report.kind.into(),
8732        disposition_detail,
8733    };
8734    terminal_ring
8735        .lock()
8736        .unwrap_or_else(|poisoned| poisoned.into_inner())
8737        .record_exit(module_id, record);
8738}
8739
8740fn untrack_if_registration_released(
8741    process_liveness: &SupervisorProcessLiveness,
8742    registry: &Registry,
8743    module_id: &str,
8744    snapshot: &SharedSnapshot,
8745) {
8746    match registry.get_module(module_id) {
8747        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8748        Ok(Some(_)) => {}
8749        Err(err) => {
8750            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8751        }
8752    }
8753}
8754
8755/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8756/// then apply the module's configured entries minus daemon-private capture keys.
8757///
8758/// Separated from `spawn_child` only so it can be asserted without spawning a
8759/// process — a duplicate of this logic in a test would pass while the real one
8760/// drifted, which is the defect class this function exists to avoid.
8761/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8762/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8763/// either and the argument would stop a stock binary from starting at all.
8764/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8765///
8766/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8767/// through [`apply_wire_spawn_args_for_role`].
8768#[cfg(test)]
8769fn apply_wire_spawn_args(
8770    command: &mut Command,
8771    spec: &ModuleSpec,
8772    connection_file_path: Option<&std::path::Path>,
8773    handle: Option<&SupervisorHandle>,
8774) -> Result<Option<NonceHandoff>, SuperviseError> {
8775    apply_wire_spawn_args_for_role(
8776        command,
8777        spec,
8778        connection_file_path,
8779        handle,
8780        SpawnRole::Plain,
8781    )
8782}
8783
8784/// The read end of a spawn's launch-nonce pipe, prepared by
8785/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8786/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8787/// server before spawn and authenticates its client after the child exists.
8788#[cfg(unix)]
8789type NonceHandoff = subc_os::LaunchNonceHandoff;
8790#[cfg(windows)]
8791type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8792#[cfg(not(any(unix, windows)))]
8793type NonceHandoff = std::convert::Infallible;
8794
8795/// Create the launch nonce that a new module process must present when it
8796/// registers or opens a consumer connection, including a replacement process
8797/// started alongside a still-serving module.
8798///
8799/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8800/// a separate candidate token so the still-serving incumbent and its consumers
8801/// keep their nonce. Both records are installed before the process exists, so
8802/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8803///
8804/// On Unix the nonce is delivered only through a pipe. It is written into
8805/// a pipe whose read end the child gets as descriptor 3, named by
8806/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8807/// process of the same user cannot read it with `ps eww`. That handoff is
8808/// returned rather than installed here, because installing it replaces
8809/// whatever the child has at descriptor 3 and so must be the last pre-exec
8810/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8811/// the Linux callback that writes the process into its cgroup. Only after that
8812/// does `handoff.install_last` register the descriptor-3 replacement, so it
8813/// cannot close a descriptor the cgroup callback still needs.
8814/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8815fn apply_wire_spawn_args_for_role(
8816    command: &mut Command,
8817    spec: &ModuleSpec,
8818    connection_file_path: Option<&std::path::Path>,
8819    handle: Option<&SupervisorHandle>,
8820    role: SpawnRole,
8821) -> Result<Option<NonceHandoff>, SuperviseError> {
8822    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8823    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8824    // included: a daemon started from a module's process tree inherits it,
8825    // and passing it on would point the child at a descriptor it does not
8826    // have.
8827    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8828    #[cfg(windows)]
8829    {
8830        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8831        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8832    }
8833    // Remove inherited or configured copies too: withholding must mean absent.
8834    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8835    if spec.protocol == ModuleProtocol::None {
8836        return Ok(None);
8837    }
8838    if let Some(connection_file_path) = connection_file_path {
8839        command.arg(SUBC_ARG).arg(connection_file_path);
8840    }
8841
8842    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8843    // route.open attestation. Reserved modules additionally use the same nonce
8844    // for HELLO id-squatting protection. A respawn rotates both records.
8845    let nonce = generate_launch_nonce()?;
8846    if let Some(handle) = handle {
8847        match role {
8848            SpawnRole::Plain => {
8849                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8850                if spec.reserved {
8851                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8852                }
8853            }
8854            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8855        }
8856    }
8857    #[cfg(unix)]
8858    let handoff = {
8859        let handoff =
8860            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8861                program: spec.program.clone(),
8862                source,
8863                cgroup_path: None,
8864            })?;
8865        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8866        Some(handoff)
8867    };
8868    #[cfg(windows)]
8869    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8870    #[cfg(not(any(unix, windows)))]
8871    let handoff = None;
8872    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8873    // until the daemon's per-spawn delivery records show that every running
8874    // Windows module has read its secret from the named pipe.
8875    #[cfg(not(unix))]
8876    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8877    Ok(handoff)
8878}
8879
8880fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8881    command.env_remove(CK_LOG_ENV);
8882    // The spawn role is the supervisor's to set, and only on a swap candidate
8883    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8884    // it, is what makes it absent on a plain spawn: the daemon's own
8885    // environment could carry it, and so could a spec built outside daemon
8886    // config (config refuses it as an `env` key). A module reading it on a
8887    // plain restart would pick the long swap budget and leave callers waiting.
8888    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8889    for (key, value) in &spec.env {
8890        // cortexkit-log currently exposes retention only as a Rust struct, not
8891        // environment names. These values are daemon-private sink metadata and
8892        // must never become a public child-process contract by being inherited.
8893        if matches!(
8894            key.as_str(),
8895            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8896        ) || key == SUBC_SPAWN_ROLE_ENV
8897        {
8898            continue;
8899        }
8900        command.env(key, value);
8901    }
8902}
8903
8904#[cfg(windows)]
8905fn prepare_windows_nonce_pipe(
8906    command: &mut Command,
8907    spec: &ModuleSpec,
8908    nonce: &str,
8909) -> Option<NonceHandoff> {
8910    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8911        Ok(handoff) => {
8912            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8913            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8914            // still register until every module reads the pipe. A reader without
8915            // this variable set to env returns the pipe error instead of using
8916            // an inherited or configured environment copy.
8917            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8918            Some(handoff)
8919        }
8920        Err(error) => {
8921            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8922            None
8923        }
8924    }
8925}
8926
8927#[cfg(windows)]
8928fn start_windows_nonce_pipe(
8929    handoff: Option<NonceHandoff>,
8930    pid: u32,
8931    spec: &ModuleSpec,
8932) -> Option<subc_os::LaunchNoncePipeDelivery> {
8933    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8934    // same maximum wait used for registration during a reload. Older modules
8935    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8936    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8937        Ok(delivery) => Some(delivery),
8938        Err(error) => {
8939            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8940            None
8941        }
8942    })
8943}
8944
8945fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8946    #[cfg(windows)]
8947    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8948        return Some(
8949            if snapshot
8950                .pipe_consumed
8951                .as_ref()
8952                .is_some_and(|consumed| consumed.get().is_some())
8953            {
8954                "pipe"
8955            } else {
8956                "env"
8957            }
8958            .to_owned(),
8959        );
8960    }
8961    let _ = snapshot;
8962    None
8963}
8964
8965/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8966/// of a blue/green swap.
8967#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8968enum SpawnRole {
8969    Plain,
8970    SwapCandidate,
8971}
8972
8973/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8974/// `apply_child_env` has already removed the variable for every spawn.
8975fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8976    if role == SpawnRole::SwapCandidate {
8977        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8978    }
8979}
8980
8981fn previous_instance_error(
8982    spec: &ModuleSpec,
8983    pid: u32,
8984    reason: crate::live_children::RetainedReason,
8985) -> SuperviseError {
8986    SuperviseError::Spawn {
8987        program: spec.program.clone(),
8988        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8989        cgroup_path: None,
8990    }
8991}
8992
8993fn carries_containment(configured: bool, has_container: bool) -> bool {
8994    configured || has_container
8995}
8996
8997fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
8998    if !carries_containment(
8999        spec.carry_containment,
9000        containment::current(&roster.containment).is_some(),
9001    ) {
9002        return None;
9003    }
9004    Some(SuperviseError::Spawn {
9005        program: spec.program.clone(),
9006        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9007        cgroup_path: None,
9008    })
9009}
9010
9011fn spawn_child(
9012    spec: &ModuleSpec,
9013    connection_file_path: Option<&std::path::Path>,
9014    handle: Option<&SupervisorHandle>,
9015    ring: &Arc<Mutex<StderrRing>>,
9016    capture_logs_dir: Option<&std::path::Path>,
9017    roster: &ChildRoster,
9018    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9019) -> Result<SupervisedChild, SuperviseError> {
9020    #[cfg(all(test, unix))]
9021    if let Some(source) = containment_tests::join_failure(spec, roster) {
9022        containment::terminate_blocking(&roster.containment);
9023        return Err(SuperviseError::Spawn {
9024            program: spec.program.clone(),
9025            source,
9026            cgroup_path: None,
9027        });
9028    }
9029    let result = spawn_child_in_slot(
9030        spec,
9031        connection_file_path,
9032        handle,
9033        ring,
9034        capture_logs_dir,
9035        roster,
9036        #[cfg(target_os = "linux")]
9037        cgroup_placement,
9038        SpawnRole::Plain,
9039        false,
9040    );
9041    if result.is_err() {
9042        if roster.admission_pending(&spec.module_id) {
9043            containment::terminate_deferred(&roster.containment);
9044        } else {
9045            containment::terminate_blocking(&roster.containment);
9046        }
9047    }
9048    result
9049}
9050
9051/// Spawn one process of `spec` into a slot.
9052///
9053/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
9054/// A swap candidate needs a different cgroup from the process it is replacing,
9055/// which is still alive: in the same cgroup the two would be one kill domain,
9056/// and killing a failed candidate could take the incumbent with it.
9057///
9058/// The stderr capture file is `<module_id>.stderr.log` for every process of
9059/// the module, whichever slot it is in, because that is the one file
9060/// `ck module logs` reads. During a swap's overlap both processes append to it;
9061/// the daemon writes whole lines, so the two interleave by line, which is also
9062/// the merged view an operator wants while a swap runs.
9063#[allow(clippy::too_many_arguments)]
9064fn spawn_child_in_slot(
9065    spec: &ModuleSpec,
9066    connection_file_path: Option<&std::path::Path>,
9067    handle: Option<&SupervisorHandle>,
9068    ring: &Arc<Mutex<StderrRing>>,
9069    capture_logs_dir: Option<&std::path::Path>,
9070    roster: &ChildRoster,
9071    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9072    role: SpawnRole,
9073    alternate_slot: bool,
9074) -> Result<SupervisedChild, SuperviseError> {
9075    if let Some(error) = recovery_block_error(spec, roster) {
9076        return Err(error);
9077    }
9078    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9079        return Err(previous_instance_error(spec, pid, reason));
9080    }
9081    if roster.is_closed() {
9082        return Err(SuperviseError::Spawn {
9083            program: spec.program.clone(),
9084            source: io::Error::other("the daemon is shutting down; not starting a new process"),
9085            cgroup_path: None,
9086        });
9087    }
9088    let carried = containment::current(&roster.containment);
9089    let carrying = carries_containment(spec.carry_containment, carried.is_some());
9090    if role != SpawnRole::Plain && carrying {
9091        return Err(SuperviseError::InvalidSpec {
9092            reason: "a carried container cannot overlap a swap candidate".to_owned(),
9093        });
9094    }
9095    #[cfg(unix)]
9096    let launch_identity = if carrying && carried.is_none() {
9097        Some(
9098            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9099                program: spec.program.clone(),
9100                source,
9101                cgroup_path: None,
9102            })?,
9103        )
9104    } else {
9105        None
9106    };
9107    #[cfg(target_os = "linux")]
9108    let cgroup_placement = match &carried {
9109        Some(container) => container.cgroup().map(|(placement, _)| placement),
9110        None => cgroup_placement,
9111    };
9112    #[cfg(unix)]
9113    let group_target = if let Some(container) = carried
9114        .as_ref()
9115        .filter(|container| container.group().is_some())
9116    {
9117        containment::GroupTarget::Pinned(Arc::clone(container))
9118    } else {
9119        #[cfg(target_os = "linux")]
9120        let cgroup_container = carrying && cgroup_placement.is_some();
9121        #[cfg(not(target_os = "linux"))]
9122        let cgroup_container = false;
9123        if cgroup_container {
9124            containment::GroupTarget::NoGroup
9125        } else {
9126            containment::GroupTarget::OwnLeader
9127        }
9128    };
9129    #[cfg(target_os = "linux")]
9130    let cgroup_name = {
9131        // Slot names alone are not kill domains: a retired incumbent may still
9132        // be draining when a later enable/restart spawns into the same slot.
9133        // Decimal entropy keeps the suffix unambiguous; Placement performs
9134        // the module-id escaping and constructs the filesystem path.
9135        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9136            name.to_owned()
9137        } else if cgroup_placement.is_none() {
9138            swap::cgroup_name(&spec.module_id, alternate_slot)
9139        } else {
9140            let nonce = generate_launch_nonce()?;
9141            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9142            // Leave room for byte escaping and the suffix under NAME_MAX. The
9143            // label is only for humans; the nonce identifies the kill domain.
9144            let mut end = spec.module_id.len().min(64);
9145            while !spec.module_id.is_char_boundary(end) {
9146                end -= 1;
9147            }
9148            format!(
9149                "{}_{suffix}",
9150                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9151            )
9152        }
9153    };
9154    #[cfg(not(target_os = "linux"))]
9155    let _ = alternate_slot;
9156    #[cfg(target_os = "macos")]
9157    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9158    #[cfg(windows)]
9159    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9160        program: spec.program.clone(),
9161        source,
9162        cgroup_path: None,
9163    })?;
9164    #[cfg(windows)]
9165    let mut command = Command::new(image_capture.path());
9166    #[cfg(not(any(target_os = "macos", windows)))]
9167    let mut command = Command::new(&spec.program);
9168    command.args(&spec.args);
9169    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
9170    // that is the whole of the intent, so remove that one key rather than the
9171    // environment.
9172    //
9173    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
9174    // and took the POSIX environment with it. Modules spawned that way had no
9175    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
9176    // logging:
9177    //
9178    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
9179    //     both unset it fell back to the temp dir alone and `ck` could not find
9180    //     a daemon running on the same machine from inside any module's process
9181    //     tree — reporting a path the file has never lived at, which reads as
9182    //     "the daemon did not write its file".
9183    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
9184    //     the RELATIVE `.local/share`, so a module deriving its own store path
9185    //     resolved it against its own CWD. That is the store-fragmentation
9186    //     defect the daemon already refuses in config (`parse_doc` rejects a
9187    //     relative `storage.data_home`) arriving by derivation instead.
9188    //   * anything a module spawns inherited it: git without ~/.gitconfig,
9189    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
9190    //     quietly rather than erroring.
9191    //
9192    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
9193    // offered one candidate under /tmp while the file sat in /run/user/1000.
9194    //
9195    // A configured module is unaffected either way: `module_spec()` puts the
9196    // resolved CK_LOG into `spec.env`, which is applied below and therefore
9197    // wins over anything ambient.
9198    apply_child_env(&mut command, spec);
9199    apply_spawn_role(&mut command, role);
9200    let nonce_handoff =
9201        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9202
9203    #[cfg(target_os = "linux")]
9204    let cgroup_path = cgroup_placement
9205        .map(|placement| {
9206            if let Some(container) = &carried {
9207                let path = container
9208                    .cgroup_path()
9209                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9210                placement.existing_module_path(path)?.ok_or_else(|| {
9211                    io::Error::new(
9212                        io::ErrorKind::NotFound,
9213                        "carried cgroup disappeared before replacement",
9214                    )
9215                })
9216            } else {
9217                if carrying {
9218                    containment::prepare_cgroup_path(placement, &cgroup_name)
9219                } else {
9220                    placement.module_path(&cgroup_name)
9221                }
9222            }
9223        })
9224        .transpose()
9225        .map_err(|source| SuperviseError::Cgroup {
9226            module_id: spec.module_id.clone(),
9227            source,
9228        })?;
9229    #[cfg(not(target_os = "linux"))]
9230    let cgroup_path: Option<PathBuf> = None;
9231    #[cfg(unix)]
9232    let mut admission_gate = if carrying {
9233        Some(
9234            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9235                SuperviseError::Spawn {
9236                    program: spec.program.clone(),
9237                    source,
9238                    cgroup_path: cgroup_path.clone(),
9239                }
9240            })?,
9241        )
9242    } else {
9243        None
9244    };
9245    #[cfg(target_os = "macos")]
9246    exec_ack.install(command.as_std_mut());
9247    #[cfg(target_os = "linux")]
9248    if let Some(path) = &cgroup_path {
9249        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9250            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9251                remove_module_cgroup(placement, &cgroup_name);
9252            }
9253            return Err(error);
9254        }
9255    }
9256
9257    let output_sink = if let Some(logs_dir) = capture_logs_dir {
9258        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9259        match ChildOutputSink::open(&path, capture_retention(spec)) {
9260            Ok(sink) => sink,
9261            Err(error) => {
9262                warn!(
9263                    module_id = %spec.module_id,
9264                    path = %path.display(),
9265                    error = %error,
9266                    "could not open child output capture file; forwarding to stderr"
9267                );
9268                ChildOutputSink::Stderr
9269            }
9270        }
9271    } else {
9272        ChildOutputSink::Stderr
9273    };
9274
9275    command.stdout(Stdio::piped());
9276    command.stderr(Stdio::piped());
9277    command.kill_on_drop(!carrying);
9278    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
9279    // before exec). In the daemon's group, a service manager that kills the
9280    // job's process group when the daemon exits (launchd's default) killed
9281    // every module at the same moment its control connection closed, so no
9282    // module ever ran its EOF teardown on a daemon stop. Outside that group a
9283    // module is reached only by the daemon: the EOF it sees when its
9284    // connection closes, and the bounded stop in `child_roster` for anything
9285    // still running after that. On Linux this composes with the cgroup
9286    // placement above: that is a pre_exec write to cgroup.procs, std performs
9287    // setpgid in the child before running pre_exec callbacks, and the two
9288    // change independent process attributes.
9289    //
9290    // stdin is /dev/null because a process outside the terminal's foreground
9291    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
9292    // by hand would otherwise hand down. Under a service manager stdin is
9293    // already /dev/null.
9294    #[cfg(unix)]
9295    command.process_group(
9296        carried
9297            .as_ref()
9298            .map(|container| container.verified_join_group())
9299            .transpose()
9300            .map_err(|source| SuperviseError::Spawn {
9301                program: spec.program.clone(),
9302                source,
9303                cgroup_path: cgroup_path.clone(),
9304            })?
9305            .flatten()
9306            .unwrap_or(0) as i32,
9307    );
9308    command.stdin(Stdio::null());
9309    #[cfg(unix)]
9310    if let Some(gate) = &mut admission_gate {
9311        gate.install_placement(&mut command);
9312    }
9313    // The LAST pre-exec step, after the cgroup placement above: installing the
9314    // nonce at descriptor 3 replaces whatever the child had there, which could
9315    // be the descriptor an earlier step writes through.
9316    #[cfg(unix)]
9317    if let Some(handoff) = nonce_handoff {
9318        handoff.install_last(command.as_std_mut());
9319    }
9320    #[cfg(not(any(unix, windows)))]
9321    let _ = nonce_handoff;
9322
9323    // Containment, step 1 of 3 (issue #109): create the child suspended so it
9324    // cannot run a single instruction -- and therefore cannot spawn a
9325    // grandchild -- before it is in the job. See `contain_spawned_child` for the
9326    // other two steps and why the window matters.
9327    #[cfg(windows)]
9328    {
9329        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9330            program: spec.program.clone(),
9331            source,
9332            cgroup_path: None,
9333        })?;
9334        subc_jobobject::suspend_on_create_async(&mut command);
9335    }
9336    // Timestamp before the spawn call: on Unix the child can serve before that
9337    // call even returns. This conservative bound never labels an older child
9338    // as a replacement for a request received during process creation.
9339    let spawned_at = Instant::now();
9340    #[cfg(all(test, unix))]
9341    containment_tests::record_spawn_attempt(spec, &mut command);
9342    #[cfg(unix)]
9343    let spawned = match admission_gate {
9344        Some(gate) => gate
9345            .spawn(
9346                command,
9347                spec,
9348                roster,
9349                launch_identity,
9350                #[cfg(target_os = "linux")]
9351                cgroup_placement
9352                    .zip(cgroup_path.as_ref())
9353                    .map(|(placement, path)| {
9354                        (placement.clone(), cgroup_name.clone(), path.clone())
9355                    }),
9356            )
9357            .map(|(child, admitted)| (child, Some(admitted))),
9358        None => command.spawn().map(|child| (child, None)),
9359    };
9360    #[cfg(not(unix))]
9361    let spawned = command.spawn();
9362    let spawned = match spawned {
9363        Ok(child) => child,
9364        Err(source) => {
9365            if carried
9366                .as_ref()
9367                .and_then(|container| container.group())
9368                .is_some()
9369            {
9370                containment::terminate_blocking(&roster.containment);
9371            }
9372            #[cfg(target_os = "linux")]
9373            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9374                remove_module_cgroup(placement, &cgroup_name);
9375            }
9376            return Err(SuperviseError::Spawn {
9377                program: spec.program.clone(),
9378                source,
9379                cgroup_path,
9380            });
9381        }
9382    };
9383    #[cfg(unix)]
9384    let (mut child, mut admitted) = spawned;
9385    #[cfg(unix)]
9386    let group_target = admitted
9387        .as_ref()
9388        .map(|attempt| attempt.group_target())
9389        .unwrap_or(group_target);
9390    #[cfg(not(unix))]
9391    let mut child = spawned;
9392    #[cfg(test)]
9393    spawn_record_tests::after_create(&child, spec);
9394    // The parent must close its writer now: the acknowledgement pipe reports EOF
9395    // only when every writer is gone, and the child's copy closes when the
9396    // trampoline replaces itself with the module. Command holds only an integer
9397    // in its pre_exec callback, not another writer.
9398    #[cfg(target_os = "macos")]
9399    drop(exec_ack);
9400
9401    // Bind while the child is still suspended, before job assignment resumes it.
9402    // Keeping the executable handle open across CreateProcessW denies rename
9403    // and deletion, so the file cannot be exchanged during spawn. Bind that
9404    // file ID to the process handle returned by spawn, never a reopened PID.
9405    #[cfg(windows)]
9406    let windows_image = match image_capture.bind_tokio(&child) {
9407        Ok(image) => Arc::new(image),
9408        Err(source) => {
9409            kill_and_reap_rejected_child(&mut child);
9410            return Err(SuperviseError::Spawn {
9411                program: spec.program.clone(),
9412                source,
9413                cgroup_path: None,
9414            });
9415        }
9416    };
9417    // Assign the suspended child to its module job; resume only after its
9418    // process identity has been durably recorded for orphan recovery.
9419    #[cfg(windows)]
9420    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9421    let spawned_at_ms = unix_ms_now();
9422    #[cfg(not(windows))]
9423    let spawned_from = spec.program.clone();
9424    #[cfg(windows)]
9425    let spawned_from = windows_image.path().to_path_buf();
9426    #[cfg(not(windows))]
9427    let spawned_file_identity = spawned_file_identity(&spawned_from);
9428    #[cfg(windows)]
9429    let spawned_file_identity = Some(SpawnedFileIdentity {
9430        device: 0,
9431        inode: 0,
9432        windows_image: Some(Arc::clone(&windows_image)),
9433    });
9434    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9435        program: spec.program.clone(),
9436        source: io::Error::other("spawned child exposed no live pid"),
9437        cgroup_path: cgroup_path.clone(),
9438    })?;
9439    #[cfg(windows)]
9440    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9441    #[cfg(not(windows))]
9442    let process_start_time = admitted
9443        .as_ref()
9444        .map(|admitted| admitted.birth)
9445        .or_else(|| crate::provenance::process_start_time(pid));
9446    #[cfg(windows)]
9447    let process_start_time = Some(windows_image.start_time());
9448    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9449    #[cfg(all(test, target_os = "macos"))]
9450    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9451    // Record the committed kernel image, not the configured pathname: PATH
9452    // lookup and shebang interpretation can select a different executable.
9453    // Linux's posix_spawn can return before that image is installed, so the
9454    // observer also waits for the kernel's first-exec flag to clear.
9455    #[cfg(not(windows))]
9456    let recorded_image = observe_spawned_image(pid);
9457    // spawn() confirms only the first exec, into the trampoline. Never persist
9458    // the trampoline image; the asynchronous acknowledgement publishes the
9459    // module image once the trampoline has replaced itself with the module.
9460    #[cfg(target_os = "macos")]
9461    let recorded_image = if privacy_exec.is_some() {
9462        None
9463    } else {
9464        recorded_image
9465    };
9466    #[cfg(target_os = "linux")]
9467    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9468    #[cfg(not(target_os = "linux"))]
9469    let recorded_cgroup_name = None;
9470    #[cfg(unix)]
9471    let preadmitted = admitted
9472        .as_mut()
9473        .and_then(|admitted| admitted.leader.take());
9474    #[cfg(not(unix))]
9475    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9476    let roster_guard = if let Some(guard) = preadmitted {
9477        Ok(guard)
9478    } else {
9479        roster.admit(
9480            spec.module_id.clone(),
9481            pid,
9482            spec.protocol,
9483            process_start_time,
9484            crate::child_roster::RecordedIdentity {
9485                container: None,
9486                #[cfg(unix)]
9487                carried: None,
9488                #[cfg(not(windows))]
9489                start_time: recorded_image.map(|image| image.start_time),
9490                #[cfg(windows)]
9491                start_time: process_start_time,
9492                #[cfg(not(windows))]
9493                executable: recorded_image
9494                    .and_then(|image| image.executable)
9495                    .map(crate::live_children::ExecutableIdentity::from),
9496                #[cfg(windows)]
9497                executable: Some(crate::live_children::ExecutableIdentity::from(
9498                    windows_image.identity(),
9499                )),
9500                cgroup_name: recorded_cgroup_name,
9501                #[cfg(target_os = "linux")]
9502                cgroup_placement: cgroup_placement.cloned(),
9503            },
9504        )
9505    };
9506    let roster_guard = match roster_guard {
9507        Ok(guard) => guard,
9508        Err(failure) => {
9509            return failure.with_cleanup(|source| {
9510                #[cfg(windows)]
9511                if let Some(job) = &job {
9512                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9513                }
9514                #[cfg(target_os = "linux")]
9515                kill_module_cgroup(cgroup_placement, &cgroup_name);
9516                #[cfg(unix)]
9517                let rejected_group = group_target.clone();
9518                kill_and_reap_rejected_child(
9519                    &mut child,
9520                    #[cfg(unix)]
9521                    rejected_group,
9522                );
9523                #[cfg(target_os = "linux")]
9524                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9525                    remove_module_cgroup(placement, &cgroup_name);
9526                }
9527                Err(SuperviseError::Spawn {
9528                    program: spec.program.clone(),
9529                    source: io::Error::other(format!(
9530                        "could not durably record module child: {source}"
9531                    )),
9532                    cgroup_path,
9533                })
9534            });
9535        }
9536    };
9537    #[cfg(unix)]
9538    if admitted.is_some() {
9539        if let Some(image) = recorded_image {
9540            if let Err(source) = roster_guard.confirm_executable(image) {
9541                kill_and_reap_rejected_child(&mut child, group_target.clone());
9542                return Err(SuperviseError::Spawn {
9543                    program: spec.program.clone(),
9544                    source,
9545                    cgroup_path,
9546                });
9547            }
9548        }
9549    }
9550    // The pre-spawn check can pass just before shutdown begins. Ownership is
9551    // published at the start of admission, even while a record write is queued.
9552    // If shutdown already saw an empty roster before that publication, this
9553    // check sees the roster closed and ends the child before starting a module.
9554    if roster.is_closed() {
9555        if carrying {
9556            #[cfg(unix)]
9557            let rejected_group = admitted
9558                .as_ref()
9559                .map(|admitted| admitted.rejection_target())
9560                .unwrap_or_else(|| group_target.clone());
9561            #[cfg(windows)]
9562            if let Some(job) = &job {
9563                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9564            }
9565            kill_and_reap_rejected_child(
9566                &mut child,
9567                #[cfg(unix)]
9568                rejected_group,
9569            );
9570            #[cfg(all(test, unix))]
9571            let _ =
9572                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9573            drop(roster_guard);
9574            return Err(SuperviseError::Spawn {
9575                program: spec.program.clone(),
9576                source: io::Error::other(
9577                    "the daemon began shutting down during carried admission; ended the container",
9578                ),
9579                cgroup_path,
9580            });
9581        }
9582        // This child was never admitted, so there is no module protocol shutdown to wait for.
9583        #[cfg(target_os = "linux")]
9584        kill_module_cgroup(cgroup_placement, &cgroup_name);
9585        if let Err(error) = child.start_kill() {
9586            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9587        }
9588        #[cfg(target_os = "linux")]
9589        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9590            // This spawn was never admitted, so shutdown has no roster entry
9591            // to await. Do not detach its cleanup: the runtime could exit
9592            // before that task reaps the rejected child and removes its group.
9593            while matches!(child.try_wait(), Ok(None)) {
9594                std::thread::yield_now();
9595            }
9596            if matches!(
9597                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9598                subc_cgroup::KillOutcome::Killed
9599            ) {
9600                if let Ok(path) = placement.module_path(&cgroup_name) {
9601                    while std::fs::read_to_string(path.join("cgroup.events"))
9602                        .ok()
9603                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9604                    {
9605                        std::thread::yield_now();
9606                    }
9607                }
9608            }
9609            remove_module_cgroup(placement, &cgroup_name);
9610        }
9611        drop(roster_guard);
9612        return Err(SuperviseError::Spawn {
9613            program: spec.program.clone(),
9614            source: io::Error::other(
9615                "the daemon began shutting down while this process was starting; ended it",
9616            ),
9617            cgroup_path,
9618        });
9619    }
9620
9621    let stdout_pump = match child.stdout.take() {
9622        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9623        None => {
9624            warn!(
9625                module_id = %spec.module_id,
9626                "spawned child exposed no stdout pipe; file capture will be incomplete"
9627            );
9628            None
9629        }
9630    };
9631    let stderr_pump = match child.stderr.take() {
9632        Some(stderr) => {
9633            let generation = ring
9634                .lock()
9635                .unwrap_or_else(|poisoned| poisoned.into_inner())
9636                .begin_process();
9637            Some(StderrPump {
9638                task: tokio::spawn(pump_stderr_to(
9639                    stderr,
9640                    Arc::clone(ring),
9641                    generation,
9642                    output_sink,
9643                )),
9644                generation,
9645            })
9646        }
9647        None => {
9648            // Spawning succeeded but the pipe did not materialise. Recording it as
9649            // uncaptured keeps the tail honest: the alternative is an empty tail
9650            // that reads as a module which printed nothing.
9651            ring.lock()
9652                .unwrap_or_else(|poisoned| poisoned.into_inner())
9653                .mark_not_captured("stderr pipe was not available on spawn");
9654            warn!(
9655                module_id = %spec.module_id,
9656                "spawned child exposed no stderr pipe; tail will be unavailable"
9657            );
9658            None
9659        }
9660    };
9661
9662    let exec = Arc::new(crate::activity::ExecOutcome::new());
9663    #[cfg(target_os = "macos")]
9664    let pending_exec = privacy_exec.is_some();
9665    #[cfg(not(target_os = "macos"))]
9666    let pending_exec = false;
9667    if !pending_exec {
9668        exec.finish(crate::activity::ExecState::CommittedExec);
9669    }
9670    #[cfg(windows)]
9671    let subtree_target = job
9672        .as_ref()
9673        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9674        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9675    #[cfg(target_os = "linux")]
9676    let subtree_target = cgroup_path
9677        .map(crate::subtree_resources::Target::Cgroup)
9678        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9679    #[cfg(not(any(target_os = "linux", windows)))]
9680    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9681    let mut supervised = SupervisedChild {
9682        subtree_target,
9683        child: containment::ProcessChild::plain(child, pid),
9684        exec: crate::activity::ExecOwner(exec),
9685        #[cfg(windows)]
9686        nonce_delivery,
9687        protocol: spec.protocol,
9688        #[cfg(target_os = "linux")]
9689        module_id: cgroup_name,
9690        #[cfg(target_os = "linux")]
9691        cgroup_placement: cgroup_placement.cloned(),
9692        #[cfg(windows)]
9693        job,
9694        stdout_pump,
9695        stderr_pump,
9696        stderr_ring: Arc::clone(ring),
9697        spawned_at,
9698        registration_deadline: None,
9699        registration_watch: None,
9700        spawned_at_ms,
9701        spawned_from,
9702        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9703        spawned_file_identity,
9704        process_start_time,
9705        process_identity,
9706        pid,
9707        roster_guard: Some(roster_guard),
9708        #[cfg(target_os = "macos")]
9709        privacy_exec,
9710        #[cfg(target_os = "macos")]
9711        report_ready: Arc::new(OnceLock::new()),
9712        spawn_failure: None,
9713    };
9714    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9715        supervised.child.reject(
9716            #[cfg(unix)]
9717            group_target,
9718        );
9719        #[cfg(all(test, unix))]
9720        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9721        return Err(error);
9722    }
9723    #[cfg(unix)]
9724    if let Some(admitted) = admitted.take() {
9725        admitted.disarm();
9726    }
9727    #[cfg(windows)]
9728    if let Err(error) = resume_suspended_child(pid, spec) {
9729        containment::terminate_blocking(&roster.containment);
9730        let _ = supervised.child.start_kill();
9731        return Err(error);
9732    }
9733    Ok(supervised)
9734}
9735
9736#[cfg(test)]
9737mod spawn_record_tests {
9738    use super::*;
9739
9740    thread_local! {
9741        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9742    }
9743
9744    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9745        LAST_PID.set(child.id());
9746        #[cfg(windows)]
9747        if let Some((_, address)) = _spec
9748            .env
9749            .iter()
9750            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9751        {
9752            use std::io::{Read, Write};
9753            let mut stream = std::net::TcpStream::connect(address).unwrap();
9754            stream
9755                .write_all(&child.id().unwrap().to_ne_bytes())
9756                .unwrap();
9757            let mut release = [0];
9758            stream.read_exact(&mut release).unwrap();
9759        }
9760    }
9761
9762    #[cfg(windows)]
9763    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9764        if spec
9765            .env
9766            .iter()
9767            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9768        {
9769            Err(io::Error::other("injected module job assignment failure"))
9770        } else {
9771            Ok(())
9772        }
9773    }
9774
9775    fn spec() -> ModuleSpec {
9776        ModuleSpec {
9777            module_id: "spawn-record-worker".into(),
9778            #[cfg(unix)]
9779            program: "/bin/sleep".into(),
9780            #[cfg(windows)]
9781            program: std::env::current_exe().unwrap(),
9782            #[cfg(unix)]
9783            args: vec!["60".into()],
9784            #[cfg(windows)]
9785            args: vec![
9786                "--exact".into(),
9787                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9788                "--nocapture".into(),
9789            ],
9790            env: vec![],
9791            reserved: false,
9792            reserved_prefixes: vec![],
9793            protocol: ModuleProtocol::None,
9794            overlap: Default::default(),
9795            carry_containment: false,
9796        }
9797    }
9798
9799    #[tokio::test]
9800    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9801        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9802        let supervisor =
9803            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9804                .with_live_children_record(home.join("live-children.json"));
9805        crate::live_children::fail_next_file_sync();
9806        let result = supervisor.spawn(spec());
9807        let failed = result.is_err();
9808        if let Ok(module) = result {
9809            module.stop().await.unwrap();
9810        }
9811        assert!(failed, "record failure must fail spawn");
9812        let pid = LAST_PID.get().expect("the real OS child was created");
9813        assert!(
9814            subc_os::Process::open(pid).unwrap().is_none(),
9815            "child {pid} survived rejected admission or was not reaped"
9816        );
9817    }
9818
9819    #[cfg(target_os = "macos")]
9820    #[test]
9821    #[ignore = "manual on-device latency measurement"]
9822    fn durable_record_write_latency_50() {
9823        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9824        let path = home.join("live-children.json");
9825        let children = vec![crate::live_children::LiveChild {
9826            module_id: "latency-worker".into(),
9827            pid: std::process::id(),
9828            protocol: ModuleProtocol::None,
9829            start_time: Some(1),
9830            executable: None,
9831            cgroup_name: None,
9832            container: None,
9833        }];
9834        let mut samples = Vec::new();
9835        for _ in 0..50 {
9836            let start = std::time::Instant::now();
9837            crate::live_children::write_record(&path, &children).unwrap();
9838            samples.push(start.elapsed());
9839        }
9840        samples.sort();
9841        eprintln!(
9842            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9843            (samples[24] + samples[25]) / 2,
9844            samples[47]
9845        );
9846    }
9847
9848    #[cfg(windows)]
9849    #[test]
9850    fn windows_store_touch_fixture() {
9851        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9852            std::fs::write(path, b"module ran").unwrap();
9853        }
9854    }
9855
9856    #[cfg(windows)]
9857    #[tokio::test]
9858    async fn windows_daemon_fixture() {
9859        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9860            return;
9861        };
9862        let mut spec = spec();
9863        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9864        let supervisor =
9865            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9866        let _module = supervisor.spawn(spec).unwrap();
9867    }
9868
9869    #[cfg(windows)]
9870    #[test]
9871    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9872        use std::io::Read;
9873        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9874        // An outer job models a scheduler/service host. The daemon's root and
9875        // root job must nest without requiring breakaway from this job.
9876        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9877        command
9878            .args([
9879                "--exact",
9880                "supervise::spawn_record_tests::windows_daemon_fixture",
9881                "--nocapture",
9882            ])
9883            .env(
9884                "SUBC_TEST_DAEMON_BARRIER",
9885                listener.local_addr().unwrap().to_string(),
9886            );
9887        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9888        listener.set_nonblocking(true).unwrap();
9889        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9890        let mut stream = loop {
9891            if let Ok((stream, _)) = listener.accept() {
9892                break stream;
9893            }
9894            assert!(
9895                daemon.child.try_wait().unwrap().is_none(),
9896                "daemon failed before rendezvous"
9897            );
9898            assert!(
9899                std::time::Instant::now() < deadline,
9900                "daemon did not reach CreateProcess failpoint"
9901            );
9902            std::thread::yield_now();
9903        };
9904        stream
9905            .set_read_timeout(Some(Duration::from_secs(30)))
9906            .unwrap();
9907        let mut pid = [0; 4];
9908        stream.read_exact(&mut pid).unwrap();
9909        let pid = u32::from_ne_bytes(pid);
9910        assert!(subc_jobobject::process_exists(pid));
9911        // Terminate only the daemon, not its outer job: otherwise the host job
9912        // would also kill a child whose daemon never joined its root job.
9913        daemon.child.kill().unwrap();
9914        daemon.child.wait().unwrap();
9915        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9916        if !exited {
9917            daemon.job.terminate().unwrap();
9918        }
9919        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9920    }
9921
9922    #[cfg(windows)]
9923    #[tokio::test]
9924    async fn windows_containment_failure_never_resumes() {
9925        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9926        let sentinel = home.join("store-touched");
9927        let mut spec = spec();
9928        spec.env
9929            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9930        spec.env.push((
9931            "SUBC_TEST_STORE_TOUCH".into(),
9932            sentinel.display().to_string(),
9933        ));
9934        let supervisor =
9935            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9936        let result = supervisor.spawn(spec);
9937        // A version that resumes despite assignment failure must have time to
9938        // execute its sentinel before dropping the returned module kills it.
9939        if let Ok(module) = result {
9940            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9941                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9942                    tokio::task::yield_now().await;
9943                }
9944            })
9945            .await;
9946            let _ = module.stop().await;
9947            panic!(
9948                "assignment failure resumed a module; store touched={}",
9949                sentinel.exists()
9950            );
9951        }
9952        assert!(
9953            !sentinel.exists(),
9954            "refused module executed store-touch sentinel"
9955        );
9956        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9957    }
9958
9959    #[cfg(windows)]
9960    #[tokio::test]
9961    async fn windows_nested_daemon_fixture() {
9962        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9963            return;
9964        };
9965        let mut spec = spec();
9966        spec.env.push((
9967            "SUBC_TEST_STORE_TOUCH".into(),
9968            sentinel.to_str().unwrap().into(),
9969        ));
9970        let supervisor =
9971            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9972        let module = supervisor
9973            .spawn(spec)
9974            .expect("per-module job must nest below daemon and host jobs");
9975        tokio::time::timeout(Duration::from_secs(10), async {
9976            while !PathBuf::from(&sentinel).exists() {
9977                tokio::task::yield_now().await;
9978            }
9979        })
9980        .await
9981        .unwrap();
9982        module.stop().await.unwrap();
9983    }
9984
9985    #[cfg(windows)]
9986    #[test]
9987    fn windows_module_job_nests_under_host_job() {
9988        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9989        let sentinel = home.join("nested-store-touched");
9990        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9991        command
9992            .args([
9993                "--exact",
9994                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9995                "--nocapture",
9996            ])
9997            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9998        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9999        assert!(daemon.child.wait().unwrap().success());
10000        assert!(sentinel.exists(), "nested protocol:none module never ran");
10001    }
10002}
10003
10004#[cfg(target_os = "linux")]
10005pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10006    use subc_cgroup::KillOutcome;
10007    match subc_cgroup::kill_module(placement, module_id) {
10008        KillOutcome::Killed => {}
10009        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10010            debug!(
10011                module_id,
10012                "cgroup tree kill unavailable; using direct-child kill"
10013            );
10014        }
10015        KillOutcome::IoError { path, error } => {
10016            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10017        }
10018    }
10019}
10020
10021/// Assign a freshly spawned Windows child without ever resuming on failure.
10022///
10023/// Steps 2 and 3 of the suspended-create contract: the job is created and the
10024/// child assigned **while it is still suspended** (step 1 is
10025/// `suspend_on_create_async` at the spawn site). The caller resumes only after
10026/// the child's roster admission is durable. Failed containment kills and reaps
10027/// the suspended child instead of running an unprotected module.
10028#[cfg(windows)]
10029fn contain_spawned_child(
10030    child: &mut Child,
10031    spec: &ModuleSpec,
10032    carried: Option<&Arc<containment::CarriedContainment>>,
10033) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10034    let assignment = || {
10035        let job = match carried {
10036            Some(container) => container.job(),
10037            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10038        };
10039        #[cfg(test)]
10040        spawn_record_tests::assignment_failpoint(spec)?;
10041        job.lock()
10042            .unwrap_or_else(|p| p.into_inner())
10043            .assign(child)?;
10044        Ok::<_, io::Error>(job)
10045    };
10046    match assignment() {
10047        Ok(job) => Ok(Some(job)),
10048        Err(source) => {
10049            kill_and_reap_rejected_child(child);
10050            Err(SuperviseError::Spawn {
10051                program: spec.program.clone(),
10052                source: io::Error::other(format!(
10053                    "Windows module job containment failed: {source}"
10054                )),
10055                cgroup_path: None,
10056            })
10057        }
10058    }
10059}
10060
10061/// Rejection is synchronous: do not return a spawn error while the child can
10062/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
10063fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10064    #[cfg(unix)]
10065    group.signal_rejected(child);
10066    let _ = child.start_kill();
10067    while matches!(child.try_wait(), Ok(None)) {
10068        std::thread::yield_now();
10069    }
10070}
10071
10072/// Resume a suspended child, killing it if it cannot be started.
10073///
10074/// A suspended process holds a pid and does nothing, so there is no useful
10075/// state to return: the caller gets an error and the spawn fails.
10076#[cfg(windows)]
10077fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10078    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10079        // Kill it here rather than leaving a suspended process for the caller
10080        // to notice; `kill_on_drop` would eventually do this, but the module
10081        // would have been reported as running in between.
10082        let _ = std::process::Command::new("taskkill.exe")
10083            .args(["/PID", &pid.to_string(), "/T", "/F"])
10084            .stdin(Stdio::null())
10085            .stdout(Stdio::null())
10086            .stderr(Stdio::null())
10087            .status();
10088        return Err(SuperviseError::Spawn {
10089            program: spec.program.clone(),
10090            source,
10091            cgroup_path: None,
10092        });
10093    }
10094    Ok(())
10095}
10096
10097#[cfg(target_os = "linux")]
10098fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10099    match placement.remove_module(module_id) {
10100        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10101        Err(error) => warn!(
10102            module_id,
10103            error = %error,
10104            "could not remove module cgroup after process exit; continuing teardown"
10105        ),
10106    }
10107}
10108
10109#[cfg(target_os = "linux")]
10110const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10111#[cfg(target_os = "linux")]
10112const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10113
10114#[cfg(target_os = "linux")]
10115async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10116    let path = placement.module_path(module_id).ok();
10117    if path.as_ref().is_some_and(|path| {
10118        std::fs::read_to_string(path.join("cgroup.events"))
10119            .ok()
10120            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10121    }) {
10122        remove_module_cgroup(placement, module_id);
10123        return;
10124    }
10125
10126    // Freeze before enumerating: a frozen task cannot exit and release its PID
10127    // for reuse by a process outside this module while SIGTERM is being sent.
10128    if terminate_reaped_cgroup(placement, module_id).await {
10129        if let Some(path) = &path {
10130            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10131            while cgroup_is_populated(path) && Instant::now() < deadline {
10132                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10133            }
10134        }
10135    }
10136
10137    // Reaping the direct child is not proof its descendants exited. Kill any
10138    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
10139    // cgroup empty before rmdir.
10140    if matches!(
10141        subc_cgroup::kill_module(Some(placement), module_id),
10142        subc_cgroup::KillOutcome::Killed
10143    ) {
10144        if let Some(path) = &path {
10145            while cgroup_is_populated(path) {
10146                sleep(Duration::from_millis(1)).await;
10147            }
10148        }
10149    }
10150    remove_module_cgroup(placement, module_id);
10151}
10152
10153#[cfg(target_os = "linux")]
10154fn cgroup_is_populated(path: &std::path::Path) -> bool {
10155    std::fs::read_to_string(path.join("cgroup.events"))
10156        .ok()
10157        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10158}
10159
10160#[cfg(target_os = "linux")]
10161async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10162    let frozen =
10163        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10164            .await
10165        {
10166            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10167            outcome => {
10168                tracing::debug!(
10169                    module_id,
10170                    ?outcome,
10171                    "cgroup freeze failed; skipping SIGTERM grace"
10172                );
10173                return false;
10174            }
10175        };
10176    let listed = match frozen.processes() {
10177        subc_cgroup::ProcsOutcome::Listed(pids) => {
10178            for pid in pids.into_iter().filter(|pid| *pid > 1) {
10179                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10180                    if let Err(error) =
10181                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10182                    {
10183                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10184                    }
10185                }
10186            }
10187            true
10188        }
10189        outcome => {
10190            tracing::debug!(
10191                module_id,
10192                ?outcome,
10193                "cgroup process listing failed; skipping SIGTERM grace"
10194            );
10195            false
10196        }
10197    };
10198    match frozen.thaw() {
10199        subc_cgroup::ThawOutcome::Thawed => listed,
10200        outcome => {
10201            tracing::debug!(
10202                module_id,
10203                ?outcome,
10204                "cgroup thaw failed; proceeding to cgroup.kill"
10205            );
10206            false
10207        }
10208    }
10209}
10210
10211#[cfg(target_os = "linux")]
10212fn apply_cgroup_placement(
10213    command: &mut Command,
10214    spec: &ModuleSpec,
10215    path: &std::path::Path,
10216) -> Result<(), SuperviseError> {
10217    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10218        module_id: spec.module_id.clone(),
10219        source,
10220    })
10221}
10222
10223fn capture_retention(spec: &ModuleSpec) -> Retention {
10224    let defaults = Retention::default();
10225    let value = |name: &str| {
10226        spec.env
10227            .iter()
10228            .rev()
10229            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10230    };
10231    Retention {
10232        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10233            .and_then(|value| value.parse().ok())
10234            .unwrap_or(defaults.max_file_mb),
10235        keep: value(CAPTURE_KEEP_ENV)
10236            .and_then(|value| value.parse().ok())
10237            .unwrap_or(defaults.keep),
10238        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10239            .and_then(|value| value.parse().ok())
10240            .unwrap_or(defaults.max_age_days),
10241    }
10242}
10243
10244/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
10245/// module's registration to the exact process the supervisor spawned.
10246fn generate_launch_nonce() -> Result<String, SuperviseError> {
10247    let mut bytes = [0u8; 32];
10248    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10249        reason: source.to_string(),
10250    })?;
10251    let mut hex = String::with_capacity(64);
10252    for b in bytes {
10253        use std::fmt::Write;
10254        let _ = write!(hex, "{b:02x}");
10255    }
10256    Ok(hex)
10257}
10258
10259/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
10260/// signal about how many leading bytes matched.
10261fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10262    if a.len() != b.len() {
10263        return false;
10264    }
10265    let mut diff = 0u8;
10266    for (x, y) in a.iter().zip(b.iter()) {
10267        diff |= x ^ y;
10268    }
10269    diff == 0
10270}
10271
10272/// The kernel's image after an acknowledged exec, shared by ordinary launches
10273/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
10274/// not identities inferred from a configured pathname.
10275#[cfg(not(windows))]
10276fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10277    let process = subc_os::Process::open(pid).ok().flatten()?;
10278    #[cfg(target_os = "linux")]
10279    return sample_after_linux_exec(
10280        || {
10281            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10282            let committed = linux_exec_committed(&stat);
10283            #[cfg(test)]
10284            spawn_image_tests::report_exec_state(committed);
10285            committed
10286        },
10287        || process.observe(),
10288    );
10289    #[cfg(not(target_os = "linux"))]
10290    process.observe()
10291}
10292
10293/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
10294/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
10295/// installation. Until it clears, /proc/pid/exe can still name the daemon.
10296#[cfg(target_os = "linux")]
10297fn linux_exec_committed(stat: &str) -> Option<bool> {
10298    const PF_FORKNOEXEC: u64 = 0x40;
10299    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10300    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10301}
10302
10303#[cfg(target_os = "linux")]
10304fn sample_after_linux_exec(
10305    mut exec_committed: impl FnMut() -> Option<bool>,
10306    observe: impl FnOnce() -> Option<subc_os::Observation>,
10307) -> Option<subc_os::Observation> {
10308    let deadline = std::time::Instant::now() + Duration::from_secs(1);
10309    while !exec_committed()? {
10310        // A failed or indefinitely stalled exec must not publish the inherited
10311        // image as authority to signal a future orphan. Missing identity keeps
10312        // the existing conservative orphan-sweep behavior.
10313        if std::time::Instant::now() >= deadline {
10314            return None;
10315        }
10316        std::thread::sleep(Duration::from_micros(50));
10317    }
10318    observe()
10319}
10320
10321#[cfg(all(test, target_os = "linux"))]
10322mod spawn_image_tests {
10323    use super::*;
10324    use std::cell::{Cell, RefCell};
10325
10326    enum ExecEvent {
10327        Uncommitted,
10328        Image(Option<subc_os::Observation>),
10329    }
10330
10331    thread_local! {
10332        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10333    }
10334
10335    pub(super) fn report_exec_state(committed: Option<bool>) {
10336        if committed == Some(false) {
10337            EXEC_EVENTS.with(|events| {
10338                if let Some(events) = events.borrow_mut().take() {
10339                    let _ = events.send(ExecEvent::Uncommitted);
10340                }
10341            });
10342        }
10343    }
10344
10345    #[test]
10346    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10347        use std::io::{Read, Write};
10348        use std::os::fd::AsRawFd;
10349
10350        let descriptor = std::fs::File::open("/dev/null").unwrap();
10351        let mut command = std::process::Command::new("/bin/sleep");
10352        command.arg("30");
10353        let (mut ready, mut release) =
10354            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10355                .unwrap();
10356        let spawn = std::thread::spawn(move || command.spawn());
10357        let mut message = [0u8; 8];
10358        ready.read_exact(&mut message).unwrap();
10359        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10360        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10361        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10362        let (events, received) = std::sync::mpsc::channel();
10363        let observer = std::thread::spawn(move || {
10364            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10365            let image = observe_spawned_image(pid);
10366            let _ = events.send(ExecEvent::Image(image));
10367        });
10368        // Release only once the production observer has seen the uncommitted
10369        // exec, or has incorrectly returned the inherited image. Either way,
10370        // release and reap before asserting so a regression cannot leak a child.
10371        let first = received.recv_timeout(Duration::from_secs(5));
10372        release.write_all(&[1]).unwrap();
10373        let mut child = spawn.join().unwrap().unwrap();
10374        let after = match first {
10375            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10376                Ok(ExecEvent::Image(image)) => image,
10377                _ => None,
10378            },
10379            Ok(ExecEvent::Image(image)) => image,
10380            Err(_) => None,
10381        };
10382        observer.join().unwrap();
10383        child.kill().unwrap();
10384        child.wait().unwrap();
10385
10386        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10387        assert_eq!(
10388            inherited.unwrap().executable,
10389            subc_os::file_identity(&std::env::current_exe().unwrap())
10390        );
10391        assert_eq!(
10392            after.unwrap().executable,
10393            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10394        );
10395    }
10396
10397    #[test]
10398    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10399        let committed = Cell::new(false);
10400        let inherited = subc_os::Observation {
10401            start_time: 23,
10402            executable: Some(subc_os::FileIdentity {
10403                device: 29,
10404                inode: 31,
10405            }),
10406        };
10407        let module = subc_os::Observation {
10408            start_time: 23,
10409            executable: Some(subc_os::FileIdentity {
10410                device: 29,
10411                inode: 37,
10412            }),
10413        };
10414        let polls = Cell::new(0);
10415        let image = sample_after_linux_exec(
10416            || {
10417                polls.set(polls.get() + 1);
10418                // Hold the inherited image through the first poll, as in the
10419                // posix_spawn parent-wake window, then commit the module image.
10420                committed.set(polls.get() == 2);
10421                Some(committed.get())
10422            },
10423            || Some(if committed.get() { module } else { inherited }),
10424        );
10425        assert_eq!(image, Some(module));
10426        assert_eq!(polls.get(), 2);
10427    }
10428
10429    #[test]
10430    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10431        assert_eq!(
10432            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10433            Some(false)
10434        );
10435        assert_eq!(
10436            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10437            Some(true)
10438        );
10439        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10440        assert_eq!(linux_exec_committed("17 (a) R"), None);
10441    }
10442
10443    #[test]
10444    fn unreadable_exec_state_never_samples_an_inherited_image() {
10445        assert_eq!(
10446            sample_after_linux_exec(
10447                || None,
10448                || panic!("must not sample before exec confirmation")
10449            ),
10450            None
10451        );
10452    }
10453
10454    #[test]
10455    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10456        assert_eq!(
10457            sample_after_linux_exec(
10458                || Some(false),
10459                || panic!("must not sample after an exec timeout")
10460            ),
10461            None
10462        );
10463    }
10464}
10465
10466/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
10467/// different executable after the no-delete capture was taken.
10468#[cfg(windows)]
10469fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10470    let mut program = spec.program.clone();
10471    if program.is_absolute() || program.components().count() > 1 {
10472        // An explicit path can name a PE image with no extension. Try that
10473        // exact file first, as Command does, before falling back to .exe.
10474        match subc_os::ExecutableCapture::open(&program) {
10475            Ok(capture) => return Ok(capture),
10476            Err(error)
10477                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10478            Err(error) => return Err(error),
10479        }
10480        program.set_extension("exe");
10481        return subc_os::ExecutableCapture::open(&program);
10482    }
10483    if program.extension().is_none() {
10484        program.set_extension("exe");
10485    }
10486    let custom_path = spec
10487        .env
10488        .iter()
10489        .rev()
10490        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10491    let mut directories = Vec::new();
10492    // These are Windows' default executable search locations. A configured
10493    // PATH still overrides the inherited PATH, not the system directories.
10494    if let Some(parent) = std::env::current_exe()?.parent() {
10495        directories.push(parent.to_path_buf());
10496    }
10497    directories.push(std::env::current_dir()?);
10498    if let Some(windows) = std::env::var_os("SystemRoot") {
10499        let windows = PathBuf::from(windows);
10500        directories.push(windows.join("System32"));
10501        directories.push(windows.clone());
10502    }
10503    let path = custom_path
10504        .map(|(_, value)| std::ffi::OsString::from(value))
10505        .or_else(|| std::env::var_os("PATH"));
10506    if let Some(path) = path {
10507        directories.extend(std::env::split_paths(&path));
10508    }
10509    for directory in directories {
10510        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10511            Ok(capture) => return Ok(capture),
10512            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10513            Err(error) => return Err(error),
10514        }
10515    }
10516    Err(io::Error::new(
10517        io::ErrorKind::NotFound,
10518        "resolved executable not found",
10519    ))
10520}
10521
10522fn spawn_and_mark_running(
10523    spec: &ModuleSpec,
10524    runtime: &SupervisorRuntimeConfig,
10525    snapshot: &SharedSnapshot,
10526) -> Result<SupervisedChild, SuperviseError> {
10527    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10528    let registration_watch = watch_registration(&runtime.registry, spec);
10529    #[cfg(unix)]
10530    let joining_group = containment::current(&runtime.child_roster.containment)
10531        .is_some_and(|container| container.group().is_some());
10532    let spawn = || {
10533        spawn_child(
10534            spec,
10535            runtime.connection_file_path.as_deref(),
10536            runtime.supervisor_handle.as_ref(),
10537            &runtime.stderr_ring,
10538            runtime.capture_logs_dir.as_deref(),
10539            &runtime.child_roster,
10540            #[cfg(target_os = "linux")]
10541            runtime.cgroup_placement.as_ref(),
10542        )
10543    };
10544    let result = spawn();
10545    #[cfg(unix)]
10546    let result = if result.is_err()
10547        && joining_group
10548        && !runtime.child_roster.is_closed()
10549        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10550        && runtime
10551            .child_roster
10552            .previous_instance(&spec.module_id)
10553            .is_none()
10554    {
10555        // The failed join already destroyed and verified the old container.
10556        // Try one fresh group as part of this same restart, without spending
10557        // another crash-budget entry. Cgroups and Windows jobs do not retry.
10558        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10559        spawn()
10560    } else {
10561        result
10562    };
10563    let child = result?;
10564    let mut child = child;
10565    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10566    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10567        containment::terminate_blocking(&runtime.child_roster.containment);
10568        return Err(error);
10569    }
10570    if let Some(spawn) = activity_spawn {
10571        spawn.complete(snapshot);
10572    }
10573    Ok(child)
10574}
10575
10576struct RegistrationWatch {
10577    changes: watch::Receiver<u64>,
10578    baseline: u64,
10579}
10580
10581// Capture before spawning: a fast child's accepted HELLO can disappear from the
10582// active registry before the monitor first runs. A pre-existing registration
10583// must not satisfy a new process's deadline either.
10584fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10585    if spec.protocol != ModuleProtocol::Subc {
10586        return None;
10587    }
10588    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10589    let baseline = *changes.borrow();
10590    Some(RegistrationWatch { changes, baseline })
10591}
10592
10593fn arm_registration_deadline(
10594    child: &mut SupervisedChild,
10595    budget: Duration,
10596    registration_watch: Option<RegistrationWatch>,
10597) {
10598    child.registration_deadline =
10599        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10600    child.registration_watch = registration_watch;
10601}
10602
10603async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10604    if let Some(container) = child.child.container() {
10605        container.stop_members().await;
10606        child.kill_leader()?;
10607        child.wait().await?;
10608        return Ok(());
10609    }
10610    #[cfg(unix)]
10611    {
10612        // Children lead their own process groups. Signal the whole group so a
10613        // helper stuck before HELLO cannot survive its supervised parent.
10614        let group = unreaped_registration_group(child)?;
10615        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10616        // Process-group safety rule: only signal a group while its leader is
10617        // our unreaped child. Even a zombie leader reserves its pid/group id;
10618        // reaping it before KILL would let that id name an unrelated group.
10619        // Sleep without waiting so TERM-ignoring helpers get the full grace.
10620        sleep(crate::child_roster::TERM_TO_KILL).await;
10621        let group = unreaped_registration_group(child)?;
10622        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10623    }
10624    // Windows terminates the containing Job Object as well as the direct
10625    // child. Linux also kills the child's cgroup when placement is enabled.
10626    child.start_kill()?;
10627    child.wait().await?;
10628    Ok(())
10629}
10630
10631#[cfg(unix)]
10632fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10633    if let Some(container) = child.child.container() {
10634        return container
10635            .group()
10636            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10637            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10638    }
10639    // The group id comes only from the pid recorded at spawn, when the child
10640    // created its own group. Never admit init's group or a negative group id.
10641    let group = i32::try_from(child.pid)
10642        .ok()
10643        .filter(|pid| *pid > 1)
10644        .and_then(rustix::process::Pid::from_raw)
10645        .ok_or_else(|| {
10646            io::Error::new(
10647                io::ErrorKind::InvalidInput,
10648                "invalid supervised process group",
10649            )
10650        })?;
10651    if child.child.id() != Some(child.pid) {
10652        return Err(io::Error::new(
10653            io::ErrorKind::InvalidInput,
10654            "refusing process-group signal after supervised child was reaped",
10655        ));
10656    }
10657    Ok(group)
10658}
10659
10660enum RegistrationWaitOutcome {
10661    Registered,
10662    Exited(ExitReport),
10663    TimedOut,
10664}
10665
10666struct ReloadRegistrationFailure {
10667    exit_report: ExitReport,
10668    reason: String,
10669}
10670
10671#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10672enum BusyGaugeObservation {
10673    Quiescent,
10674    Busy,
10675    Omitted,
10676}
10677
10678/// What one health answer said about one declared busy gauge.
10679#[derive(Debug, Clone, PartialEq, Eq)]
10680enum GaugeVerdict {
10681    /// The gauge read 0: this gauge is not what holds the drain.
10682    Zero,
10683    /// The gauge read a positive count.
10684    NonZero(u64),
10685    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10686    /// a negative number). It counts as busy, because the daemon cannot tell
10687    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10688    NotACount(String),
10689    /// The health answer had no metrics object, or one without this gauge.
10690    Missing,
10691    /// The probe produced no health answer to read; carries the probe error's
10692    /// evidence label (`no-answer` for a probe that timed out).
10693    ProbeFailed(&'static str),
10694}
10695
10696impl GaugeVerdict {
10697    fn reads_quiet(&self) -> bool {
10698        matches!(self, Self::Zero)
10699    }
10700}
10701
10702impl fmt::Display for GaugeVerdict {
10703    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10704        match self {
10705            Self::Zero => f.write_str("zero"),
10706            Self::NonZero(value) => write!(f, "non_zero({value})"),
10707            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10708            Self::Missing => f.write_str("missing_from_health_body"),
10709            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10710        }
10711    }
10712}
10713
10714/// The longest JSON text kept for a gauge value that is not a count, so a
10715/// module reporting a large object as a gauge cannot flood the drain log.
10716const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10717
10718/// Read each declared gauge out of one health answer's metrics.
10719fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10720    let metrics = metrics.and_then(Value::as_object);
10721    gauges
10722        .iter()
10723        .map(
10724            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10725                None => GaugeVerdict::Missing,
10726                Some(value) => match value.as_u64() {
10727                    Some(0) => GaugeVerdict::Zero,
10728                    Some(count) => GaugeVerdict::NonZero(count),
10729                    None => {
10730                        let mut text = value.to_string();
10731                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10732                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10733                            while !text.is_char_boundary(end) {
10734                                end -= 1;
10735                            }
10736                            text.truncate(end);
10737                            text.push_str("...");
10738                        }
10739                        GaugeVerdict::NotACount(text)
10740                    }
10741                },
10742            },
10743        )
10744        .collect()
10745}
10746
10747/// The drain's overall reading of one health answer: omitted when any
10748/// declared gauge is missing, quiescent only when every gauge reads 0.
10749fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10750    if verdicts
10751        .iter()
10752        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10753    {
10754        BusyGaugeObservation::Omitted
10755    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10756        BusyGaugeObservation::Quiescent
10757    } else {
10758        BusyGaugeObservation::Busy
10759    }
10760}
10761
10762/// One declared gauge's history over a drain: the latest verdict on it, and
10763/// the latest value actually read for it (which a later failed probe or
10764/// missing gauge does not erase). Times are offsets from the start of the
10765/// drain's wait.
10766#[derive(Debug, Clone, Default, PartialEq, Eq)]
10767struct GaugeTrace {
10768    latest: Option<(GaugeVerdict, Duration)>,
10769    last_value: Option<(GaugeVerdict, Duration)>,
10770}
10771
10772/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10773/// its ceiling can say in its timeout line which declared gauge held it and
10774/// why. Without it, a gauge that read busy, a gauge the module never
10775/// reported, and a probe that never got an answer all looked the same: a
10776/// drain that ended at its ceiling with nothing in flight.
10777#[derive(Debug, Clone, Default, PartialEq, Eq)]
10778struct DrainGaugeTrace {
10779    /// Health probes the drain sent.
10780    probes: u32,
10781    /// Each declared gauge, in declaration order.
10782    gauges: Vec<(String, GaugeTrace)>,
10783}
10784
10785impl DrainGaugeTrace {
10786    fn new(gauges: &[String]) -> Self {
10787        Self {
10788            probes: 0,
10789            gauges: gauges
10790                .iter()
10791                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10792                .collect(),
10793        }
10794    }
10795
10796    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10797    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10798        self.probes = self.probes.saturating_add(1);
10799        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10800            trace.latest = Some((verdict.clone(), at));
10801            if matches!(
10802                verdict,
10803                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10804            ) {
10805                trace.last_value = Some((verdict.clone(), at));
10806            }
10807        }
10808    }
10809}
10810
10811/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10812/// declared gauge, separated by ` | `. A value is shown as its verdict
10813/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10814/// and whether that counted as busy.
10815impl fmt::Display for DrainGaugeTrace {
10816    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10817        if self.gauges.is_empty() {
10818            return f.write_str("none declared");
10819        }
10820        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10821            if index > 0 {
10822                f.write_str(" | ")?;
10823            }
10824            write!(f, "{name}: ")?;
10825            match &trace.latest {
10826                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10827                None => f.write_str("never probed")?,
10828            }
10829            match &trace.last_value {
10830                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10831                None => f.write_str("; no value ever read")?,
10832            }
10833        }
10834        Ok(())
10835    }
10836}
10837
10838pub(crate) fn declared_busy_gauges(
10839    registry: &Registry,
10840    module_id: &str,
10841) -> Result<Vec<String>, SuperviseError> {
10842    busy_gauges_of(
10843        registry
10844            .get_module(module_id)
10845            .map_err(SuperviseError::Registry)?,
10846    )
10847}
10848
10849/// [`declared_busy_gauges`] for the registration a connection holds, in any
10850/// slot: after cutover the incumbent is no longer the id's active
10851/// registration, and its own manifest is the one that names its gauges.
10852fn declared_busy_gauges_for_connection(
10853    registry: &Registry,
10854    connection_id: ConnectionId,
10855) -> Result<Vec<String>, SuperviseError> {
10856    busy_gauges_of(
10857        registry
10858            .get_module_by_connection(connection_id)
10859            .map_err(SuperviseError::Registry)?,
10860    )
10861}
10862
10863fn busy_gauges_of(
10864    registration: Option<crate::registry::ModuleRegistration>,
10865) -> Result<Vec<String>, SuperviseError> {
10866    let Some(registration) = registration else {
10867        return Ok(Vec::new());
10868    };
10869    let Some(self_signals) = registration.manifest.self_signals else {
10870        return Ok(Vec::new());
10871    };
10872
10873    let mut gauges = Vec::new();
10874    for declaration in self_signals {
10875        if declaration.kind != SelfSignalKind::Busy {
10876            continue;
10877        }
10878        match declaration.anchored_to {
10879            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10880                gauges.extend(declared)
10881            }
10882            _ => {
10883                // An invalid Busy anchor is fail-safe: the empty name cannot be
10884                // present in a conforming health report, so this drain stays busy.
10885                gauges.push(String::new());
10886            }
10887        }
10888    }
10889    Ok(gauges)
10890}
10891
10892/// Wait for `endpoint` to have nothing in flight and, when the module declares
10893/// busy gauges, for a health probe to report them quiet. The probe is addressed
10894/// by `scope`: a swap's superseded incumbent must be asked about its own
10895/// gauges, and by module id the probe would reach the promoted candidate.
10896/// While the gauges read busy, the probe repeats every
10897/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10898/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10899/// timeout line.
10900#[allow(clippy::too_many_arguments)]
10901async fn wait_for_forwarding_quiescence(
10902    forwarding: &ForwardingTable,
10903    module_id: &str,
10904    runtime: &SupervisorRuntimeConfig,
10905    endpoint: crate::ModuleEndpointId,
10906    deadline: Instant,
10907    busy_gauges: &[String],
10908    scope: DrainScope,
10909    trace: &mut DrainGaugeTrace,
10910) -> Result<bool, SuperviseError> {
10911    let wait_started = Instant::now();
10912    let mut gauges_quiescent = busy_gauges.is_empty();
10913    let mut next_probe_at = wait_started;
10914    let mut omission_reported = false;
10915
10916    loop {
10917        let now = Instant::now();
10918        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10919            let report = match scope {
10920                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10921                DrainScope::Endpoint(endpoint) => {
10922                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10923                }
10924            };
10925            let verdicts = match &report {
10926                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10927                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10928            };
10929            trace.record(&verdicts, wait_started.elapsed());
10930            gauges_quiescent = match report {
10931                Ok(_) => match busy_gauge_observation(&verdicts) {
10932                    BusyGaugeObservation::Quiescent => true,
10933                    BusyGaugeObservation::Busy => false,
10934                    BusyGaugeObservation::Omitted => {
10935                        // Once per drain: a module that never reports a gauge
10936                        // it declared would otherwise say so on every probe.
10937                        if !omission_reported {
10938                            forwarding
10939                                .counters()
10940                                .increment_drains_with_undeclared_gauge();
10941                            let missing = busy_gauges
10942                                .iter()
10943                                .zip(&verdicts)
10944                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10945                                .map(|(gauge, _)| gauge.as_str())
10946                                .collect::<Vec<_>>()
10947                                .join(",");
10948                            warn!(
10949                                module_id,
10950                                missing_gauges = %missing,
10951                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10952                            );
10953                            omission_reported = true;
10954                        }
10955                        false
10956                    }
10957                },
10958                Err(err) => {
10959                    warn!(
10960                        module_id,
10961                        error = %err,
10962                        "drain health.check did not produce declared busy gauges; treating module as busy"
10963                    );
10964                    false
10965                }
10966            };
10967            // Counted from the probe's completion: the probe above was awaited,
10968            // so no second probe can start while one is still outstanding.
10969            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10970        }
10971
10972        let in_flight = forwarding
10973            .endpoint_in_flight_count(endpoint)
10974            .map_err(SuperviseError::Forwarding)?;
10975        if in_flight == 0 && gauges_quiescent {
10976            return Ok(true);
10977        }
10978
10979        let now = Instant::now();
10980        if now >= deadline {
10981            return Ok(false);
10982        }
10983        let mut wait = deadline
10984            .saturating_duration_since(now)
10985            .min(REGISTRY_RELEASE_POLL);
10986        if !busy_gauges.is_empty() {
10987            wait = wait.min(next_probe_at.saturating_duration_since(now));
10988        }
10989        sleep(wait).await;
10990    }
10991}
10992
10993/// The wait between drain busy-gauge probes: the larger of
10994/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10995/// drain's own in-flight poll interval.
10996fn drain_busy_reprobe_interval() -> Duration {
10997    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10998}
10999
11000/// Logs why a drain gave up: the drain waited its full timeout without the
11001/// module going quiet, and teardown is being forced.
11002///
11003/// The line names two kinds of evidence, because a bare "timed out" hides
11004/// causes that need different fixes:
11005/// - `holdouts`: the requests and routes still open when the timeout hit,
11006///   and on which connections. One stuck call is a module bug; a held stream
11007///   on every session is a module that should end its streams when it
11008///   receives `module.draining`.
11009/// - `gauges`: for each busy gauge the module declares, its latest reading
11010///   and how many probes ran. This separates a module that kept reporting
11011///   itself busy, one whose health answer never contained the gauge, and one
11012///   that never answered the probes, which otherwise all look like a timeout
11013///   with nothing in flight.
11014fn warn_drain_timed_out(
11015    module_id: &str,
11016    drain_timeout: Duration,
11017    reason: RouteCloseReason,
11018    holdouts: &crate::forwarding::DrainHoldouts,
11019    gauges: &DrainGaugeTrace,
11020) {
11021    warn!(
11022        module_id = %module_id,
11023        waited = ?drain_timeout,
11024        ?reason,
11025        held_requests = holdouts.requests,
11026        held_routes = holdouts.routes,
11027        total_routes = holdouts.total_routes,
11028        top_connections = ?holdouts.top_connections,
11029        // `module_channel:corr`, so the module can find each held request
11030        // in its own log; capped, so `held_requests` is the full count.
11031        held = %holdouts
11032            .held
11033            .iter()
11034            .map(|(channel, corr)| format!("{channel}:{corr}"))
11035            .collect::<Vec<_>>()
11036            .join(","),
11037        gauge_probes = gauges.probes,
11038        busy_gauges = %gauges,
11039        "route drain timed out before request quiescence; forcing teardown"
11040    );
11041}
11042
11043/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
11044///
11045/// `Ok` is always honest and passed straight through -- the wait actually measured
11046/// in-flight state. `Err` means the wait produced no measurement at all (the
11047/// forwarding table's lock was poisoned), so `false` is reported as the one honest
11048/// constant: the drain did not complete. Never recomputed from route state, never a
11049/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
11050fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11051    match wait_result {
11052        Ok(drained) => *drained,
11053        Err(_) => false,
11054    }
11055}
11056
11057fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11058    for released in released_routes {
11059        let frame = match Frame::build_with_version(
11060            released.negotiated_ver,
11061            FrameType::Goodbye,
11062            control_flags(),
11063            released.channel,
11064            released.epoch,
11065            0,
11066            Vec::new(),
11067        ) {
11068            Ok(frame) => frame,
11069            Err(err) => {
11070                warn!(
11071                    route_channel = released.channel,
11072                    error = %err,
11073                    "failed to build supervisor drain route GOODBYE frame"
11074                );
11075                continue;
11076            }
11077        };
11078        if !released.close_on_delivery_failure() {
11079            crate::forwarding::send_module_route_goodbye(
11080                &forwarding.counters(),
11081                &released.sink,
11082                frame,
11083                released.module_id.as_deref(),
11084                "supervisor drain",
11085            );
11086            continue;
11087        }
11088        if let Err(err) = released.sink.try_send(frame) {
11089            warn!(
11090                target_connection_id = released.connection_id.get(),
11091                route_channel = released.channel,
11092                error = %err,
11093                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11094            );
11095            let _ = forwarding.escalate_client_delivery_failure(
11096                released.connection_id,
11097                released.channel,
11098                released.epoch,
11099                CloseReason::new(
11100                    "route_goodbye_delivery_failed",
11101                    format!(
11102                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11103                        released.channel
11104                    ),
11105                ),
11106                crate::forwarding::UndeliveredFrame {
11107                    module_id: released.module_id.as_deref(),
11108                    sink: &released.sink,
11109                },
11110            );
11111        }
11112    }
11113}
11114
11115async fn send_module_draining(
11116    module_id: &str,
11117    reason: RouteCloseReason,
11118    deadline_ms: u64,
11119    target: &ModuleDrainTarget,
11120    _wait_for_flush: bool,
11121) -> bool {
11122    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11123        reason,
11124        deadline_ms,
11125    }) {
11126        Ok(body) => body,
11127        Err(err) => {
11128            warn!(
11129                module_id,
11130                error = %err,
11131                "failed to encode module draining command"
11132            );
11133            return false;
11134        }
11135    };
11136    let frame = match Frame::build_with_version(
11137        target.negotiated_ver,
11138        FrameType::Push,
11139        control_flags(),
11140        0,
11141        0,
11142        0,
11143        body,
11144    ) {
11145        Ok(frame) => frame,
11146        Err(err) => {
11147            warn!(
11148                module_id,
11149                error = %err,
11150                "failed to build module draining command frame"
11151            );
11152            return false;
11153        }
11154    };
11155    #[cfg(unix)]
11156    if _wait_for_flush {
11157        return target.sink.send_flushed(frame).await.is_ok();
11158    }
11159    if let Err(err) = target.sink.try_send(frame) {
11160        warn!(
11161            module_id,
11162            target_connection_id = target.endpoint.connection_id.get(),
11163            error = %err,
11164            "module draining command was not delivered to peer"
11165        );
11166        return false;
11167    }
11168    true
11169}
11170
11171/// The channel-0 GOODBYE that tells a module its stop is planned.
11172fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11173    match Frame::build_with_version(
11174        negotiated_ver,
11175        FrameType::Goodbye,
11176        control_flags(),
11177        0,
11178        0,
11179        0,
11180        Vec::new(),
11181    ) {
11182        Ok(frame) => Some(frame),
11183        Err(err) => {
11184            warn!(
11185                module_id,
11186                error = %err,
11187                "failed to build module GOODBYE frame"
11188            );
11189            None
11190        }
11191    }
11192}
11193
11194/// Send every registered module connection its module GOODBYE at daemon
11195/// shutdown, then request that connection's close.
11196///
11197/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
11198/// before EOF, so the GOODBYE must reach the socket before the close. A close
11199/// request does not wait for the connection's queued frames: its writer gets a
11200/// bounded grace after the close, is aborted if it overruns it, and the daemon
11201/// process may exit before that grace ends. So with `wait_for_flush`, each
11202/// connection is closed only after its writer has acknowledged writing the
11203/// GOODBYE, or once a short shared budget runs out, so one module that is not
11204/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
11205/// are only queued, for a shutdown the operator has told to stop waiting.
11206/// A connection that is already gone is skipped.
11207#[cfg(unix)]
11208async fn send_module_goodbyes_for_daemon_shutdown(
11209    forwarding: &Arc<ForwardingTable>,
11210    reason: &CloseReason,
11211    wait_for_flush: bool,
11212) {
11213    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11214    let targets = match forwarding.module_connections() {
11215        Ok(targets) => targets,
11216        Err(err) => {
11217            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11218            return;
11219        }
11220    };
11221    let deadline = Instant::now() + GOODBYE_BUDGET;
11222    let mut sends = tokio::task::JoinSet::new();
11223    for target in targets {
11224        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11225            continue;
11226        };
11227        if !wait_for_flush {
11228            if let Err(err) = target.sink.try_send(frame) {
11229                debug!(
11230                    module_id = %target.module_id,
11231                    error = %err,
11232                    "shutdown module GOODBYE was not queued"
11233                );
11234            }
11235            continue;
11236        }
11237        let forwarding = Arc::clone(forwarding);
11238        let reason = reason.clone();
11239        sends.spawn(async move {
11240            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11241                Ok(Ok(())) => {}
11242                Ok(Err(err)) => debug!(
11243                    module_id = %target.module_id,
11244                    error = %err,
11245                    "module connection closed before its shutdown GOODBYE was written"
11246                ),
11247                Err(_) => warn!(
11248                    module_id = %target.module_id,
11249                    budget = ?GOODBYE_BUDGET,
11250                    "shutdown module GOODBYE was not written within its budget; closing anyway"
11251                ),
11252            }
11253            forwarding.request_connection_close(target.endpoint.connection_id, reason);
11254        });
11255    }
11256    // Every task ends by the shared deadline, so this wait is bounded too.
11257    while sends.join_next().await.is_some() {}
11258}
11259
11260fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11261    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11262        return;
11263    };
11264    if let Err(err) = target.sink.try_send(frame) {
11265        warn!(
11266            module_id,
11267            target_connection_id = target.endpoint.connection_id.get(),
11268            error = %err,
11269            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11270        );
11271        forwarding.request_connection_close(
11272            target.endpoint.connection_id,
11273            CloseReason::new(
11274                "module_goodbye_delivery_failed",
11275                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11276            ),
11277        );
11278    }
11279}
11280
11281#[derive(Clone, Copy)]
11282struct ForwardingDrainContext<'a> {
11283    spec: &'a ModuleSpec,
11284    runtime: &'a SupervisorRuntimeConfig,
11285    registry: &'a Registry,
11286    scope: DrainScope,
11287}
11288
11289/// Which process a forwarding drain addresses.
11290#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11291enum DrainScope {
11292    /// Whatever endpoint is active for the module id: every plain stop,
11293    /// restart and reload. Also moves the module's state to `Draining`.
11294    Active,
11295    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
11296    /// module id would resolve to the promoted candidate and leave neither
11297    /// process routable. The module's state is left alone, since the promoted
11298    /// candidate is what it describes and that process is running.
11299    Endpoint(crate::ModuleEndpointId),
11300}
11301
11302/// Whether a child being drained has already been asked to stop by the time
11303/// its drain wait starts.
11304///
11305/// The drain wait is the same budget whatever this says. What it decides is
11306/// whether the supervisor must ask by signal before that wait begins: a child
11307/// that nobody asked will sit out the whole budget and then be SIGKILLed,
11308/// healthy or not.
11309#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11310enum StopNotice {
11311    /// The module was sent `module.draining` and a module GOODBYE over its own
11312    /// registered connection, and stops itself.
11313    SentOverConnection,
11314    /// The forwarding drain found no registered connection for the module: a
11315    /// subc child spawned moments ago that has not sent HELLO yet, or a
11316    /// `protocol: "none"` child, which never registers.
11317    NoConnection,
11318    /// This path sends nothing over the module's connection: the supervisor has
11319    /// no forwarding table, or the caller stops the child without a forwarding
11320    /// drain.
11321    NotSent,
11322}
11323
11324async fn begin_forwarding_drain(
11325    spec: &ModuleSpec,
11326    runtime: &SupervisorRuntimeConfig,
11327    registry: &Registry,
11328    snapshot: &SharedSnapshot,
11329    enabled: Option<bool>,
11330    reason: RouteCloseReason,
11331) -> Result<StopNotice, SuperviseError> {
11332    let Some(forwarding) = runtime.forwarding.as_ref() else {
11333        return Err(SuperviseError::ReloadUnavailable {
11334            module_id: spec.module_id.clone(),
11335            reason: "supervisor was not configured with a forwarding table".to_string(),
11336        });
11337    };
11338
11339    begin_forwarding_drain_with(
11340        forwarding,
11341        ForwardingDrainContext {
11342            spec,
11343            runtime,
11344            registry,
11345            scope: DrainScope::Active,
11346        },
11347        snapshot,
11348        enabled,
11349        reason,
11350        runtime.drain_timeout,
11351    )
11352    .await
11353}
11354
11355async fn begin_forwarding_drain_if_configured(
11356    spec: &ModuleSpec,
11357    runtime: &SupervisorRuntimeConfig,
11358    registry: &Registry,
11359    snapshot: &SharedSnapshot,
11360    enabled: Option<bool>,
11361    reason: RouteCloseReason,
11362) -> Result<StopNotice, SuperviseError> {
11363    begin_forwarding_drain_with_timeout(
11364        spec,
11365        runtime,
11366        registry,
11367        snapshot,
11368        enabled,
11369        reason,
11370        runtime.drain_timeout,
11371    )
11372    .await
11373}
11374
11375/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
11376/// budget, for paths where the operator overrides the module's configured one
11377/// (`supervisor.restart{drain_timeout_ms}`).
11378async fn begin_forwarding_drain_with_timeout(
11379    spec: &ModuleSpec,
11380    runtime: &SupervisorRuntimeConfig,
11381    registry: &Registry,
11382    snapshot: &SharedSnapshot,
11383    enabled: Option<bool>,
11384    reason: RouteCloseReason,
11385    drain_timeout: Duration,
11386) -> Result<StopNotice, SuperviseError> {
11387    let Some(forwarding) = runtime.forwarding.as_ref() else {
11388        return Ok(StopNotice::NotSent);
11389    };
11390
11391    begin_forwarding_drain_with(
11392        forwarding,
11393        ForwardingDrainContext {
11394            spec,
11395            runtime,
11396            registry,
11397            scope: DrainScope::Active,
11398        },
11399        snapshot,
11400        enabled,
11401        reason,
11402        drain_timeout,
11403    )
11404    .await
11405}
11406
11407async fn begin_forwarding_drain_with(
11408    forwarding: &ForwardingTable,
11409    context: ForwardingDrainContext<'_>,
11410    snapshot: &SharedSnapshot,
11411    enabled: Option<bool>,
11412    reason: RouteCloseReason,
11413    drain_timeout: Duration,
11414) -> Result<StopNotice, SuperviseError> {
11415    let ForwardingDrainContext {
11416        spec,
11417        runtime,
11418        registry,
11419        scope,
11420    } = context;
11421    debug_assert_ne!(reason, RouteCloseReason::Crash);
11422    let terminal = matches!(reason, RouteCloseReason::Disable);
11423    let drain_started_at = Instant::now();
11424    let drain_deadline = drain_started_at + drain_timeout;
11425    let deadline_ms =
11426        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11427    let busy_gauges = match scope {
11428        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11429        DrainScope::Endpoint(endpoint) => {
11430            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11431        }
11432    };
11433
11434    // Admission gate first: route.open/commit and route REQUEST admission are closed
11435    // before the first quiescence check, so the outstanding count can only fall.
11436    let gate_started = Instant::now();
11437    let drain_target = match scope {
11438        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11439        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11440    }
11441    .map_err(SuperviseError::Forwarding)?;
11442    // The instant admission closed, and how long taking the forwarding write
11443    // lock to close it took. The timeout line reports only the quiescence
11444    // wait, so without this a drain that started late looked like one that
11445    // started on time.
11446    info!(
11447        module_id = %spec.module_id,
11448        ?reason,
11449        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11450        connected = drain_target.is_some(),
11451        "module drain began; route admission closed"
11452    );
11453    if scope == DrainScope::Active {
11454        update_snapshot(snapshot, Some(&spec.module_id), |state| {
11455            state.state = ModuleState::Draining;
11456            state.draining_to_replace =
11457                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11458            if let Some(enabled) = enabled {
11459                state.enabled = enabled;
11460            }
11461        })?;
11462    }
11463
11464    let Some(target) = drain_target.as_ref() else {
11465        // Nothing was sent: the module has no registered connection to carry
11466        // `module.draining` or a GOODBYE. The caller must not assume the child
11467        // was asked to stop.
11468        return Ok(StopNotice::NoConnection);
11469    };
11470    {
11471        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11472        #[cfg(feature = "test-support")]
11473        forwarding.wait_before_route_closing_for_test().await;
11474        let routes = forwarding
11475            .endpoint_routes(target.endpoint)
11476            .map_err(SuperviseError::Forwarding)?;
11477        let routes_notified = routes.len();
11478        crate::control::send_route_control_pushes(
11479            forwarding,
11480            routes.clone(),
11481            ClientControlPush::RouteClosing {
11482                module_id: spec.module_id.clone(),
11483                channels: Vec::new(),
11484                reason,
11485            },
11486        );
11487        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11488
11489        // `route.closing` was just sent above: from here on every return path,
11490        // including an early one, MUST send `route.closed` before propagating
11491        // anything else. A client holds `closing` as a promise that a verdict is
11492        // coming; leaving early without `closed` strands it waiting forever, since
11493        // `closing` carries no timeout of its own.
11494        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11495        let wait_result = wait_for_forwarding_quiescence(
11496            forwarding,
11497            &spec.module_id,
11498            runtime,
11499            target.endpoint,
11500            drain_deadline,
11501            &busy_gauges,
11502            scope,
11503            &mut gauge_trace,
11504        )
11505        .await;
11506        let drained = drained_after_quiescence_wait(&wait_result);
11507        if let Err(err) = &wait_result {
11508            error!(
11509                module_id = %spec.module_id,
11510                ?reason,
11511                error = %err,
11512                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11513            );
11514        } else if !drained {
11515            // Read before teardown releases the routes.
11516            let holdouts = forwarding
11517                .endpoint_drain_holdouts(target.endpoint)
11518                .unwrap_or_default();
11519            warn_drain_timed_out(
11520                &spec.module_id,
11521                drain_timeout,
11522                reason,
11523                &holdouts,
11524                &gauge_trace,
11525            );
11526        }
11527        crate::control::send_route_control_pushes(
11528            forwarding,
11529            routes,
11530            ClientControlPush::RouteClosed {
11531                module_id: spec.module_id.clone(),
11532                channels: Vec::new(),
11533                reason,
11534                drained,
11535                abandoned: target.abandoned_bindings.len() as u32,
11536                excluded_subscriptions: target.excluded_subscriptions,
11537                terminal: Some(terminal),
11538            },
11539        );
11540        wait_result?;
11541
11542        // `route.closed` has now been sent unconditionally above. From here the
11543        // remaining steps are cleanup (route + module GOODBYE) rather than a
11544        // promise the client is waiting on, but a lock-poisoned
11545        // `release_module_endpoint_routes` would otherwise skip the module
11546        // GOODBYE silently too -- send it before propagating the error.
11547        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11548            Ok(routes) => routes,
11549            Err(err) => {
11550                warn!(
11551                    module_id = %spec.module_id,
11552                    ?reason,
11553                    error = %err,
11554                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11555                );
11556                send_module_goodbye(&spec.module_id, forwarding, target);
11557                return Err(SuperviseError::Forwarding(err));
11558            }
11559        };
11560        let route_goodbye_count = released_routes.len();
11561        send_route_goodbyes(forwarding, released_routes);
11562        send_module_goodbye(&spec.module_id, forwarding, target);
11563
11564        // The drain's happy path was previously silent: every emission above is
11565        // best-effort with only its failure arm logged, so "were consumers told"
11566        // was unprovable from the daemon log (surfaced by a 30-minute consumer
11567        // hang where the open question was exactly whether teardown notice went
11568        // out). One summary line makes that class decidable in one grep.
11569        info!(
11570            module_id = %spec.module_id,
11571            ?reason,
11572            routes_notified,
11573            route_goodbyes = route_goodbye_count,
11574            abandoned_reservations = target.abandoned_bindings.len(),
11575            excluded_subscriptions = target.excluded_subscriptions,
11576            drained,
11577            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11578        );
11579    }
11580
11581    Ok(StopNotice::SentOverConnection)
11582}
11583
11584/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
11585/// the only slot a plain (non-swap) spawn can register into.
11586async fn wait_for_registration_after_reload(
11587    registry: &Registry,
11588    module_id: &str,
11589    snapshot: &SharedSnapshot,
11590    child: &mut SupervisedChild,
11591    wait: Duration,
11592) -> Result<RegistrationWaitOutcome, SuperviseError> {
11593    wait_for_slot_registration(
11594        registry,
11595        crate::registry::RegistrationSlot::Active(module_id),
11596        module_id,
11597        snapshot,
11598        child,
11599        wait,
11600    )
11601    .await
11602}
11603
11604/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
11605///
11606/// Keyed on the slot rather than the bare module id because during a swap the
11607/// id's active slot is already held by the incumbent: an id-keyed wait would
11608/// report the incumbent's registration as the candidate's and a candidate that
11609/// never registers would look registered. A swap candidate waits on
11610/// `crate::registry::RegistrationSlot::Candidate`.
11611async fn wait_for_slot_registration(
11612    registry: &Registry,
11613    slot: crate::registry::RegistrationSlot<'_>,
11614    module_id: &str,
11615    snapshot: &SharedSnapshot,
11616    child: &mut SupervisedChild,
11617    wait: Duration,
11618) -> Result<RegistrationWaitOutcome, SuperviseError> {
11619    let deadline = Instant::now() + wait;
11620    loop {
11621        if registry
11622            .registration(slot)
11623            .map_err(SuperviseError::Registry)?
11624            .is_some()
11625        {
11626            return Ok(RegistrationWaitOutcome::Registered);
11627        }
11628
11629        let now = Instant::now();
11630        if now >= deadline {
11631            return Ok(RegistrationWaitOutcome::TimedOut);
11632        }
11633        let remaining = deadline.saturating_duration_since(now);
11634        let poll = remaining.min(REGISTRY_RELEASE_POLL);
11635
11636        tokio::select! {
11637            wait_result = child.wait() => {
11638                let status = wait_result.map_err(|source| SuperviseError::Wait {
11639                    module_id: module_id.to_string(),
11640                    source,
11641                })?;
11642                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11643                    snapshot,
11644                    child,
11645                    &status,
11646                )));
11647            }
11648            _ = sleep(poll) => {}
11649        }
11650    }
11651}
11652
11653fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11654    // A process stopped for missing its first HELLO did not provide service,
11655    // even if its TERM handler exits zero. Reload also uses this classification
11656    // when a replacement exits before registering.
11657    if exit_report.kind != ExitKind::DeliberateSeverance {
11658        exit_report.kind = ExitKind::Crash;
11659    }
11660    exit_report
11661}
11662
11663async fn handle_reload_child_registration_failure(
11664    spec: &ModuleSpec,
11665    runtime: &SupervisorRuntimeConfig,
11666    registry: &Registry,
11667    process_liveness: &SupervisorProcessLiveness,
11668    snapshot: &SharedSnapshot,
11669    _child: &mut Option<SupervisedChild>,
11670    failure: ReloadRegistrationFailure,
11671) -> Result<(), SuperviseError> {
11672    let ReloadRegistrationFailure {
11673        exit_report,
11674        reason,
11675    } = failure;
11676    match on_child_exit(
11677        spec,
11678        runtime.restart_policy,
11679        registry,
11680        snapshot,
11681        &runtime.terminal_ring,
11682        &runtime.spawn_events,
11683        &runtime.child_roster,
11684        exit_report,
11685        runtime.recovery_is_daemon,
11686    )
11687    .await
11688    {
11689        NextAction::Stop {
11690            registration_released,
11691        } => {
11692            if registration_released {
11693                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11694            }
11695        }
11696        NextAction::Restart { schedule } => {
11697            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11698                schedule.delay
11699            });
11700            if let Some(schedule) = schedule {
11701                log_crash_respawn(&spec.module_id, schedule);
11702            }
11703            schedule_respawn(
11704                runtime,
11705                snapshot,
11706                &spec.module_id,
11707                delay,
11708                RespawnKind::Spawn,
11709                true,
11710            )?;
11711        }
11712    }
11713    Err(SuperviseError::ReloadFailed {
11714        module_id: spec.module_id.clone(),
11715        reason,
11716    })
11717}
11718
11719async fn handle_reload_spawn_failure(
11720    spec: &ModuleSpec,
11721    runtime: &SupervisorRuntimeConfig,
11722    process_liveness: &SupervisorProcessLiveness,
11723    snapshot: &SharedSnapshot,
11724    _child: &mut Option<SupervisedChild>,
11725    reason: String,
11726) -> Result<(), SuperviseError> {
11727    let now = Instant::now();
11728    let mut schedule = None;
11729    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11730        clear_current_process_facts(state);
11731        if state.enabled {
11732            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11733            state.state = if schedule.is_some() {
11734                ModuleState::Restarting
11735            } else {
11736                if runtime
11737                    .failure_context(runtime.recovery_is_daemon)
11738                    .may_retry()
11739                {
11740                    state.failed_retry.schedule(&spec.module_id);
11741                }
11742                ModuleState::Failed
11743            };
11744        } else {
11745            state.state = ModuleState::Disabled;
11746        }
11747    })?;
11748    if let Some(schedule) = schedule {
11749        schedule_respawn(
11750            runtime,
11751            snapshot,
11752            &spec.module_id,
11753            schedule.delay,
11754            RespawnKind::Spawn,
11755            true,
11756        )?;
11757    } else {
11758        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11759    }
11760    Err(SuperviseError::ReloadFailed {
11761        module_id: spec.module_id.clone(),
11762        reason,
11763    })
11764}
11765
11766fn control_flags() -> Flags {
11767    Flags::new(false, Priority::Passive, false)
11768}
11769
11770#[allow(clippy::too_many_arguments)]
11771async fn drain_optional_child(
11772    module_id: &str,
11773    protocol: ModuleProtocol,
11774    stop_notice: StopNotice,
11775    registry: &Registry,
11776    forwarding: Option<&ForwardingTable>,
11777    snapshot: &SharedSnapshot,
11778    terminal_ring: &Arc<Mutex<TerminalRing>>,
11779    spawn_events: &SpawnEventFeed,
11780    child: &mut Option<SupervisedChild>,
11781    drain_timeout: Duration,
11782    final_state: ModuleState,
11783    enabled: Option<bool>,
11784    failure: &FailureContext,
11785) -> Result<(), SuperviseError> {
11786    if let Some(child) = child.take() {
11787        drain_child_to_state(
11788            module_id,
11789            protocol,
11790            stop_notice,
11791            registry,
11792            forwarding,
11793            snapshot,
11794            terminal_ring,
11795            spawn_events,
11796            child,
11797            drain_timeout,
11798            final_state,
11799            enabled,
11800            failure,
11801        )
11802        .await
11803    } else {
11804        update_snapshot(snapshot, Some(module_id), |state| {
11805            state.state = final_state;
11806            if final_state == ModuleState::Failed {
11807                state.failed_retry.failed_at = Some(Instant::now());
11808            }
11809            if let Some(enabled) = enabled {
11810                state.enabled = enabled;
11811            }
11812            clear_current_process_facts(state);
11813        })?;
11814        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11815    }
11816}
11817
11818#[allow(clippy::too_many_arguments)]
11819async fn drain_child_to_state(
11820    module_id: &str,
11821    _protocol: ModuleProtocol,
11822    stop_notice: StopNotice,
11823    registry: &Registry,
11824    forwarding: Option<&ForwardingTable>,
11825    snapshot: &SharedSnapshot,
11826    terminal_ring: &Arc<Mutex<TerminalRing>>,
11827    spawn_events: &SpawnEventFeed,
11828    mut child: SupervisedChild,
11829    drain_timeout: Duration,
11830    final_state: ModuleState,
11831    enabled: Option<bool>,
11832    failure: &FailureContext,
11833) -> Result<(), SuperviseError> {
11834    let protocol = child.protocol;
11835    update_snapshot(snapshot, Some(module_id), |state| {
11836        state.state = ModuleState::Draining;
11837        state.draining_to_replace = final_state == ModuleState::Restarting;
11838        if let Some(enabled) = enabled {
11839            state.enabled = enabled;
11840        }
11841    })?;
11842
11843    // The wait below is the same budget in every case; what differs is
11844    // whether anything has ASKED the child to stop before it starts. Only a
11845    // forwarding drain that reached the module's registered connection has
11846    // (`module.draining`, then a module GOODBYE). Every other child was told
11847    // nothing: a `protocol: "none"` module, which never registers; a subc
11848    // module spawned moments ago that has not sent HELLO yet; or a stop that
11849    // runs no forwarding drain. Without a signal the budget is only a delay
11850    // in front of SIGKILL -- and the not-yet-registered child is the worst
11851    // case, because it registers into a module that is already draining,
11852    // is never told, and is killed while healthy.
11853    if stop_notice != StopNotice::SentOverConnection {
11854        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11855            info!(
11856                module_id,
11857                pid = child.pid,
11858                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11859                "module has no connection yet; requesting stop by signal"
11860            );
11861        }
11862        request_graceful_stop(module_id, &child);
11863    }
11864
11865    let exit_report = match timeout(drain_timeout, child.wait()).await {
11866        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11867        Ok(Err(source)) => {
11868            fail_after_wait_error(
11869                module_id,
11870                snapshot,
11871                terminal_ring,
11872                spawn_events,
11873                format!("failed to wait during drain: {source}"),
11874                failure,
11875            );
11876            return Err(SuperviseError::Wait {
11877                module_id: module_id.to_string(),
11878                source,
11879            });
11880        }
11881        Err(_) => {
11882            // Mirror the sibling arm above: state is already `Draining`, and an
11883            // error propagated from here would strand it there -- a state
11884            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11885            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11886            // `Failed` before `?` keeps the module operator-visible and
11887            // revivable. Trigger is an ESRCH race (process exits between the
11888            // drain timeout firing and the kill) or a post-kill wait failure
11889            // (issue #34).
11890            //
11891            // Logged because the kill is otherwise visible only as signal 9 in
11892            // the terminal ring, and the budget it follows can be long enough
11893            // that consumers see a stretch of refusals with no stated cause.
11894            warn!(
11895                module_id,
11896                pid = child.pid,
11897                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11898                reason = ?final_state,
11899                ?stop_notice,
11900                "drain budget expired before the module exited; killing it"
11901            );
11902            let kill =
11903                if final_state == ModuleState::Restarting && child.child.container().is_some() {
11904                    child.kill_leader()
11905                } else {
11906                    child.start_kill()
11907                };
11908            kill.map_err(|source| {
11909                fail_after_wait_error(
11910                    module_id,
11911                    snapshot,
11912                    terminal_ring,
11913                    spawn_events,
11914                    format!("failed to kill during drain: {source}"),
11915                    failure,
11916                );
11917                SuperviseError::Kill {
11918                    module_id: module_id.to_string(),
11919                    source,
11920                }
11921            })?;
11922            let status = child.wait().await.map_err(|source| {
11923                fail_after_wait_error(
11924                    module_id,
11925                    snapshot,
11926                    terminal_ring,
11927                    spawn_events,
11928                    format!("failed to wait after drain kill: {source}"),
11929                    failure,
11930                );
11931                SuperviseError::Wait {
11932                    module_id: module_id.to_string(),
11933                    source,
11934                }
11935            })?;
11936            classify_reaped_child_exit(snapshot, &child, &status)
11937        }
11938    };
11939
11940    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11941    publish_after_terminal_record(
11942        module_id,
11943        || {
11944            record_terminal_with_detail(
11945                module_id,
11946                terminal_ring,
11947                spawn_events,
11948                &exit_report,
11949                terminal_disposition(final_state),
11950                detail,
11951            )
11952        },
11953        || {
11954            update_snapshot(snapshot, Some(module_id), |state| {
11955                state.state = final_state;
11956                if final_state == ModuleState::Failed {
11957                    state.failed_retry.failed_at = Some(Instant::now());
11958                }
11959                if let Some(enabled) = enabled {
11960                    state.enabled = enabled;
11961                }
11962                clear_current_process_facts(state);
11963                state.last_exit = Some(exit_report.clone());
11964                if exit_report.kind == ExitKind::DeliberateSeverance {
11965                    state.lifetime_restarts += 1;
11966                }
11967            })
11968        },
11969    )?;
11970    child.drain_stderr(module_id).await;
11971
11972    let released =
11973        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11974    child
11975        .finish_exit(
11976            released.is_ok()
11977                && final_state == ModuleState::Restarting
11978                && child.spawn_failure.is_none(),
11979        )
11980        .await
11981        .map_err(|source| SuperviseError::Wait {
11982            module_id: module_id.to_owned(),
11983            source,
11984        })?;
11985    released
11986}
11987
11988/// Ask a child that nothing else has asked to stop, by signal.
11989///
11990/// A registered subc module is asked over its own connection: the drain sends
11991/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11992/// module GOODBYE, and the module stops itself. A module that speaks no subc
11993/// wire receives none of that, and neither does a subc module that has not
11994/// registered yet, so for them the drain budget would be pure delay in front of
11995/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11996/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11997/// into a recovery on the next start.
11998///
11999/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
12000/// rule rather than an optimisation: that module's graceful stop is already
12001/// running by the time its child is drained, and a signal would race it.
12002///
12003/// Best-effort by construction. A child that has already exited is the ordinary
12004/// case rather than an error (the kill lands on a reaped or exiting pid), so a
12005/// failure is logged at debug and the wait-then-kill below still decides the
12006/// outcome.
12007#[cfg(unix)]
12008fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12009    let Some(pid) = child
12010        .id()
12011        .and_then(|pid| i32::try_from(pid).ok())
12012        .and_then(rustix::process::Pid::from_raw)
12013    else {
12014        debug!(
12015            module_id,
12016            "no pid to signal for teardown; falling through to the drain wait"
12017        );
12018        return;
12019    };
12020    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12021        Ok(()) => debug!(
12022            module_id,
12023            "sent SIGTERM to a module nothing else asked to stop"
12024        ),
12025        Err(err) => debug!(
12026            module_id,
12027            error = %err,
12028            "SIGTERM to module failed; the drain wait and kill still apply"
12029        ),
12030    }
12031}
12032
12033/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
12034/// Windows does offer need cooperation this supervisor cannot assume: a console
12035/// control event requires sharing a console with the child, and `WM_CLOSE`
12036/// requires the child to pump a message loop. A supervised server process does
12037/// neither, so there is nothing to send and teardown is the wait followed by the
12038/// kill. Emulating a signal here would mean inventing a stop protocol, which is
12039/// the thing `protocol: "none"` exists to avoid.
12040#[cfg(not(unix))]
12041fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12042    debug!(
12043        module_id,
12044        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12045    );
12046}
12047
12048fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12049    match final_state {
12050        ModuleState::Stopped => TerminalDisposition::Stopped,
12051        ModuleState::Disabled => TerminalDisposition::Disabled,
12052        ModuleState::Restarting => TerminalDisposition::Restarting,
12053        ModuleState::Failed => TerminalDisposition::Failed,
12054        ModuleState::Starting
12055        | ModuleState::Running
12056        | ModuleState::Unresponsive
12057        | ModuleState::Draining => {
12058            unreachable!("terminal exits only finish in terminal or restarting states")
12059        }
12060    }
12061}
12062
12063/// Release a reaped child's registration before allowing another spawn.
12064///
12065/// EOF is not a process-lifetime signal: an inherited socket can stay open
12066/// indefinitely, and serial frame dispatch can be waiting on egress instead of
12067/// reading EOF. After the normal release grace, request connection close (which
12068/// cancels both reads and dispatch), then allow one more release grace for the
12069/// connection guard's forwarding cleanup. Never evict a different connection.
12070async fn release_dead_registration(
12071    registry: &Registry,
12072    forwarding: Option<&ForwardingTable>,
12073    snapshot: &SharedSnapshot,
12074    module_id: &str,
12075    failure: &FailureContext,
12076) -> Result<(), SuperviseError> {
12077    let result = async {
12078        let registration = registry
12079            .get_module(module_id)
12080            .map_err(SuperviseError::Registry)?;
12081        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12082            Ok(()) => return Ok(()),
12083            Err(SuperviseError::RegistrationStillActive { .. }) => {}
12084            Err(err) => return Err(err),
12085        }
12086        let pid = lock_snapshot(snapshot)?.reaped_pid;
12087        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12088            warn!(
12089                module_id,
12090                pid,
12091                connection_id = registration.connection_id.get(),
12092                "reaped module registration outlived release grace; closing dead connection"
12093            );
12094            forwarding.request_connection_close(
12095                registration.connection_id,
12096                CloseReason::new(
12097                    "supervised_process_reaped",
12098                    format!("module '{module_id}' pid {pid} exited"),
12099                ),
12100            );
12101            wait_for_slot_registration_release(
12102                registry,
12103                crate::registry::RegistrationSlot::Connection(registration.connection_id),
12104                REGISTRY_RELEASE_TIMEOUT,
12105            )
12106            .await?;
12107        }
12108        wait_for_registration_release(registry, module_id, Duration::ZERO).await
12109    }
12110    .await;
12111    if let Err(err) = &result {
12112        fail_snapshot(snapshot, Some(module_id), None, failure);
12113        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12114    }
12115    result
12116}
12117
12118/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
12119/// plain stop or restart waits for before it spawns a replacement.
12120async fn wait_for_registration_release(
12121    registry: &Registry,
12122    module_id: &str,
12123    wait: Duration,
12124) -> Result<(), SuperviseError> {
12125    wait_for_slot_registration_release(
12126        registry,
12127        crate::registry::RegistrationSlot::Active(module_id),
12128        wait,
12129    )
12130    .await
12131}
12132
12133/// Wait for the registration in `slot` to go away.
12134///
12135/// Keyed on the slot rather than the bare module id because a successful swap
12136/// never empties the id's active slot (the promoted candidate is in it), so an
12137/// id-keyed wait for the incumbent's release would always time out. Draining a
12138/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
12139/// incumbent's connection instead.
12140async fn wait_for_slot_registration_release(
12141    registry: &Registry,
12142    slot: crate::registry::RegistrationSlot<'_>,
12143    wait: Duration,
12144) -> Result<(), SuperviseError> {
12145    let deadline = Instant::now() + wait;
12146    let mut release_events = registration_release_events().subscribe();
12147    let still_active = |registration: &crate::registry::ModuleRegistration| {
12148        SuperviseError::RegistrationStillActive {
12149            module_id: registration.manifest.module_id.clone(),
12150            waited: wait,
12151        }
12152    };
12153    loop {
12154        let _observed_generation = *release_events.borrow_and_update();
12155        let Some(registration) = registry
12156            .registration(slot)
12157            .map_err(SuperviseError::Registry)?
12158        else {
12159            return Ok(());
12160        };
12161
12162        let now = Instant::now();
12163        if now >= deadline {
12164            return Err(still_active(&registration));
12165        }
12166
12167        let remaining = deadline.saturating_duration_since(now);
12168        match timeout(remaining, release_events.changed()).await {
12169            Ok(Ok(())) | Ok(Err(_)) => {}
12170            Err(_) => return Err(still_active(&registration)),
12171        }
12172    }
12173}
12174
12175#[cfg(test)]
12176mod slot_registration_wait_tests {
12177    use super::*;
12178    use crate::registry::{ConnectionId, RegistrationSlot};
12179    use subc_protocol::manifest::ModuleManifest;
12180
12181    #[tokio::test]
12182    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12183        let registry = Arc::new(Registry::default());
12184        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
12185        let runtime = supervisor.runtime_config();
12186        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12187        let spec = ModuleSpec {
12188            module_id: "enable-stale-registration".to_string(),
12189            program: PathBuf::from("/missing/enable-retry-test"),
12190            args: Vec::new(),
12191            env: Vec::new(),
12192            reserved: false,
12193            reserved_prefixes: Vec::new(),
12194            protocol: ModuleProtocol::Subc,
12195            overlap: Default::default(),
12196            carry_containment: false,
12197        };
12198        let connection = ConnectionId::new(90);
12199        registry
12200            .register_with_control_ops(
12201                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12202                1,
12203                connection,
12204                Vec::new(),
12205            )
12206            .unwrap();
12207        let mut child = None;
12208        let err = set_child_enabled(
12209            &spec,
12210            &runtime,
12211            &registry,
12212            &supervisor.process_liveness,
12213            &snapshot,
12214            &mut child,
12215            true,
12216            &runtime.failure_context(false),
12217        )
12218        .await
12219        .unwrap_err();
12220        assert!(matches!(
12221            err,
12222            SuperviseError::RegistrationStillActive { .. }
12223        ));
12224        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12225        assert!(child.is_none());
12226        registry.deregister_connection(connection).unwrap();
12227        let err = set_child_enabled(
12228            &spec,
12229            &runtime,
12230            &registry,
12231            &supervisor.process_liveness,
12232            &snapshot,
12233            &mut child,
12234            true,
12235            &runtime.failure_context(false),
12236        )
12237        .await
12238        .unwrap_err();
12239        assert!(
12240            matches!(err, SuperviseError::Spawn { .. }),
12241            "second enable must attempt a spawn: {err}"
12242        );
12243        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12244    }
12245
12246    const INCUMBENT: u64 = 1;
12247    const CANDIDATE: u64 = 2;
12248
12249    fn swapped_registry() -> Arc<Registry> {
12250        let registry = Arc::new(Registry::default());
12251        let manifest = ModuleManifest::builder("m", "0.1.0").build();
12252        registry
12253            .register_with_control_ops(
12254                manifest.clone(),
12255                1,
12256                ConnectionId::new(INCUMBENT),
12257                Vec::new(),
12258            )
12259            .unwrap();
12260        registry
12261            .register_candidate_with_control_ops(
12262                manifest,
12263                1,
12264                ConnectionId::new(CANDIDATE),
12265                Vec::new(),
12266            )
12267            .unwrap();
12268        registry
12269    }
12270
12271    /// After a promotion the id's active slot is held by the new process, so an
12272    /// id-keyed wait for the incumbent's release can never succeed; the
12273    /// connection-keyed wait completes as soon as the incumbent deregisters.
12274    #[tokio::test]
12275    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12276        let registry = swapped_registry();
12277        registry.promote_candidate("m").unwrap().unwrap();
12278
12279        assert!(matches!(
12280            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
12281            Err(SuperviseError::RegistrationStillActive { .. })
12282        ));
12283
12284        // Still held while the incumbent's connection has not deregistered.
12285        assert!(matches!(
12286            wait_for_slot_registration_release(
12287                &registry,
12288                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12289                Duration::from_millis(50),
12290            )
12291            .await,
12292            Err(SuperviseError::RegistrationStillActive { .. })
12293        ));
12294
12295        let releaser = Arc::clone(&registry);
12296        let release = tokio::spawn(async move {
12297            sleep(Duration::from_millis(20)).await;
12298            releaser
12299                .deregister_connection(ConnectionId::new(INCUMBENT))
12300                .unwrap();
12301            notify_registration_release();
12302        });
12303        wait_for_slot_registration_release(
12304            &registry,
12305            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12306            Duration::from_secs(5),
12307        )
12308        .await
12309        .expect("the incumbent's own registration is released");
12310        release.await.unwrap();
12311        assert!(registry.get_module("m").unwrap().is_some());
12312    }
12313
12314    /// The candidate slot is waited on separately from the active slot: the
12315    /// incumbent's registration neither holds up nor stands in for it.
12316    #[tokio::test]
12317    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12318        let registry = swapped_registry();
12319        assert!(matches!(
12320            wait_for_slot_registration_release(
12321                &registry,
12322                RegistrationSlot::Candidate("m"),
12323                Duration::from_millis(50),
12324            )
12325            .await,
12326            Err(SuperviseError::RegistrationStillActive { .. })
12327        ));
12328        registry
12329            .deregister_connection(ConnectionId::new(CANDIDATE))
12330            .unwrap();
12331        wait_for_slot_registration_release(
12332            &registry,
12333            RegistrationSlot::Candidate("m"),
12334            Duration::from_millis(50),
12335        )
12336        .await
12337        .expect("a candidate slot with no candidate is released");
12338        assert!(registry
12339            .registration(RegistrationSlot::Active("m"))
12340            .unwrap()
12341            .is_some());
12342    }
12343}
12344
12345fn classify_exit(status: &ExitStatus) -> ExitReport {
12346    ExitReport {
12347        kind: if status.success() {
12348            ExitKind::Clean
12349        } else {
12350            ExitKind::Crash
12351        },
12352        code: status.code(),
12353        signal: exit_signal(status),
12354        at_ms: unix_ms_now(),
12355    }
12356}
12357
12358/// The terminal record for a module whose `wait()` call itself errored (e.g. the
12359/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
12360/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
12361/// disposition still must be `Failed` so the terminal ring is not silently missing
12362/// an entry, matching what `fail_snapshot` records for this same arm.
12363fn wait_error_exit_report() -> ExitReport {
12364    ExitReport {
12365        kind: ExitKind::Crash,
12366        code: None,
12367        signal: None,
12368        at_ms: unix_ms_now(),
12369    }
12370}
12371
12372#[cfg(unix)]
12373fn exit_signal(status: &ExitStatus) -> Option<i32> {
12374    use std::os::unix::process::ExitStatusExt;
12375
12376    status.signal()
12377}
12378
12379#[cfg(not(unix))]
12380fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12381    None
12382}
12383
12384/// Give an operator-touched module its full crash budget back.
12385///
12386/// Named for the counter it used to zero; it now empties the in-window ring,
12387/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
12388/// ledger of what happened survives every operator action.
12389fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12390    update_snapshot(snapshot, Some(module_id), |state| {
12391        state.clear_crash_restarts();
12392    })
12393}
12394
12395fn set_running(
12396    snapshot: &SharedSnapshot,
12397    child: &SupervisedChild,
12398    module_id: &str,
12399    spawn_events: &SpawnEventFeed,
12400) -> Result<(), SuperviseError> {
12401    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12402        module_id: Some(module_id.to_string()),
12403    })?;
12404    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12405    if std::mem::take(&mut state.coalesced_restart_pending) {
12406        let generation = state.spawn_generation;
12407        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12408    }
12409    state.drain_disposition_detail = None;
12410    state.spawn_failure = None;
12411    state.waiting_for_previous_pid = None;
12412    state.waiting_for_previous_detail = None;
12413    // Every caller of this is a plain spawn, which always uses the primary key;
12414    // a promoted swap candidate sets the flag itself after this returns.
12415    state.in_alternate_slot = false;
12416    state.configuration_updated_since_spawn = false;
12417    state.spawned_protocol = Some(child.protocol);
12418    state.state = ModuleState::Running;
12419    state.enabled = true;
12420    state.process_alive = true;
12421    state.pid = child.id();
12422    state.subtree_target = child.subtree_target.clone();
12423    // Probe samples belong to one process. Keep module-level escalation and
12424    // late-answer history, but never charge a new child for its predecessor's misses.
12425    state.health.status = SupervisorHealthStatus::Unknown;
12426    state.health.consecutive_failures = 0;
12427    state.health.detail = None;
12428    state.health.metrics = None;
12429    state.health.last_probe_ms = None;
12430    state.failed_retry.healthy_since = None;
12431    state.failed_retry.failed_at = None;
12432    state.failed_retry.running_since = Some(child.spawned_at);
12433    #[cfg(windows)]
12434    {
12435        state.pipe_consumed = child
12436            .nonce_delivery
12437            .as_ref()
12438            .map(|delivery| delivery.consumed());
12439    }
12440    #[cfg(target_os = "macos")]
12441    {
12442        state.report_ready = Some(Arc::clone(&child.report_ready));
12443    }
12444    state.spawned_at_ms = Some(child.spawned_at_ms);
12445    state.exec_outcome = Some(Arc::clone(&child.exec.0));
12446    state.spawned_from = Some(child.spawned_from.clone());
12447    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12448    state.spawned_file_identity = child.spawned_file_identity.clone();
12449    state.process_start_time = child.process_start_time;
12450    Ok(())
12451}
12452
12453fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12454    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12455    state.failed_retry.healthy_since = None;
12456    state.failed_retry.running_since = None;
12457    state.waiting_for_previous_pid = None;
12458    state.waiting_for_previous_detail = None;
12459    state.process_alive = false;
12460    state.spawned_protocol = None;
12461    state.pid = None;
12462    #[cfg(windows)]
12463    {
12464        state.pipe_consumed = None;
12465    }
12466    #[cfg(target_os = "macos")]
12467    {
12468        state.report_ready = None;
12469    }
12470    state.spawned_at_ms = None;
12471    state.exec_outcome = None;
12472    state.spawned_from = None;
12473    state.launch_spec_digest = None;
12474    state.spawned_file_identity = None;
12475    state.process_start_time = None;
12476    state.deliberate_severance = None;
12477}
12478
12479#[cfg(test)]
12480fn record_deliberate_severance(
12481    snapshot: &SharedSnapshot,
12482    identity: ProcessIdentity,
12483) -> Result<(), SuperviseError> {
12484    update_snapshot(snapshot, None, |state| {
12485        state.deliberate_severance = Some(identity);
12486    })
12487}
12488
12489fn apply_deliberate_severance_marker(
12490    snapshot: &SharedSnapshot,
12491    exited_identity: Option<ProcessIdentity>,
12492    mut exit_report: ExitReport,
12493) -> ExitReport {
12494    let marker = lock_snapshot(snapshot)
12495        .ok()
12496        .and_then(|mut state| state.deliberate_severance.take());
12497    if marker.is_some() && marker == exited_identity {
12498        exit_report.kind = ExitKind::DeliberateSeverance;
12499    }
12500    exit_report
12501}
12502
12503fn classify_reaped_child_exit(
12504    snapshot: &SharedSnapshot,
12505    child: &SupervisedChild,
12506    status: &ExitStatus,
12507) -> ExitReport {
12508    let _ = update_snapshot(snapshot, None, |state| {
12509        state.reaped_pid = Some(child.pid);
12510        state.spawn_failure = child.spawn_failure.clone();
12511    });
12512    if child.spawn_failure.as_deref() == Some("registration_timeout") {
12513        registration_failure_exit_report(classify_exit(status))
12514    } else {
12515        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12516    }
12517}
12518
12519fn fail_snapshot(
12520    snapshot: &SharedSnapshot,
12521    module_id: Option<&str>,
12522    last_exit: Option<ExitReport>,
12523    failure: &FailureContext,
12524) {
12525    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12526        state.state = ModuleState::Failed;
12527        clear_current_process_facts(state);
12528        if failure.may_retry() && state.enabled {
12529            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12530        } else if failure.shutdown.is_set() {
12531            state.failed_retry.cancel();
12532        }
12533        if let Some(last_exit) = last_exit {
12534            state.last_exit = Some(last_exit);
12535        }
12536    }) {
12537        error!(error = %err, "failed to mark supervisor state failed");
12538    }
12539}
12540
12541fn update_snapshot(
12542    snapshot: &SharedSnapshot,
12543    module_id: Option<&str>,
12544    update: impl FnOnce(&mut SupervisorSnapshot),
12545) -> Result<(), SuperviseError> {
12546    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12547        module_id: module_id.map(ToOwned::to_owned),
12548    })?;
12549    update(&mut state);
12550    Ok(())
12551}
12552
12553const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12554
12555fn lock_snapshot_for_control<'a>(
12556    snapshot: &'a SharedSnapshot,
12557    module_id: &str,
12558    caller: &'static str,
12559) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12560    let started_at = Instant::now();
12561    let guard = lock_snapshot(snapshot)?;
12562    let waited = started_at.elapsed();
12563    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12564        warn!(
12565            module_id = %module_id,
12566            waited_ms = waited.as_millis() as u64,
12567            caller = %caller,
12568            "slow snapshot lock"
12569        );
12570    }
12571    Ok(guard)
12572}
12573
12574fn lock_snapshot(
12575    snapshot: &SharedSnapshot,
12576) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12577    snapshot
12578        .lock()
12579        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12580}
12581
12582#[cfg(test)]
12583mod terminal_history_tests {
12584    use std::{
12585        path::PathBuf,
12586        sync::Arc,
12587        time::{Duration, Instant},
12588    };
12589
12590    use tokio::{sync::mpsc, time::sleep};
12591
12592    use super::{
12593        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12594        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12595        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12596        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12597        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12598        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12599        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12600        SupervisorHealthStatus, SupervisorSnapshot,
12601    };
12602    // The supervisor's clock, distinct from the `std::time::Instant` these tests
12603    // use for their own wall-clock deadlines: crash-restart instants must be on
12604    // the same clock the production code stamps them with, which is tokio's (and
12605    // is what `start_paused` tests can move).
12606    use super::Instant as ClockInstant;
12607    use crate::{
12608        registry::Registry,
12609        terminal_ring::{TerminalRing, TerminalRingConfig},
12610    };
12611    use std::sync::Mutex;
12612    use subc_control::TerminalDisposition;
12613
12614    /// See the twin in `control.rs` for why this derives the path from
12615    /// `current_exe()` and why the existence check is here: `--lib` alone does
12616    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
12617    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
12618    pub(super) fn fake_aft_stub_path() -> PathBuf {
12619        let mut path = std::env::current_exe().expect("current_exe available in tests");
12620        path.pop();
12621        path.pop();
12622        path.push(if cfg!(windows) {
12623            "fake-aft-stub.exe"
12624        } else {
12625            "fake-aft-stub"
12626        });
12627        assert!(
12628            path.exists(),
12629            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12630             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12631            path.display()
12632        );
12633        path
12634    }
12635
12636    #[test]
12637    fn reserved_never_spawned_refuses_every_hello() {
12638        // The canary hole: a reserved id whose module has never spawned had NO
12639        // gate entry and admitted anyone -- the reservation protected the nonce
12640        // holder, not the NAME. Now the entry is present with no legitimate
12641        // holder and refuses all comers.
12642        let supervisor = SupervisorHandle::default();
12643        supervisor.apply_identity_configuration(&ModuleSpec {
12644            module_id: "never-spawned".to_string(),
12645            program: PathBuf::from("/usr/bin/false"),
12646            args: Vec::new(),
12647            env: Vec::new(),
12648            reserved: true,
12649            reserved_prefixes: Vec::new(),
12650            protocol: ModuleProtocol::Subc,
12651            overlap: Default::default(),
12652            carry_containment: false,
12653        });
12654        assert!(
12655            supervisor
12656                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12657                .is_err(),
12658            "forged nonce must refuse on a reserved never-spawned id"
12659        );
12660        assert!(
12661            supervisor
12662                .reserved_hello_verification("never-spawned", None)
12663                .is_err(),
12664            "absent nonce must refuse on a reserved never-spawned id"
12665        );
12666        // And a real spawn nonce minted later admits exactly that nonce.
12667        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12668        supervisor.apply_identity_configuration(&ModuleSpec {
12669            module_id: "never-spawned".to_string(),
12670            program: PathBuf::from("/usr/bin/false"),
12671            args: Vec::new(),
12672            env: Vec::new(),
12673            reserved: true,
12674            reserved_prefixes: Vec::new(),
12675            protocol: ModuleProtocol::Subc,
12676            overlap: Default::default(),
12677            carry_containment: false,
12678        });
12679        assert!(supervisor
12680            .reserved_hello_verification("never-spawned", Some("minted"))
12681            .is_ok());
12682        assert!(supervisor
12683            .reserved_hello_verification("never-spawned", Some("forged"))
12684            .is_err());
12685    }
12686
12687    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12688    /// happened, which is what "spent budget" looks like to every reader.
12689    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12690        let now = ClockInstant::now();
12691        for _ in 0..count {
12692            state.crash_restarts.push_back(now);
12693        }
12694    }
12695
12696    /// Age the oldest recorded restart out of `window`, standing in for the hours
12697    /// that would otherwise have to pass. Injecting the instant is the point: a
12698    /// test that slept a real window would take ten minutes and still prove less.
12699    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12700        let aged = state
12701            .crash_restarts
12702            .front()
12703            .expect("a crash restart must be recorded before it can be aged")
12704            .checked_sub(window + Duration::from_secs(1))
12705            .expect("the test clock is far enough from its origin to age an instant");
12706        state.crash_restarts[0] = aged;
12707    }
12708
12709    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12710        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12711        seed_crash_restarts(&mut state, count);
12712        state
12713    }
12714
12715    #[test]
12716    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12717        let policy = RestartPolicy::new(3, Duration::ZERO);
12718        let now = ClockInstant::now();
12719        assert!(daemon_will_restart(
12720            &mut snapshot_with_restarts(true, 2),
12721            &policy,
12722            now
12723        ));
12724        assert!(!daemon_will_restart(
12725            &mut snapshot_with_restarts(true, 3),
12726            &policy,
12727            now
12728        ));
12729        assert!(!daemon_will_restart(
12730            &mut snapshot_with_restarts(false, 0),
12731            &policy,
12732            now
12733        ));
12734    }
12735
12736    #[test]
12737    fn crash_restart_backoff_escalates_with_in_window_count() {
12738        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12739            .with_max_backoff(Duration::from_secs(30));
12740        let now = ClockInstant::now();
12741        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12742        let schedules = (0..4)
12743            .map(|_| {
12744                state
12745                    .next_crash_restart(&policy, now)
12746                    .expect("the test policy allows four crash restarts")
12747            })
12748            .collect::<Vec<_>>();
12749
12750        assert_eq!(
12751            schedules
12752                .iter()
12753                .map(|schedule| schedule.restart_in_window)
12754                .collect::<Vec<_>>(),
12755            vec![0, 1, 2, 3]
12756        );
12757        assert_eq!(
12758            schedules
12759                .iter()
12760                .map(|schedule| schedule.delay)
12761                .collect::<Vec<_>>(),
12762            vec![
12763                Duration::from_millis(100),
12764                Duration::from_secs(1),
12765                Duration::from_secs(10),
12766                Duration::from_secs(30),
12767            ]
12768        );
12769    }
12770
12771    #[test]
12772    fn crash_restart_backoff_resets_after_ring_clear() {
12773        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12774        let now = ClockInstant::now();
12775        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12776        assert_eq!(
12777            state.next_crash_restart(&policy, now).unwrap().delay,
12778            Duration::from_millis(100)
12779        );
12780        assert_eq!(
12781            state.next_crash_restart(&policy, now).unwrap().delay,
12782            Duration::from_secs(1)
12783        );
12784
12785        state.clear_crash_restarts();
12786        let schedule = state
12787            .next_crash_restart(&policy, now)
12788            .expect("a cleared ring must allow another restart");
12789        assert_eq!(schedule.restart_in_window, 0);
12790        assert_eq!(schedule.delay, Duration::from_millis(100));
12791    }
12792
12793    #[test]
12794    fn crash_restart_backoff_ignores_aged_restarts() {
12795        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12796        let now = ClockInstant::now();
12797        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12798        state
12799            .next_crash_restart(&policy, now)
12800            .expect("the first restart is allowed");
12801        state
12802            .next_crash_restart(&policy, now)
12803            .expect("the second restart is allowed");
12804        state.crash_restarts[0] = now
12805            .checked_sub(policy.window + Duration::from_secs(1))
12806            .expect("the fake clock can age a restart past the window");
12807
12808        let schedule = state
12809            .next_crash_restart(&policy, now)
12810            .expect("an aged restart must release its slot");
12811        assert_eq!(schedule.restart_in_window, 1);
12812        assert_eq!(schedule.delay, Duration::from_secs(1));
12813        assert_eq!(state.crash_restarts.len(), 2);
12814    }
12815
12816    /// The budget is a rate: the same three spent restarts refuse a respawn
12817    /// while they are recent and allow one once they have aged past the window.
12818    /// Nothing about the module changed in between, which is the whole point.
12819    #[test]
12820    fn a_budget_spent_before_the_window_no_longer_refuses() {
12821        let policy = RestartPolicy::new(3, Duration::ZERO);
12822        let mut state = snapshot_with_restarts(true, 3);
12823        let now = ClockInstant::now();
12824        assert!(!daemon_will_restart(&mut state, &policy, now));
12825
12826        assert!(daemon_will_restart(
12827            &mut state,
12828            &policy,
12829            now + policy.window + Duration::from_secs(1)
12830        ));
12831        assert!(
12832            state.crash_restarts.is_empty(),
12833            "reading the budget must drop the instants that left the window"
12834        );
12835    }
12836
12837    fn module_with_recovery_snapshot(
12838        state: ModuleState,
12839        enabled: bool,
12840        restart_count: u32,
12841    ) -> SupervisedModule {
12842        let registry = Arc::new(Registry::default());
12843        let supervisor =
12844            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12845        let runtime = supervisor.runtime_config();
12846        let spec = ModuleSpec {
12847            module_id: "recovery-snapshot".to_string(),
12848            program: fake_aft_stub_path(),
12849            args: Vec::new(),
12850            env: Vec::new(),
12851            reserved: false,
12852            reserved_prefixes: Vec::new(),
12853            protocol: ModuleProtocol::Subc,
12854            overlap: Default::default(),
12855            carry_containment: false,
12856        };
12857        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12858        seed_crash_restarts(&mut snapshot, restart_count);
12859        // These tests read synthetic snapshots. A real child and monitor would
12860        // race those reads by replacing the requested state during startup.
12861        let (commands, _rx) = mpsc::channel(4);
12862        SupervisedModule {
12863            inner: Arc::new(SupervisedModuleInner {
12864                module_id: spec.module_id.clone(),
12865                registry,
12866                snapshot: Arc::new(Mutex::new(snapshot)),
12867                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12868                    spec,
12869                    health: runtime.health,
12870                })),
12871                stderr_ring: runtime.stderr_ring,
12872                terminal_ring: runtime.terminal_ring,
12873                commands,
12874                monitor: Mutex::new(None),
12875                restart_policy: runtime.restart_policy,
12876                effective_drain_timeout: runtime.effective_drain_timeout,
12877                provenance_probe: supervisor.provenance_probe.clone(),
12878                retained: runtime.child_roster.retained_view(),
12879            }),
12880        }
12881    }
12882
12883    #[cfg(target_os = "linux")]
12884    #[tokio::test]
12885    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12886        let supervisor =
12887            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12888                .with_cgroup_placement(None);
12889        let result = supervisor.spawn(ModuleSpec {
12890            module_id: "no-cgroup-placement".to_string(),
12891            program: fake_aft_stub_path(),
12892            args: Vec::new(),
12893            env: Vec::new(),
12894            reserved: false,
12895            reserved_prefixes: Vec::new(),
12896            protocol: ModuleProtocol::Subc,
12897            overlap: Default::default(),
12898            carry_containment: false,
12899        });
12900
12901        assert!(
12902            result.is_ok(),
12903            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12904        );
12905    }
12906
12907    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12908    async fn undecided_snapshot_uses_shared_restart_predicate() {
12909        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12910            .will_recover_after_connection_loss()
12911            .unwrap());
12912        assert!(
12913            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12914                .will_recover_after_connection_loss()
12915                .unwrap()
12916        );
12917    }
12918
12919    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12920    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12921        assert!(
12922            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12923                .will_recover_after_connection_loss()
12924                .unwrap()
12925        );
12926    }
12927
12928    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12929    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12930        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12931            .will_recover_after_connection_loss()
12932            .unwrap());
12933        assert!(
12934            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12935                .will_recover_after_connection_loss()
12936                .unwrap()
12937        );
12938    }
12939
12940    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12941    async fn warming_snapshot_is_limited_to_startup_phases() {
12942        for state in [
12943            ModuleState::Starting,
12944            ModuleState::Running,
12945            ModuleState::Restarting,
12946        ] {
12947            assert!(
12948                module_with_recovery_snapshot(state, true, 0)
12949                    .is_warming()
12950                    .unwrap(),
12951                "{state:?} should be warming"
12952            );
12953        }
12954        for state in [
12955            ModuleState::Unresponsive,
12956            ModuleState::Draining,
12957            ModuleState::Stopped,
12958            ModuleState::Failed,
12959            ModuleState::Disabled,
12960        ] {
12961            assert!(
12962                !module_with_recovery_snapshot(state, true, 0)
12963                    .is_warming()
12964                    .unwrap(),
12965                "{state:?} should not be warming"
12966            );
12967        }
12968    }
12969
12970    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12971    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12972        let registry = Arc::new(Registry::default());
12973        let supervisor =
12974            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12975        let module = supervisor
12976            .spawn(ModuleSpec {
12977                module_id: "terminal-history".to_string(),
12978                program: fake_aft_stub_path(),
12979                args: Vec::new(),
12980                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12981                reserved: false,
12982                reserved_prefixes: Vec::new(),
12983                protocol: ModuleProtocol::Subc,
12984                overlap: Default::default(),
12985                carry_containment: false,
12986            })
12987            .unwrap();
12988
12989        let deadline = Instant::now() + Duration::from_secs(5);
12990        loop {
12991            let history = module.terminal_history();
12992            if history.entries.len() == 2 {
12993                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12994                assert_eq!(history.dropped, 0);
12995                assert_eq!(
12996                    history
12997                        .entries
12998                        .iter()
12999                        .map(|entry| entry.exit_code)
13000                        .collect::<Vec<_>>(),
13001                    vec![Some(23), Some(23)]
13002                );
13003                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13004                return;
13005            }
13006            assert!(
13007                Instant::now() < deadline,
13008                "module did not retain two terminal exits: {history:?}"
13009            );
13010            sleep(Duration::from_millis(10)).await;
13011        }
13012    }
13013
13014    /// A disable issued while a crash respawn is still backing off must preempt
13015    /// that respawn: the operator's stop wins, the disable must not queue behind
13016    /// the backoff, and the module must never come back up afterwards.
13017    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13018    async fn disable_during_crash_backoff_cancels_pending_respawn() {
13019        let backoff = Duration::from_secs(2);
13020        let supervisor = Supervisor::new_for_test(
13021            Arc::new(Registry::default()),
13022            RestartPolicy::new(10, backoff),
13023        );
13024        let module = supervisor
13025            .spawn(ModuleSpec {
13026                module_id: "disable-during-backoff".to_string(),
13027                program: fake_aft_stub_path(),
13028                args: Vec::new(),
13029                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13030                reserved: false,
13031                reserved_prefixes: Vec::new(),
13032                protocol: ModuleProtocol::Subc,
13033                overlap: Default::default(),
13034                carry_containment: false,
13035            })
13036            .unwrap();
13037
13038        // Wait for the first crash to put the module into its backoff window.
13039        let deadline = Instant::now() + Duration::from_secs(5);
13040        loop {
13041            if module.status().unwrap().state == ModuleState::Restarting {
13042                break;
13043            }
13044            assert!(
13045                Instant::now() < deadline,
13046                "module never entered the crash backoff"
13047            );
13048            sleep(Duration::from_millis(10)).await;
13049        }
13050
13051        let started = Instant::now();
13052        module.set_enabled(false).await.unwrap();
13053        let waited = started.elapsed();
13054
13055        assert!(
13056            waited < backoff / 2,
13057            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13058        );
13059        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13060
13061        // Outlast the backoff: the respawn it was counting down to must never run.
13062        sleep(backoff + Duration::from_millis(500)).await;
13063        let status = module.status().unwrap();
13064        assert_eq!(status.state, ModuleState::Disabled);
13065        assert_eq!(
13066            status.spawn_generation, 1,
13067            "module respawned after the operator disabled it"
13068        );
13069    }
13070
13071    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
13072    /// the shape of nats-server, the program this rule exists for.
13073    #[cfg(unix)]
13074    fn protocol_none_sigterm_exits_clean_spec(
13075        module_id: &str,
13076        dir: &std::path::Path,
13077    ) -> (ModuleSpec, PathBuf, PathBuf) {
13078        let ready = dir.join("ready");
13079        let marker = dir.join("sigterm");
13080        let spec = ModuleSpec {
13081            module_id: module_id.to_string(),
13082            program: fake_aft_stub_path(),
13083            args: Vec::new(),
13084            env: vec![
13085                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13086                (
13087                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13088                    marker.display().to_string(),
13089                ),
13090                (
13091                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13092                    ready.display().to_string(),
13093                ),
13094            ],
13095            reserved: false,
13096            reserved_prefixes: Vec::new(),
13097            protocol: ModuleProtocol::None,
13098            overlap: Default::default(),
13099            carry_containment: false,
13100        };
13101        (spec, ready, marker)
13102    }
13103
13104    /// Wait for a file the child writes, so a signal is never sent before the
13105    /// child's SIGTERM handler is installed (the default disposition would
13106    /// kill it by signal and the exit would not be clean).
13107    #[cfg(unix)]
13108    async fn wait_for_file(path: &std::path::Path) {
13109        let deadline = Instant::now() + Duration::from_secs(10);
13110        while !path.exists() {
13111            assert!(
13112                Instant::now() < deadline,
13113                "{} never appeared",
13114                path.display()
13115            );
13116            sleep(Duration::from_millis(10)).await;
13117        }
13118    }
13119
13120    /// A protocol-none module that exits 0 because something OUTSIDE the
13121    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
13122    /// the crash-path disposition rather than `stopped`.
13123    #[cfg(unix)]
13124    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13125    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13126        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13127        let (spec, ready, marker) =
13128            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13129        let supervisor = Supervisor::new_for_test(
13130            Arc::new(Registry::default()),
13131            RestartPolicy::new(3, Duration::ZERO),
13132        );
13133        let module = supervisor.spawn(spec).unwrap();
13134        wait_for_file(&ready).await;
13135        // The ready file proves the child installed its SIGTERM handler, not
13136        // that the supervisor has processed the privacy trampoline's exec
13137        // acknowledgement. On macOS status withholds the pid until then.
13138        let deadline = Instant::now() + Duration::from_secs(10);
13139        let first_pid = loop {
13140            let status = module.status().unwrap();
13141            if status.state == ModuleState::Running {
13142                if let Some(pid) = status.pid {
13143                    break pid;
13144                }
13145            }
13146            assert!(
13147                Instant::now() < deadline,
13148                "a running module must report its pid after exec confirmation: {status:?}"
13149            );
13150            sleep(Duration::from_millis(10)).await;
13151        };
13152
13153        rustix::process::kill_process(
13154            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13155            rustix::process::Signal::TERM,
13156        )
13157        .unwrap();
13158
13159        let deadline = Instant::now() + Duration::from_secs(10);
13160        let respawned = loop {
13161            let status = module.status().unwrap();
13162            if status.state == ModuleState::Running
13163                && status.pid.is_some_and(|pid| pid != first_pid)
13164            {
13165                break status;
13166            }
13167            assert!(
13168                Instant::now() < deadline,
13169                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13170            );
13171            sleep(Duration::from_millis(10)).await;
13172        };
13173        assert_eq!(respawned.spawn_generation, 2);
13174        assert!(
13175            marker.exists(),
13176            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13177        );
13178
13179        let history = module.terminal_history();
13180        assert_eq!(history.entries.len(), 1, "{history:?}");
13181        let entry = &history.entries[0];
13182        assert_eq!(entry.exit_code, Some(0));
13183        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13184        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13185
13186        module.stop().await.unwrap();
13187    }
13188
13189    /// Repeated unrequested clean exits of a protocol-none module spend the
13190    /// restart budget exactly as crashes do, and the module ends `failed` with
13191    /// the budget named.
13192    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13193    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13194        let supervisor = Supervisor::new_for_test(
13195            Arc::new(Registry::default()),
13196            RestartPolicy::new(1, Duration::ZERO),
13197        );
13198        let module = supervisor
13199            .spawn(ModuleSpec {
13200                module_id: "none-clean-exit-budget".to_string(),
13201                program: fake_aft_stub_path(),
13202                args: Vec::new(),
13203                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13204                reserved: false,
13205                reserved_prefixes: Vec::new(),
13206                protocol: ModuleProtocol::None,
13207                overlap: Default::default(),
13208                carry_containment: false,
13209            })
13210            .unwrap();
13211
13212        // Failed follows the terminal write; this deadline only bounds a hang,
13213        // not an assumed duration for the two launches or their exit recording.
13214        let deadline = Instant::now() + Duration::from_secs(10);
13215        loop {
13216            let status = module.status().unwrap();
13217            if status.state == ModuleState::Failed {
13218                break;
13219            }
13220            assert!(
13221                Instant::now() < deadline,
13222                "module never exhausted its budget: {status:?} {:?}",
13223                module.terminal_history()
13224            );
13225            sleep(Duration::from_millis(10)).await;
13226        }
13227        let history = module.terminal_history();
13228        assert_eq!(
13229            history
13230                .entries
13231                .iter()
13232                .map(|entry| (entry.exit_code, entry.disposition.clone()))
13233                .collect::<Vec<_>>(),
13234            vec![
13235                (Some(0), TerminalDisposition::Restarting),
13236                (Some(0), TerminalDisposition::Failed),
13237            ]
13238        );
13239        let detail = history.entries[1]
13240            .disposition_detail
13241            .as_deref()
13242            .expect("a budget failure names the budget");
13243        assert!(detail.contains("max_restarts=1"), "{detail}");
13244        assert_eq!(module.status().unwrap().spawn_generation, 2);
13245    }
13246
13247    #[test]
13248    fn restart_budget_failure_is_published_after_its_terminal_record() {
13249        use std::io::{Read, Write};
13250        let supervisor = Supervisor::new_for_test(
13251            Arc::new(Registry::default()),
13252            RestartPolicy::new(0, Duration::ZERO),
13253        );
13254        let runtime = supervisor.runtime_config();
13255        let spec = ModuleSpec {
13256            protocol: ModuleProtocol::None,
13257            ..windowed_crash_spec("budget-publication-order")
13258        };
13259        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13260            ModuleState::Running,
13261            true,
13262        )));
13263        lock_snapshot(&snapshot).unwrap().process_alive = true;
13264        let reaped_snapshot = snapshot.clone();
13265        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13266        let history = runtime.terminal_ring.clone();
13267        let reap = std::thread::spawn(move || {
13268            tokio::runtime::Builder::new_current_thread()
13269                .enable_all()
13270                .build()
13271                .unwrap()
13272                .block_on(on_child_exit(
13273                    &spec,
13274                    runtime.restart_policy,
13275                    &supervisor.registry,
13276                    &reaped_snapshot,
13277                    &runtime.terminal_ring,
13278                    &runtime.spawn_events,
13279                    &runtime.child_roster,
13280                    ExitReport {
13281                        kind: ExitKind::Clean,
13282                        code: Some(0),
13283                        signal: None,
13284                        at_ms: 1,
13285                    },
13286                    true,
13287                ))
13288        });
13289        let mut reached = [0];
13290        barrier.read_exact(&mut reached).unwrap();
13291        assert_eq!(&reached, b"R");
13292        let (before_state, before_alive) = {
13293            let state = lock_snapshot(&snapshot).unwrap();
13294            (state.state, state.process_alive)
13295        };
13296        let before_history = history.lock().unwrap().snapshot();
13297        barrier.write_all(b"X").unwrap();
13298        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13299        assert!(before_history.entries.is_empty());
13300        assert_ne!(
13301            before_state,
13302            ModuleState::Failed,
13303            "Failed was visible before its terminal record could be written"
13304        );
13305        assert!(before_alive);
13306        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13307        let history = history.lock().unwrap().snapshot();
13308        assert_eq!(history.entries.len(), 1);
13309        assert_eq!(history.entries[0].exit_code, Some(0));
13310        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13311    }
13312
13313    #[test]
13314    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13315        use std::io::{Read, Write};
13316
13317        let supervisor = Supervisor::new_for_test(
13318            Arc::new(Registry::default()),
13319            RestartPolicy::new(3, Duration::ZERO),
13320        );
13321        let mut runtime = supervisor.runtime_config();
13322        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13323        let snapshot = module.inner.snapshot.clone();
13324        lock_snapshot(&snapshot).unwrap().process_alive = true;
13325        runtime.terminal_ring = module.inner.terminal_ring.clone();
13326        let spec = windowed_crash_spec(module.module_id());
13327        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13328        let reaped_snapshot = snapshot.clone();
13329        let reap = std::thread::spawn(move || {
13330            tokio::runtime::Builder::new_current_thread()
13331                .enable_all()
13332                .build()
13333                .unwrap()
13334                .block_on(on_child_exit(
13335                    &spec,
13336                    runtime.restart_policy,
13337                    &supervisor.registry,
13338                    &reaped_snapshot,
13339                    &runtime.terminal_ring,
13340                    &runtime.spawn_events,
13341                    &runtime.child_roster,
13342                    ExitReport {
13343                        kind: ExitKind::Crash,
13344                        code: Some(2),
13345                        signal: None,
13346                        at_ms: 1,
13347                    },
13348                    true,
13349                ))
13350        });
13351
13352        // Hold the crash handler between its budget decision and publication.
13353        // Connection cleanup must still see the recovery that this crash chose,
13354        // even when that recovery consumes the last available restart slot.
13355        let mut reached = [0];
13356        barrier.read_exact(&mut reached).unwrap();
13357        assert_eq!(&reached, b"R");
13358        let before_status = module.status().unwrap();
13359        let before_history = module.terminal_history();
13360        let before_recovers = module.will_recover_after_connection_loss().unwrap();
13361        barrier.write_all(b"X").unwrap();
13362        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13363
13364        assert_eq!(before_status.state, ModuleState::Running);
13365        assert!(before_status.process_alive);
13366        assert_eq!(before_status.restart_count, 3);
13367        assert!(before_history.entries.is_empty());
13368        assert!(
13369            before_recovers,
13370            "connection cleanup would report a terminal close for a scheduled crash restart"
13371        );
13372        let restarting = module.status().unwrap();
13373        assert_eq!(restarting.state, ModuleState::Restarting);
13374        assert_eq!(restarting.restart_count, 3);
13375        assert!(module.will_recover_after_connection_loss().unwrap());
13376        let history = module.terminal_history();
13377        assert_eq!(history.entries.len(), 1);
13378        assert_eq!(
13379            history.entries[0].disposition,
13380            TerminalDisposition::Restarting
13381        );
13382
13383        // Once the replacement is running, its next crash must be judged
13384        // against the exhausted budget rather than the previous recovery.
13385        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13386            state.state = ModuleState::Running;
13387            state.respawn_pending = false;
13388        })
13389        .unwrap();
13390        assert!(!module.will_recover_after_connection_loss().unwrap());
13391        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13392            state.enabled = false;
13393            state.respawn_pending = true;
13394        })
13395        .unwrap();
13396        assert!(!module.will_recover_after_connection_loss().unwrap());
13397    }
13398
13399    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13400        use std::net::{TcpListener, TcpStream};
13401        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13402        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13403        let (barrier, _) = listener.accept().unwrap();
13404        for stream in [&writer, &barrier] {
13405            stream
13406                .set_read_timeout(Some(Duration::from_secs(10)))
13407                .unwrap();
13408            stream
13409                .set_write_timeout(Some(Duration::from_secs(10)))
13410                .unwrap();
13411        }
13412        *events.1.lock().unwrap() = Some(writer);
13413        barrier
13414    }
13415
13416    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13417        use std::io::{Read, Write};
13418        let barrier = events.1.lock().unwrap().take();
13419        if let Some(mut barrier) = barrier {
13420            barrier.write_all(b"R").unwrap();
13421            let mut release = [0];
13422            barrier.read_exact(&mut release).unwrap();
13423            assert_eq!(&release, b"X");
13424        }
13425    }
13426
13427    #[test]
13428    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13429        use std::io::{Read, Write};
13430
13431        let supervisor = Supervisor::new_for_test(
13432            Arc::new(Registry::default()),
13433            RestartPolicy::new(0, Duration::ZERO),
13434        );
13435        let mut runtime = supervisor.runtime_config();
13436        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13437        let snapshot = module.inner.snapshot.clone();
13438        runtime.terminal_ring = module.inner.terminal_ring.clone();
13439        let spec = ModuleSpec {
13440            module_id: module.module_id().into(),
13441            program: fake_aft_stub_path(),
13442            protocol: ModuleProtocol::None,
13443            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13444            ..windowed_crash_spec(module.module_id())
13445        };
13446        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13447        let reap = std::thread::spawn(move || {
13448            tokio::runtime::Builder::new_current_thread()
13449                .enable_all()
13450                .build()
13451                .unwrap()
13452                .block_on(async {
13453                    let mut child =
13454                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13455                    health_restart_child(
13456                        &spec,
13457                        &runtime,
13458                        &supervisor.registry,
13459                        &supervisor.process_liveness,
13460                        &snapshot,
13461                        &mut child,
13462                        SupervisorHealthStatus::Failing,
13463                        Some("health probe did not answer"),
13464                        1,
13465                    )
13466                    .await
13467                    .unwrap();
13468                    assert!(child.is_none());
13469                });
13470        });
13471        // The loopback barrier acknowledges entry into terminal recording by
13472        // health_restart_child's drain, then waits for the test's release byte.
13473        // Status and history are observed while that writer is held, not after
13474        // an elapsed-time guess about when the health action has completed.
13475        let mut reached = [0];
13476        barrier.read_exact(&mut reached).unwrap();
13477        assert_eq!(&reached, b"R");
13478        let before_status = module.status().unwrap();
13479        let before_history = module.terminal_history();
13480        barrier.write_all(b"X").unwrap();
13481        reap.join().unwrap();
13482        assert!(before_history.entries.is_empty());
13483        assert_ne!(
13484            before_status.state,
13485            ModuleState::Failed,
13486            "Failed was visible before its terminal record could be written"
13487        );
13488        assert!(
13489            before_status.process_alive,
13490            "process death was visible before its terminal record could be written"
13491        );
13492        let failed = module.status().unwrap();
13493        assert_eq!(failed.state, ModuleState::Failed);
13494        assert!(!failed.process_alive);
13495        let history = module.terminal_history();
13496        assert_eq!(history.entries.len(), 1);
13497        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13498        assert_eq!(
13499            history.entries[0].disposition_detail.as_deref(),
13500            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13501        );
13502    }
13503
13504    #[test]
13505    fn observed_exits_publish_process_death_after_terminal_history() {
13506        use std::io::{Read, Write};
13507        for (kind, enabled, shutdown, wait_error, expected_state) in [
13508            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13509            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13510            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13511            (
13512                ExitKind::DeliberateSeverance,
13513                true,
13514                false,
13515                false,
13516                ModuleState::Restarting,
13517            ),
13518            #[cfg(unix)]
13519            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13520            (ExitKind::Crash, true, false, true, ModuleState::Failed),
13521        ] {
13522            let supervisor = Supervisor::new_for_test(
13523                Arc::new(Registry::default()),
13524                RestartPolicy::new(1, Duration::ZERO),
13525            );
13526            let runtime = supervisor.runtime_config();
13527            #[cfg(unix)]
13528            if shutdown {
13529                runtime.child_roster.close();
13530            }
13531            #[cfg(not(unix))]
13532            let _ = shutdown;
13533            let spec = windowed_crash_spec("exit-publication-order");
13534            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13535                ModuleState::Running,
13536                enabled,
13537            )));
13538            lock_snapshot(&snapshot).unwrap().process_alive = true;
13539            let reaped_snapshot = snapshot.clone();
13540            let history = runtime.terminal_ring.clone();
13541            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13542            let reap = std::thread::spawn(move || {
13543                if wait_error {
13544                    super::fail_after_wait_error(
13545                        &spec.module_id,
13546                        &reaped_snapshot,
13547                        &runtime.terminal_ring,
13548                        &runtime.spawn_events,
13549                        "wait failed".into(),
13550                        &runtime.failure_context(true),
13551                    );
13552                } else {
13553                    tokio::runtime::Builder::new_current_thread()
13554                        .enable_all()
13555                        .build()
13556                        .unwrap()
13557                        .block_on(on_child_exit(
13558                            &spec,
13559                            runtime.restart_policy,
13560                            &supervisor.registry,
13561                            &reaped_snapshot,
13562                            &runtime.terminal_ring,
13563                            &runtime.spawn_events,
13564                            &runtime.child_roster,
13565                            ExitReport {
13566                                kind,
13567                                code: Some(0),
13568                                signal: None,
13569                                at_ms: 1,
13570                            },
13571                            true,
13572                        ));
13573                }
13574            });
13575            let mut reached = [0];
13576            barrier.read_exact(&mut reached).unwrap();
13577            assert_eq!(&reached, b"R");
13578            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13579            let before_history = history.lock().unwrap().snapshot();
13580            barrier.write_all(b"X").unwrap();
13581            reap.join().unwrap();
13582            assert!(
13583                before_alive,
13584                "{kind:?}: death was published before recording"
13585            );
13586            assert!(before_history.entries.is_empty());
13587            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13588            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13589            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13590        }
13591    }
13592
13593    #[tokio::test]
13594    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13595        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13596        let path = dir.join("terminals.jsonl");
13597        std::fs::create_dir(&path).unwrap();
13598        let supervisor = Supervisor::new_for_test(
13599            Arc::new(Registry::default()),
13600            RestartPolicy::new(0, Duration::ZERO),
13601        )
13602        .with_terminal_journal(path, "health-budget-journal-failure".into());
13603        let runtime = supervisor.runtime_config();
13604        let spec = ModuleSpec {
13605            program: fake_aft_stub_path(),
13606            protocol: ModuleProtocol::None,
13607            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13608            ..windowed_crash_spec("health-budget-journal-failure")
13609        };
13610        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13611        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13612        health_restart_child(
13613            &spec,
13614            &runtime,
13615            &supervisor.registry,
13616            &supervisor.process_liveness,
13617            &snapshot,
13618            &mut child,
13619            SupervisorHealthStatus::Failing,
13620            None,
13621            1,
13622        )
13623        .await
13624        .unwrap();
13625        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13626        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13627        let history = runtime
13628            .terminal_ring
13629            .lock()
13630            .unwrap()
13631            .durable_history(&spec.module_id);
13632        assert!(history.journal_write_failures > 0);
13633        assert_eq!(history.entries.len(), 1);
13634        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13635    }
13636
13637    #[test]
13638    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13639        let supervisor = Supervisor::new_for_test(
13640            Arc::new(Registry::default()),
13641            RestartPolicy::new(0, Duration::ZERO),
13642        );
13643        let runtime = supervisor.runtime_config();
13644        let spec = ModuleSpec {
13645            program: fake_aft_stub_path(),
13646            protocol: ModuleProtocol::None,
13647            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13648            ..windowed_crash_spec("health-budget-recording-panic")
13649        };
13650        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13651        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13652            tokio::runtime::Builder::new_current_thread()
13653                .enable_all()
13654                .build()
13655                .unwrap()
13656                .block_on(async {
13657                    let mut child =
13658                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13659                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13660                    health_restart_child(
13661                        &spec,
13662                        &runtime,
13663                        &supervisor.registry,
13664                        &supervisor.process_liveness,
13665                        &snapshot,
13666                        &mut child,
13667                        SupervisorHealthStatus::Failing,
13668                        None,
13669                        1,
13670                    )
13671                    .await
13672                    .unwrap();
13673                });
13674        }));
13675        let panic = result.expect_err("recording must still unwind");
13676        assert_eq!(
13677            panic.downcast_ref::<String>().map(String::as_str),
13678            Some("spawn event sequence exhausted")
13679        );
13680        let state = lock_snapshot(&snapshot).unwrap();
13681        assert_eq!(state.state, ModuleState::Failed);
13682        assert!(!state.process_alive);
13683        assert!(state.last_exit.is_some());
13684    }
13685
13686    #[tokio::test]
13687    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13688        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13689        let path = dir.join("terminals.jsonl");
13690        std::fs::create_dir(&path).unwrap();
13691        let supervisor = Supervisor::new_for_test(
13692            Arc::new(Registry::default()),
13693            RestartPolicy::new(0, Duration::ZERO),
13694        )
13695        .with_terminal_journal(path, "budget-journal-failure".into());
13696        let runtime = supervisor.runtime_config();
13697        let spec = windowed_crash_spec("budget-journal-failure");
13698        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13699        assert!(matches!(
13700            on_child_exit(
13701                &spec,
13702                runtime.restart_policy,
13703                &supervisor.registry,
13704                &snapshot,
13705                &runtime.terminal_ring,
13706                &runtime.spawn_events,
13707                &runtime.child_roster,
13708                crash_exit_report(1),
13709                true,
13710            )
13711            .await,
13712            NextAction::Stop { .. }
13713        ));
13714        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13715        let history = runtime
13716            .terminal_ring
13717            .lock()
13718            .unwrap()
13719            .durable_history(&spec.module_id);
13720        assert!(history.journal_write_failures > 0);
13721        assert_eq!(history.entries.len(), 1);
13722        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13723    }
13724
13725    #[test]
13726    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13727        let supervisor = Supervisor::new_for_test(
13728            Arc::new(Registry::default()),
13729            RestartPolicy::new(0, Duration::ZERO),
13730        );
13731        let runtime = supervisor.runtime_config();
13732        let spec = windowed_crash_spec("budget-recording-panic");
13733        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13734        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13735        // Exhausting the event sequence makes emit_exited panic before the
13736        // terminal write, exercising publication on the recording unwind.
13737        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13738        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13739            tokio::runtime::Builder::new_current_thread()
13740                .enable_all()
13741                .build()
13742                .unwrap()
13743                .block_on(on_child_exit(
13744                    &spec,
13745                    runtime.restart_policy,
13746                    &supervisor.registry,
13747                    &snapshot,
13748                    &runtime.terminal_ring,
13749                    &runtime.spawn_events,
13750                    &runtime.child_roster,
13751                    crash_exit_report(1),
13752                    true,
13753                ))
13754        }));
13755        let panic = result.err().expect("recording must still unwind");
13756        assert_eq!(
13757            panic.downcast_ref::<String>().map(String::as_str),
13758            Some("spawn event sequence exhausted")
13759        );
13760        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13761    }
13762
13763    /// A stop the supervisor itself requests still stops a protocol-none
13764    /// module, even though the child answers the SIGTERM with exit 0.
13765    #[cfg(unix)]
13766    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13767    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13768        for disable in [false, true] {
13769            let label = if disable {
13770                "none-requested-disable"
13771            } else {
13772                "none-requested-stop"
13773            };
13774            let dir = cortexkit_test_support::ScratchDir::new(label);
13775            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13776            let supervisor = Supervisor::new_for_test(
13777                Arc::new(Registry::default()),
13778                RestartPolicy::new(3, Duration::ZERO),
13779            );
13780            let module = supervisor.spawn(spec).unwrap();
13781            wait_for_file(&ready).await;
13782
13783            if disable {
13784                module.set_enabled(false).await.unwrap();
13785            } else {
13786                module.stop().await.unwrap();
13787            }
13788            assert!(
13789                marker.exists(),
13790                "{label}: the child must have left through its SIGTERM handler with exit 0"
13791            );
13792
13793            // Long enough for a zero-backoff respawn to have happened if the
13794            // exit had been treated as a crash.
13795            sleep(Duration::from_millis(500)).await;
13796            let status = module.status().unwrap();
13797            let expected = if disable {
13798                ModuleState::Disabled
13799            } else {
13800                ModuleState::Stopped
13801            };
13802            assert_eq!(status.state, expected, "{label}");
13803            assert_eq!(
13804                status.spawn_generation, 1,
13805                "{label}: respawned after a requested stop"
13806            );
13807            let history = module.terminal_history();
13808            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13809            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13810            assert_ne!(
13811                history.entries[0].disposition,
13812                TerminalDisposition::Restarting,
13813                "{label}"
13814            );
13815        }
13816    }
13817
13818    /// A subc-wire module that exits 0 on its own is still a stop: the
13819    /// protocol-none rule must not reach it.
13820    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13821    async fn subc_wire_clean_exit_is_still_a_stop() {
13822        let supervisor = Supervisor::new_for_test(
13823            Arc::new(Registry::default()),
13824            RestartPolicy::new(3, Duration::ZERO),
13825        );
13826        let module = supervisor
13827            .spawn(ModuleSpec {
13828                module_id: "wire-clean-exit".to_string(),
13829                program: fake_aft_stub_path(),
13830                args: Vec::new(),
13831                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13832                reserved: false,
13833                reserved_prefixes: Vec::new(),
13834                protocol: ModuleProtocol::Subc,
13835                overlap: Default::default(),
13836                carry_containment: false,
13837            })
13838            .unwrap();
13839
13840        let deadline = Instant::now() + Duration::from_secs(10);
13841        while module.terminal_history().entries.is_empty() {
13842            assert!(Instant::now() < deadline, "module never exited");
13843            sleep(Duration::from_millis(10)).await;
13844        }
13845        // Long enough for a zero-backoff respawn to have happened.
13846        sleep(Duration::from_millis(500)).await;
13847        let status = module.status().unwrap();
13848        assert_eq!(status.state, ModuleState::Stopped);
13849        assert_eq!(status.spawn_generation, 1);
13850        let history = module.terminal_history();
13851        assert_eq!(history.entries.len(), 1, "{history:?}");
13852        assert_eq!(history.entries[0].exit_code, Some(0));
13853        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13854    }
13855
13856    #[cfg(unix)]
13857    #[tokio::test]
13858    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13859        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13860        let record = dir.join("live-children.json");
13861        let supervisor = Supervisor::new_for_test(
13862            Arc::new(Registry::default()),
13863            RestartPolicy::new(0, Duration::ZERO),
13864        );
13865        let mut runtime = supervisor.runtime_config();
13866        runtime.child_roster.record_to(record.clone());
13867        let gate = Arc::new(super::ReloadExitRecordGate::default());
13868        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13869        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13870        let spec = ModuleSpec {
13871            module_id: "reload-exit-roster".into(),
13872            program: fake_aft_stub_path(),
13873            args: Vec::new(),
13874            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13875            reserved: false,
13876            reserved_prefixes: Vec::new(),
13877            protocol: ModuleProtocol::Subc,
13878            overlap: Default::default(),
13879            carry_containment: false,
13880        };
13881        let mut child = None;
13882        let reload = super::finish_reload_child(
13883            &spec,
13884            &runtime,
13885            &supervisor.registry,
13886            &supervisor.process_liveness,
13887            &snapshot,
13888            &mut child,
13889        );
13890        tokio::pin!(reload);
13891        tokio::select! {
13892            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13893            _ = gate.reached.notified() => {}
13894        }
13895        assert!(runtime
13896            .terminal_ring
13897            .lock()
13898            .unwrap()
13899            .snapshot()
13900            .entries
13901            .is_empty());
13902        assert_eq!(
13903            crate::live_children::read_record(&record).unwrap().len(),
13904            1,
13905            "shutdown must still wait for the reaped child until its terminal record exists"
13906        );
13907        runtime.child_roster.close();
13908        gate.resume.notify_one();
13909        assert!(reload.await.is_err());
13910        assert!(crate::live_children::read_record(&record)
13911            .unwrap()
13912            .is_empty());
13913        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13914        assert_eq!(history.entries.len(), 1);
13915        assert_eq!(
13916            history.entries[0].disposition,
13917            TerminalDisposition::DaemonShutdown
13918        );
13919    }
13920
13921    /// Each restart-producing arm has its own state transition. Keeping their
13922    /// lifetime count assertions adjacent prevents a later new arm from silently
13923    /// spending budget without recording the historical restart.
13924    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13925    async fn every_restart_increment_path_advances_lifetime_count() {
13926        let supervisor = Supervisor::new_for_test(
13927            Arc::new(Registry::default()),
13928            RestartPolicy::new(1, Duration::ZERO),
13929        );
13930        let runtime = supervisor.runtime_config();
13931        let spec = ModuleSpec {
13932            module_id: "lifetime-increment-path".to_string(),
13933            program: PathBuf::from("/unused/lifetime-increment-path"),
13934            args: Vec::new(),
13935            env: Vec::new(),
13936            reserved: false,
13937            reserved_prefixes: Vec::new(),
13938            protocol: ModuleProtocol::Subc,
13939            overlap: Default::default(),
13940            carry_containment: false,
13941        };
13942
13943        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13944        assert!(matches!(
13945            on_child_exit(
13946                &spec,
13947                runtime.restart_policy,
13948                &supervisor.registry,
13949                &crash_snapshot,
13950                &runtime.terminal_ring,
13951                &runtime.spawn_events,
13952                &runtime.child_roster,
13953                ExitReport {
13954                    kind: ExitKind::Crash,
13955                    code: Some(1),
13956                    signal: None,
13957                    at_ms: 1,
13958                },
13959                true,
13960            )
13961            .await,
13962            NextAction::Restart { schedule: _ }
13963        ));
13964        let (crash_restarts, crash_lifetime) = {
13965            let state = lock_snapshot(&crash_snapshot).unwrap();
13966            (state.crash_restarts.len(), state.lifetime_restarts)
13967        };
13968        assert_eq!(crash_restarts, 1);
13969        assert_eq!(crash_lifetime, 1);
13970
13971        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13972        let mut health_child = None;
13973        assert!(matches!(
13974            health_restart_child(
13975                &spec,
13976                &runtime,
13977                &supervisor.registry,
13978                &supervisor.process_liveness,
13979                &health_snapshot,
13980                &mut health_child,
13981                SupervisorHealthStatus::Failing,
13982                None,
13983                2,
13984            )
13985            .await,
13986            Ok(())
13987        ));
13988        assert!(health_child.is_none());
13989        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13990        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13991        let (health_restarts, health_lifetime) = {
13992            let state = lock_snapshot(&health_snapshot).unwrap();
13993            (state.crash_restarts.len(), state.lifetime_restarts)
13994        };
13995        assert_eq!(health_restarts, 1);
13996        assert_eq!(health_lifetime, 1);
13997
13998        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13999        let mut reload_child = None;
14000        assert!(matches!(
14001            handle_reload_spawn_failure(
14002                &spec,
14003                &runtime,
14004                &supervisor.process_liveness,
14005                &reload_snapshot,
14006                &mut reload_child,
14007                "forced reload spawn failure".to_string(),
14008            )
14009            .await,
14010            Err(SuperviseError::ReloadFailed { .. })
14011        ));
14012        let (reload_restarts, reload_lifetime) = {
14013            let state = lock_snapshot(&reload_snapshot).unwrap();
14014            (state.crash_restarts.len(), state.lifetime_restarts)
14015        };
14016        assert_eq!(reload_restarts, 1);
14017        assert_eq!(reload_lifetime, 1);
14018    }
14019
14020    #[tokio::test]
14021    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14022        let supervisor = Supervisor::new_for_test(
14023            Arc::new(Registry::default()),
14024            RestartPolicy::new(3, Duration::ZERO),
14025        );
14026        let runtime = supervisor.runtime_config();
14027        let spec = ModuleSpec {
14028            module_id: "deliberately-severed".to_string(),
14029            program: PathBuf::from("/unused/deliberately-severed"),
14030            args: Vec::new(),
14031            env: Vec::new(),
14032            reserved: false,
14033            reserved_prefixes: Vec::new(),
14034            protocol: ModuleProtocol::Subc,
14035            overlap: Default::default(),
14036            carry_containment: false,
14037        };
14038        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14039        let process = ProcessIdentity {
14040            pid: 41,
14041            start_time: 101,
14042        };
14043        record_deliberate_severance(&snapshot, process).unwrap();
14044        let exit_report = apply_deliberate_severance_marker(
14045            &snapshot,
14046            Some(process),
14047            ExitReport {
14048                kind: ExitKind::Crash,
14049                code: Some(1),
14050                signal: None,
14051                at_ms: 1,
14052            },
14053        );
14054        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14055
14056        assert!(matches!(
14057            on_child_exit(
14058                &spec,
14059                runtime.restart_policy,
14060                &supervisor.registry,
14061                &snapshot,
14062                &runtime.terminal_ring,
14063                &runtime.spawn_events,
14064                &runtime.child_roster,
14065                exit_report,
14066                true,
14067            )
14068            .await,
14069            NextAction::Restart { schedule: _ }
14070        ));
14071        let state = lock_snapshot(&snapshot).unwrap();
14072        assert_eq!(state.lifetime_restarts, 1);
14073        assert_eq!(state.crash_restarts.len(), 0);
14074    }
14075
14076    #[tokio::test]
14077    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14078        let supervisor = Supervisor::new_for_test(
14079            Arc::new(Registry::default()),
14080            RestartPolicy::new(3, Duration::ZERO),
14081        );
14082        let runtime = supervisor.runtime_config();
14083        let spec = ModuleSpec {
14084            module_id: "genuine-crash".to_string(),
14085            program: PathBuf::from("/unused/genuine-crash"),
14086            args: Vec::new(),
14087            env: Vec::new(),
14088            reserved: false,
14089            reserved_prefixes: Vec::new(),
14090            protocol: ModuleProtocol::Subc,
14091            overlap: Default::default(),
14092            carry_containment: false,
14093        };
14094        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14095
14096        assert!(matches!(
14097            on_child_exit(
14098                &spec,
14099                runtime.restart_policy,
14100                &supervisor.registry,
14101                &snapshot,
14102                &runtime.terminal_ring,
14103                &runtime.spawn_events,
14104                &runtime.child_roster,
14105                ExitReport {
14106                    kind: ExitKind::Crash,
14107                    code: Some(1),
14108                    signal: None,
14109                    at_ms: 1,
14110                },
14111                true,
14112            )
14113            .await,
14114            NextAction::Restart { schedule: _ }
14115        ));
14116        let state = lock_snapshot(&snapshot).unwrap();
14117        assert_eq!(state.lifetime_restarts, 1);
14118        assert_eq!(state.crash_restarts.len(), 1);
14119    }
14120
14121    fn crash_exit_report(at_ms: u64) -> ExitReport {
14122        ExitReport {
14123            kind: ExitKind::Crash,
14124            code: Some(1),
14125            signal: None,
14126            at_ms,
14127        }
14128    }
14129
14130    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14131        ModuleSpec {
14132            module_id: module_id.to_string(),
14133            program: PathBuf::from("/unused").join(module_id),
14134            args: Vec::new(),
14135            env: Vec::new(),
14136            reserved: false,
14137            reserved_prefixes: Vec::new(),
14138            protocol: ModuleProtocol::Subc,
14139            overlap: Default::default(),
14140            carry_containment: false,
14141        }
14142    }
14143
14144    /// A real crash loop still stops. Three crashes with nothing aging out spend
14145    /// a budget of two and the third respawn is refused, and both surfaces an
14146    /// operator has -- the log line and the retained terminal record -- name the
14147    /// window rather than only the cap, because `max_restarts=2` alone is what
14148    /// this budget used to mean.
14149    #[tokio::test]
14150    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14151        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14152        let supervisor = Supervisor::new_for_test(
14153            Arc::new(Registry::default()),
14154            RestartPolicy::new(2, Duration::ZERO),
14155        );
14156        let runtime = supervisor.runtime_config();
14157        let spec = windowed_crash_spec("crash-loop-in-window");
14158        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14159
14160        for attempt in 1..=2 {
14161            assert!(
14162                matches!(
14163                    on_child_exit(
14164                        &spec,
14165                        runtime.restart_policy,
14166                        &supervisor.registry,
14167                        &snapshot,
14168                        &runtime.terminal_ring,
14169                        &runtime.spawn_events,
14170                        &runtime.child_roster,
14171                        crash_exit_report(attempt),
14172                        true,
14173                    )
14174                    .await,
14175                    NextAction::Restart { schedule: _ }
14176                ),
14177                "crash {attempt} is inside the budget and must respawn"
14178            );
14179        }
14180
14181        assert!(matches!(
14182            on_child_exit(
14183                &spec,
14184                runtime.restart_policy,
14185                &supervisor.registry,
14186                &snapshot,
14187                &runtime.terminal_ring,
14188                &runtime.spawn_events,
14189                &runtime.child_roster,
14190                crash_exit_report(3),
14191                true,
14192            )
14193            .await,
14194            NextAction::Stop { .. }
14195        ));
14196
14197        {
14198            let state = lock_snapshot(&snapshot).unwrap();
14199            assert_eq!(state.state, ModuleState::Failed);
14200            assert_eq!(state.crash_restarts.len(), 2);
14201            assert_eq!(state.lifetime_restarts, 2);
14202        }
14203
14204        let history = runtime
14205            .terminal_ring
14206            .lock()
14207            .expect("terminal ring is not poisoned")
14208            .snapshot();
14209        let last = history
14210            .entries
14211            .last()
14212            .expect("the refused crash is retained");
14213        assert_eq!(last.disposition, TerminalDisposition::Failed);
14214        assert_eq!(
14215            last.disposition_detail.as_deref(),
14216            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14217        );
14218
14219        let captured = crate::router::test_log::captured_logs(&logs);
14220        assert!(
14221            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14222            "the stop must be logged with its window: {captured}"
14223        );
14224    }
14225
14226    /// The rate, stated as a test: three crashes where the first has aged past
14227    /// the window are two crashes as far as the budget is concerned, so the
14228    /// third respawn is allowed and the ring holds only the two recent ones.
14229    ///
14230    /// This is the case a lifetime counter got wrong -- and the case the daemon
14231    /// now hits routinely, since a module exits non-zero every time its
14232    /// connection to the daemon drops.
14233    #[tokio::test]
14234    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14235        let supervisor = Supervisor::new_for_test(
14236            Arc::new(Registry::default()),
14237            RestartPolicy::new(2, Duration::ZERO),
14238        );
14239        let runtime = supervisor.runtime_config();
14240        let spec = windowed_crash_spec("crash-across-windows");
14241        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14242
14243        for attempt in 1..=2 {
14244            assert!(matches!(
14245                on_child_exit(
14246                    &spec,
14247                    runtime.restart_policy,
14248                    &supervisor.registry,
14249                    &snapshot,
14250                    &runtime.terminal_ring,
14251                    &runtime.spawn_events,
14252                    &runtime.child_roster,
14253                    crash_exit_report(attempt),
14254                    true,
14255                )
14256                .await,
14257                NextAction::Restart { schedule: _ }
14258            ));
14259        }
14260
14261        // The oldest crash moves out of the window; nothing else about the
14262        // module changes.
14263        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14264            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14265        })
14266        .unwrap();
14267
14268        assert!(
14269            matches!(
14270                on_child_exit(
14271                    &spec,
14272                    runtime.restart_policy,
14273                    &supervisor.registry,
14274                    &snapshot,
14275                    &runtime.terminal_ring,
14276                    &runtime.spawn_events,
14277                    &runtime.child_roster,
14278                    crash_exit_report(3),
14279                    true,
14280                )
14281                .await,
14282                NextAction::Restart { schedule: _ }
14283            ),
14284            "a crash older than the window must not hold a budget slot"
14285        );
14286
14287        let state = lock_snapshot(&snapshot).unwrap();
14288        assert_eq!(state.state, ModuleState::Restarting);
14289        assert_eq!(
14290            state.crash_restarts.len(),
14291            2,
14292            "the aged instant is dropped and the new one takes its place"
14293        );
14294        assert_eq!(
14295            state.lifetime_restarts, 3,
14296            "the ledger counts every restart, including the ones the window forgot"
14297        );
14298    }
14299
14300    /// An operator restart hands the budget back whole, and the ledger keeps
14301    /// counting. Those are different questions -- "how close is this module to
14302    /// being stopped" and "how many times has it been replaced" -- and the
14303    /// operator action answers only the first.
14304    #[tokio::test]
14305    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14306        let supervisor = Supervisor::new_for_test(
14307            Arc::new(Registry::default()),
14308            RestartPolicy::new(2, Duration::ZERO),
14309        );
14310        let runtime = supervisor.runtime_config();
14311        let spec = windowed_crash_spec("operator-cleared-budget");
14312        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14313
14314        for attempt in 1..=2 {
14315            assert!(matches!(
14316                on_child_exit(
14317                    &spec,
14318                    runtime.restart_policy,
14319                    &supervisor.registry,
14320                    &snapshot,
14321                    &runtime.terminal_ring,
14322                    &runtime.spawn_events,
14323                    &runtime.child_roster,
14324                    crash_exit_report(attempt),
14325                    true,
14326                )
14327                .await,
14328                NextAction::Restart { schedule: _ }
14329            ));
14330        }
14331
14332        reset_restart_count(&snapshot, &spec.module_id).unwrap();
14333        {
14334            let state = lock_snapshot(&snapshot).unwrap();
14335            assert!(
14336                state.crash_restarts.is_empty(),
14337                "an operator restart returns the full budget"
14338            );
14339            assert_eq!(
14340                state.lifetime_restarts, 2,
14341                "clearing the budget must not unmake the crashes"
14342            );
14343        }
14344
14345        assert!(
14346            matches!(
14347                on_child_exit(
14348                    &spec,
14349                    runtime.restart_policy,
14350                    &supervisor.registry,
14351                    &snapshot,
14352                    &runtime.terminal_ring,
14353                    &runtime.spawn_events,
14354                    &runtime.child_roster,
14355                    crash_exit_report(3),
14356                    true,
14357                )
14358                .await,
14359                NextAction::Restart { schedule: _ }
14360            ),
14361            "the cleared budget must be spendable again"
14362        );
14363        let state = lock_snapshot(&snapshot).unwrap();
14364        assert_eq!(state.crash_restarts.len(), 1);
14365        assert_eq!(state.lifetime_restarts, 3);
14366    }
14367
14368    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14369    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14370        let severed = ProcessIdentity {
14371            pid: 41,
14372            start_time: 101,
14373        };
14374        let successor = ProcessIdentity {
14375            pid: 41,
14376            start_time: 202,
14377        };
14378        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14379        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14380            state.pid = Some(successor.pid);
14381            state.process_start_time = Some(successor.start_time);
14382        })
14383        .unwrap();
14384        assert!(!module.record_deliberate_severance(severed).unwrap());
14385
14386        let exit_report = apply_deliberate_severance_marker(
14387            &module.inner.snapshot,
14388            Some(successor),
14389            ExitReport {
14390                kind: ExitKind::Crash,
14391                code: Some(1),
14392                signal: None,
14393                at_ms: 1,
14394            },
14395        );
14396
14397        assert_eq!(exit_report.kind, ExitKind::Crash);
14398    }
14399
14400    #[tokio::test]
14401    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14402        let registry = Registry::default();
14403        let supervisor = Supervisor::new_for_test(
14404            Arc::new(Registry::default()),
14405            RestartPolicy::new(3, Duration::ZERO),
14406        );
14407        let runtime = supervisor.runtime_config();
14408        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14409        let spec = ModuleSpec {
14410            module_id: "drain-deliberate-severance".to_string(),
14411            program: fake_aft_stub_path(),
14412            args: Vec::new(),
14413            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14414            reserved: false,
14415            reserved_prefixes: Vec::new(),
14416            protocol: ModuleProtocol::Subc,
14417            overlap: Default::default(),
14418            carry_containment: false,
14419        };
14420        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14421        let process = ProcessIdentity {
14422            pid: 41,
14423            start_time: 101,
14424        };
14425        child.process_identity = Some(process);
14426        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14427            state.pid = Some(process.pid);
14428            state.process_start_time = Some(process.start_time);
14429        })
14430        .unwrap();
14431        record_deliberate_severance(&snapshot, process).unwrap();
14432
14433        drain_child_to_state(
14434            &spec.module_id,
14435            spec.protocol,
14436            // The child exits on its own; no signal may change the exit this
14437            // test classifies.
14438            StopNotice::SentOverConnection,
14439            &registry,
14440            None,
14441            &snapshot,
14442            &runtime.terminal_ring,
14443            &runtime.spawn_events,
14444            child,
14445            Duration::from_secs(1),
14446            ModuleState::Stopped,
14447            Some(false),
14448            &super::FailureContext::operator(),
14449        )
14450        .await
14451        .unwrap();
14452
14453        let state = lock_snapshot(&snapshot).unwrap();
14454        assert_eq!(
14455            state.last_exit.as_ref().map(|exit| exit.kind),
14456            Some(ExitKind::DeliberateSeverance)
14457        );
14458        assert_eq!(state.lifetime_restarts, 1);
14459        assert_eq!(state.crash_restarts.len(), 0);
14460        drop(state);
14461        let history = runtime.terminal_ring.lock().unwrap().snapshot();
14462        assert_eq!(
14463            history.entries[0].exit_kind,
14464            subc_control::TerminalExitKind::DeliberateSeverance
14465        );
14466    }
14467
14468    #[tokio::test]
14469    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14470        let registry = Registry::default();
14471        let supervisor = Supervisor::new_for_test(
14472            Arc::new(Registry::default()),
14473            RestartPolicy::new(3, Duration::ZERO),
14474        );
14475        let runtime = supervisor.runtime_config();
14476        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14477        let spec = ModuleSpec {
14478            module_id: "ordinary-drain".to_string(),
14479            program: fake_aft_stub_path(),
14480            args: Vec::new(),
14481            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14482            reserved: false,
14483            reserved_prefixes: Vec::new(),
14484            protocol: ModuleProtocol::Subc,
14485            overlap: Default::default(),
14486            carry_containment: false,
14487        };
14488        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14489
14490        drain_child_to_state(
14491            &spec.module_id,
14492            spec.protocol,
14493            // The child exits on its own; no signal may change the exit this
14494            // test classifies.
14495            StopNotice::SentOverConnection,
14496            &registry,
14497            None,
14498            &snapshot,
14499            &runtime.terminal_ring,
14500            &runtime.spawn_events,
14501            child,
14502            Duration::from_secs(1),
14503            ModuleState::Stopped,
14504            Some(false),
14505            &super::FailureContext::operator(),
14506        )
14507        .await
14508        .unwrap();
14509
14510        let state = lock_snapshot(&snapshot).unwrap();
14511        assert_eq!(
14512            state.last_exit.as_ref().map(|exit| exit.kind),
14513            Some(ExitKind::Crash)
14514        );
14515        assert_eq!(state.lifetime_restarts, 0);
14516        assert_eq!(state.crash_restarts.len(), 0);
14517    }
14518
14519    #[test]
14520    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14521        // The server's generic fatal-routing branch only knows that the
14522        // connection failed; it does not know that the daemon deliberately
14523        // initiated a process-killing severance. Keep this seam explicit so a
14524        // future connection error path cannot silently reintroduce the stale
14525        // exemption that mislabels a later genuine crash.
14526        assert!(!include_str!("server.rs")
14527            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14528    }
14529
14530    /// The `route.closed` `drained` value must be the quiescence wait's own
14531    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
14532    /// measurement at all and `false` is the one honest constant. This is the exact
14533    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
14534    /// on every return path, including the one that used to return early via `?`
14535    /// with `route.closing` already sent and no `route.closed` ever following.
14536    #[test]
14537    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14538        assert!(drained_after_quiescence_wait(&Ok(true)));
14539        assert!(!drained_after_quiescence_wait(&Ok(false)));
14540        assert!(!drained_after_quiescence_wait(&Err(
14541            SuperviseError::StatePoisoned { module_id: None }
14542        )));
14543    }
14544
14545    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
14546    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
14547    /// already reaped out-of-band) still leaves a terminal record rather than none
14548    /// at all. Triggering the real `wait()` I/O error from an integration test would
14549    /// need a genuine already-reaped-child race, which is OS-specific and not
14550    /// something this suite attempts elsewhere; this test instead verifies the
14551    /// record produced for that arm end-to-end through the real `TerminalRing`, and
14552    /// the call site itself is verified by inspection to sit in that exact arm.
14553    #[test]
14554    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14555        let ring = Arc::new(Mutex::new(TerminalRing::new(
14556            TerminalRingConfig::default(),
14557            0,
14558        )));
14559        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14560
14561        let snapshot = ring.lock().unwrap().snapshot();
14562        assert_eq!(snapshot.entries.len(), 1);
14563        let entry = &snapshot.entries[0];
14564        assert_eq!(entry.exit_code, None);
14565        assert_eq!(entry.exit_signal, None);
14566        assert_eq!(entry.disposition, TerminalDisposition::Failed);
14567    }
14568
14569    #[test]
14570    fn wait_error_exit_path_preserves_spawn_event_density() {
14571        let feed = super::SpawnEventFeed::default();
14572        feed.configure_incarnation("wait-error-density".to_string());
14573        feed.emit_spawned("wait-error", 41, 1);
14574        let ring = Arc::new(Mutex::new(TerminalRing::new(
14575            TerminalRingConfig::default(),
14576            0,
14577        )));
14578
14579        record_wait_error_terminal("wait-error", &ring, &feed);
14580        feed.emit_spawned("after-wait-error", 42, 2);
14581
14582        let state = feed.0.lock().unwrap();
14583        let sequences = state
14584            .events
14585            .iter()
14586            .map(|event| event.cursor.seq)
14587            .collect::<Vec<_>>();
14588        assert_eq!(sequences, vec![1, 2, 3]);
14589        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14590        assert_eq!(state.events[1].exit_code, None);
14591        assert_eq!(state.events[1].exit_signal, None);
14592    }
14593
14594    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
14595    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
14596    /// not a clean exit it never actually observed.
14597    #[test]
14598    fn wait_error_exit_report_is_classified_as_a_crash() {
14599        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14600    }
14601}
14602
14603#[cfg(all(test, unix))]
14604mod restart_ordering_tests {
14605    use super::*;
14606    use std::io::Write;
14607    use tokio::io::AsyncReadExt;
14608
14609    #[test]
14610    #[ignore = "child process fixture"]
14611    fn serving_child() {
14612        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14613            return;
14614        };
14615        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14616        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14617        for stream in listener.incoming() {
14618            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14619        }
14620    }
14621
14622    async fn restart_before_publication(requests: usize) {
14623        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14624        let address_file = home.path().join("address");
14625        let spec = ModuleSpec {
14626            module_id: "restart-serving-child".into(),
14627            program: std::env::current_exe().unwrap(),
14628            args: vec![
14629                "--exact".into(),
14630                "supervise::restart_ordering_tests::serving_child".into(),
14631                "--ignored".into(),
14632                "--nocapture".into(),
14633            ],
14634            env: vec![(
14635                "RESTART_TEST_ADDRESS_FILE".into(),
14636                address_file.display().to_string(),
14637            )],
14638            reserved: false,
14639            reserved_prefixes: vec![],
14640            protocol: ModuleProtocol::None,
14641            overlap: Default::default(),
14642            carry_containment: false,
14643        };
14644        let mut supervisor =
14645            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14646        let (spawned_tx, spawned_rx) = oneshot::channel();
14647        let spawned_tx = Mutex::new(Some(spawned_tx));
14648        let (release_tx, release_rx) = std::sync::mpsc::channel();
14649        let release_rx = Mutex::new(release_rx);
14650        // Hold publication, not the child: it must answer a real request before
14651        // the restart enters the queue, even though status still has generation 0.
14652        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14653            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14654            release_rx
14655                .lock()
14656                .unwrap()
14657                .recv_timeout(Duration::from_secs(20))
14658                .unwrap();
14659        })));
14660        let registered = supervisor
14661            .register_configured_with_health(
14662                spec.clone(),
14663                true,
14664                HealthConfig::default(),
14665                None,
14666                RestartPolicy::new(3, Duration::ZERO),
14667                DEFAULT_REGISTER_TIMEOUT,
14668            )
14669            .unwrap();
14670        let module = registered.module.clone();
14671        let startup = tokio::task::spawn_blocking(move || {
14672            supervisor.start_registered_configured(registered, None, true)
14673        });
14674        spawned_rx.await.unwrap();
14675        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14676            let address = loop {
14677                if let Ok(address) = std::fs::read_to_string(&address_file) {
14678                    if !address.is_empty() {
14679                        break address;
14680                    }
14681                }
14682                tokio::time::sleep(Duration::from_millis(1)).await;
14683            };
14684            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14685            let mut reply = String::new();
14686            stream.read_to_string(&mut reply).await.unwrap();
14687            reply.trim().parse::<u32>().unwrap()
14688        })
14689        .await
14690        .expect("child must serve before generation publication");
14691        assert_eq!(
14692            lock_snapshot(&module.inner.snapshot)
14693                .unwrap()
14694                .spawn_generation,
14695            0
14696        );
14697        let mut replies = Vec::new();
14698        for _ in 0..requests {
14699            let mut restart = Box::pin(module.restart(Some(0)));
14700            // Poll the public API through queue admission, leaving only its ACK
14701            // pending. The monitor cannot consume the command until released.
14702            std::future::poll_fn(|cx| {
14703                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14704                std::task::Poll::Ready(())
14705            })
14706            .await;
14707            replies.push(restart);
14708        }
14709        release_tx.send(()).unwrap();
14710        startup.await.unwrap().unwrap();
14711        for reply in replies {
14712            reply.await.unwrap();
14713        }
14714        // Restart schedules its replacement after completing the handler. Wait
14715        // for that separate actor turn before placing the command barrier.
14716        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14717            loop {
14718                if lock_snapshot(&module.inner.snapshot)
14719                    .unwrap()
14720                    .spawn_generation
14721                    > 1
14722                {
14723                    break;
14724                }
14725                tokio::time::sleep(Duration::from_millis(1)).await;
14726            }
14727        })
14728        .await;
14729        // Commands are handled serially. The update-spec reply proves every
14730        // restart handler has finished before we inspect the snapshot; restart's
14731        // own ACK only acknowledges initiation, not replacement completion.
14732        module.update_spec_for_test(spec).await.unwrap();
14733        let (generation, pid) = {
14734            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14735            (state.spawn_generation, state.pid)
14736        };
14737        module.stop().await.unwrap();
14738        replacement.expect("restart must publish a replacement for the already-serving child");
14739        assert_ne!(
14740            pid,
14741            Some(old_pid),
14742            "a serving child predating the restart must be replaced"
14743        );
14744        assert_eq!(generation, 2, "queued restarts must share one replacement");
14745    }
14746
14747    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14748    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14749        restart_before_publication(1).await;
14750    }
14751
14752    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14753    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14754        restart_before_publication(2).await;
14755    }
14756
14757    #[tokio::test]
14758    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14759        let supervisor =
14760            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14761        let registered = supervisor
14762            .register_configured_with_health(
14763                ModuleSpec {
14764                    module_id: "queued-before-replacement".into(),
14765                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14766                    // macOS a module whose running image is not the configured
14767                    // program is refused once its exec is confirmed (see
14768                    // `record_write_stall_tests::sleeping_spec`).
14769                    program: "/bin/sleep".into(),
14770                    args: vec!["600".into()],
14771                    env: vec![],
14772                    reserved: false,
14773                    reserved_prefixes: vec![],
14774                    protocol: ModuleProtocol::None,
14775                    overlap: Default::default(),
14776                    carry_containment: false,
14777                },
14778                true,
14779                HealthConfig::default(),
14780                None,
14781                RestartPolicy::default(),
14782                DEFAULT_REGISTER_TIMEOUT,
14783            )
14784            .unwrap();
14785        let RegisteredConfiguredModule {
14786            mut spec,
14787            mut runtime,
14788            module,
14789            mut commands,
14790        } = registered;
14791        let snapshot = &module.inner.snapshot;
14792        let mut restart = Box::pin(module.restart(Some(0)));
14793        std::future::poll_fn(|cx| {
14794            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14795            std::task::Poll::Ready(())
14796        })
14797        .await;
14798        // Drive the actor explicitly so a replacement is definitely spawned
14799        // after receipt, but before this queued command is dequeued.
14800        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14801        let replacement_pid = child.as_ref().unwrap().id();
14802        let command = commands.recv().await.unwrap();
14803        assert!(
14804            handle_supervisor_command(
14805                command,
14806                &mut spec,
14807                &mut runtime,
14808                &supervisor.registry,
14809                &supervisor.process_liveness,
14810                snapshot,
14811                &mut child,
14812                &mut commands,
14813                &mut VecDeque::new(),
14814            )
14815            .await
14816        );
14817        restart.await.unwrap();
14818        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14819        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14820        if let Some(mut child) = child {
14821            child.start_kill().unwrap();
14822            child.wait().await.unwrap();
14823        }
14824        assert_eq!(
14825            kept_pid, replacement_pid,
14826            "a newer replacement must survive"
14827        );
14828        assert!(
14829            !pending,
14830            "a satisfied restart must not schedule another spawn"
14831        );
14832    }
14833}
14834
14835#[cfg(all(test, unix))]
14836mod health_event_tests {
14837    use super::*;
14838    use cortexkit_test_support::ScratchDir;
14839    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14840
14841    struct Actor {
14842        module: SupervisedModule,
14843        registry: Arc<Registry>,
14844        forwarding: Arc<ForwardingTable>,
14845        spec: ModuleSpec,
14846        health: HealthConfig,
14847        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14848        roster: ChildRoster,
14849        _home: ScratchDir,
14850    }
14851
14852    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14853        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14854        // for an observed condition, rather than letting paused time auto-advance
14855        // deadlines or assuming a fixed number of yields finishes the work.
14856        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14857        let tick = Instant::now();
14858        loop {
14859            if let Some(value) = observe() {
14860                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14861                return value;
14862            }
14863            assert!(
14864                std::time::Instant::now() < deadline,
14865                "timed out waiting for {reason}"
14866            );
14867            tokio::task::yield_now().await;
14868        }
14869    }
14870
14871    impl Actor {
14872        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14873            Self::start_with_policy(
14874                protocol,
14875                cadence,
14876                DEFAULT_REGISTER_TIMEOUT,
14877                RestartPolicy::new(3, Duration::ZERO),
14878            )
14879            .await
14880        }
14881
14882        async fn start_with_policy(
14883            protocol: ModuleProtocol,
14884            cadence: Duration,
14885            register_timeout: Duration,
14886            policy: RestartPolicy,
14887        ) -> Self {
14888            let home = ScratchDir::new("health-events");
14889            let registry = Arc::new(Registry::default());
14890            let forwarding = Arc::new(ForwardingTable::default());
14891            let health = HealthConfig {
14892                cadence,
14893                deadline: Duration::from_secs(3600),
14894                ..HealthConfig::default()
14895            };
14896            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14897                .with_forwarding(forwarding.clone())
14898                .with_health_config(health.clone());
14899            let spec = ModuleSpec {
14900                module_id: "event-health-child".into(),
14901                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14902                // the privacy trampoline refuses a module whose running image
14903                // is not the configured program, and `start` below waits for
14904                // the process to be confirmed (`reported_pid`). The shell form
14905                // passed only while the image check ran before the shell had
14906                // replaced itself with `sleep` (see
14907                // `record_write_stall_tests::sleeping_spec`).
14908                program: PathBuf::from("/bin/sleep"),
14909                // The process stays alive but has no real bus peer. Registrations
14910                // below exercise the same registry writes as accepted HELLOs.
14911                args: vec!["600".into()],
14912                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14913                    .into_iter()
14914                    .map(|key| (key.into(), home.path().display().to_string()))
14915                    .collect(),
14916                reserved: false,
14917                reserved_prefixes: vec![],
14918                protocol,
14919                overlap: Default::default(),
14920                carry_containment: false,
14921            };
14922            let module = supervisor
14923                .supervise_configured_requested(
14924                    spec.clone(),
14925                    true,
14926                    health.clone(),
14927                    Some(0),
14928                    policy,
14929                    register_timeout,
14930                    None,
14931                )
14932                .unwrap();
14933            let (_, rx) = mpsc::channel(8);
14934            let actor = Self {
14935                module,
14936                registry,
14937                forwarding,
14938                spec,
14939                health,
14940                rx,
14941                roster: supervisor.child_roster.clone(),
14942                _home: home,
14943            };
14944            actor
14945                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14946                .await;
14947            actor
14948        }
14949
14950        async fn expire_registration_wait(&self, remaining: Duration) {
14951            tokio::time::advance(remaining).await;
14952            wait_for("registration timeout TERM grace", || {
14953                lock_snapshot(&self.module.inner.snapshot)
14954                    .unwrap()
14955                    .actor_select
14956                    .is_none()
14957                    .then_some(())
14958            })
14959            .await;
14960            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14961        }
14962
14963        fn turns(&self) -> u64 {
14964            lock_snapshot(&self.module.inner.snapshot)
14965                .unwrap()
14966                .actor_turns
14967        }
14968
14969        async fn wait_for_select(
14970            &self,
14971            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14972        ) -> ActorSelectCheckpoint {
14973            wait_for("actor select checkpoint", || {
14974                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14975                state.actor_select.filter(|checkpoint| {
14976                    checkpoint.generation == state.spawn_generation
14977                        && state.state == ModuleState::Running
14978                        && state.process_alive
14979                        && state.reported_pid().is_some()
14980                        && expected(checkpoint)
14981                })
14982            })
14983            .await
14984        }
14985
14986        fn hello(&mut self, connection: u64, health: bool) {
14987            let manifest =
14988                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14989                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14990                    .build();
14991            let connection = ConnectionId::new(connection);
14992            let (tx, rx) = mpsc::channel(8);
14993            self.rx = rx;
14994            self.forwarding
14995                .register_module_connection(
14996                    connection,
14997                    self.spec.module_id.clone(),
14998                    subc_protocol::PROTOCOL_VERSION,
14999                    Concurrency::ModuleManaged,
15000                    FrameSink::new(tx),
15001                )
15002                .unwrap();
15003            self.registry
15004                .register_with_control_ops(
15005                    manifest,
15006                    subc_protocol::PROTOCOL_VERSION,
15007                    connection,
15008                    if health {
15009                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15010                    } else {
15011                        vec![]
15012                    },
15013                )
15014                .unwrap();
15015        }
15016
15017        async fn probe(&mut self) -> crate::router::OutboundFrame {
15018            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15019                Ok(frame) => Some(frame),
15020                Err(mpsc::error::TryRecvError::Empty) => None,
15021                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15022            })
15023            .await;
15024            let body: Value = serde_json::from_slice(&frame.body).unwrap();
15025            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15026            frame
15027        }
15028
15029        fn no_probe(&mut self) {
15030            assert!(matches!(
15031                self.rx.try_recv(),
15032                Err(mpsc::error::TryRecvError::Empty)
15033            ));
15034        }
15035
15036        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15037            self.forwarding
15038                .complete_module_control_rpc(
15039                    ConnectionId::new(connection),
15040                    frame.header.corr,
15041                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15042                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15043                        status: HealthStatus::Ok,
15044                        detail: None,
15045                        metrics: None,
15046                    }),
15047                )
15048                .unwrap();
15049        }
15050    }
15051
15052    #[tokio::test(start_paused = true)]
15053    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15054        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15055        let initial = actor.module.status().unwrap();
15056        tokio::time::advance(Duration::from_secs(119)).await;
15057        tokio::task::yield_now().await;
15058        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15059        assert!(actor.module.terminal_history().entries.is_empty());
15060        actor.expire_registration_wait(Duration::from_secs(1)).await;
15061        actor
15062            .wait_for_select(|at| at.generation > initial.spawn_generation)
15063            .await;
15064        let status = actor.module.status().unwrap();
15065        assert_ne!(status.pid, initial.pid);
15066        assert!(!cortexkit_test_support::process_alive(
15067            initial.pid.unwrap() as i32
15068        ));
15069        assert_eq!(status.restart_count, 1);
15070        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15071        let entries = actor.module.terminal_history().entries;
15072        assert_eq!(entries.len(), 1);
15073        assert_eq!(
15074            entries[0].disposition_detail.as_deref(),
15075            Some("registration_timeout")
15076        );
15077        actor.module.stop().await.unwrap();
15078    }
15079
15080    #[tokio::test(start_paused = true)]
15081    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15082        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15083        let pid = actor.module.status().unwrap().pid;
15084        tokio::time::advance(Duration::from_secs(119)).await;
15085        actor.hello(1, false);
15086        let registered = actor
15087            .wait_for_select(|at| at.registered_connection.is_some())
15088            .await;
15089        assert_eq!(
15090            registered.wake_after, None,
15091            "HELLO must cancel the deadline"
15092        );
15093        tokio::time::advance(Duration::from_secs(300)).await;
15094        tokio::task::yield_now().await;
15095        assert_eq!(actor.module.status().unwrap().pid, pid);
15096        assert!(actor.module.terminal_history().entries.is_empty());
15097        // A disconnect after the first HELLO must not re-arm a startup timer.
15098        actor
15099            .registry
15100            .deregister_connection(ConnectionId::new(1))
15101            .unwrap();
15102        actor
15103            .wait_for_select(|at| at.registered_connection.is_none())
15104            .await;
15105        tokio::time::advance(Duration::from_secs(300)).await;
15106        tokio::task::yield_now().await;
15107        assert_eq!(actor.module.status().unwrap().pid, pid);
15108        assert!(actor.module.terminal_history().entries.is_empty());
15109        actor.module.stop().await.unwrap();
15110
15111        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15112        let pid = transient.module.status().unwrap().pid;
15113        tokio::time::advance(Duration::from_secs(119)).await;
15114        let turns = transient.turns();
15115        // Both writes happen before the supervisor can refresh the registry.
15116        // An accepted HELLO must remain evidence after its connection is gone.
15117        transient.hello(1, false);
15118        transient
15119            .registry
15120            .deregister_connection(ConnectionId::new(1))
15121            .unwrap();
15122        let parked = transient.wait_for_select(|at| at.turn > turns).await;
15123        assert_eq!(
15124            parked.wake_after, None,
15125            "transient HELLO must cancel the deadline"
15126        );
15127        tokio::time::advance(Duration::from_secs(300)).await;
15128        tokio::task::yield_now().await;
15129        assert_eq!(transient.module.status().unwrap().pid, pid);
15130        assert!(transient.module.terminal_history().entries.is_empty());
15131        transient.module.stop().await.unwrap();
15132    }
15133
15134    #[tokio::test(start_paused = true)]
15135    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15136        let actor = Actor::start_with_policy(
15137            ModuleProtocol::Subc,
15138            Duration::from_secs(30),
15139            Duration::from_secs(10),
15140            RestartPolicy::new(1, Duration::ZERO),
15141        )
15142        .await;
15143        let generation = actor.module.status().unwrap().spawn_generation;
15144        actor
15145            .expire_registration_wait(Duration::from_secs(10))
15146            .await;
15147        actor.wait_for_select(|at| at.generation > generation).await;
15148        actor
15149            .expire_registration_wait(Duration::from_secs(10))
15150            .await;
15151        let retry = wait_for("failed retry schedule", || {
15152            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15153            (state.state == ModuleState::Failed)
15154                .then_some(state.failed_retry.pending)
15155                .flatten()
15156        })
15157        .await;
15158        assert_eq!(retry.delay, Duration::from_secs(600));
15159        assert_eq!(retry.attempt, 1);
15160        let entries = actor.module.terminal_history().entries;
15161        assert_eq!(entries.len(), 2);
15162        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15163        assert!(entries[1]
15164            .disposition_detail
15165            .as_deref()
15166            .unwrap()
15167            .contains("registration_timeout"));
15168        tokio::time::advance(Duration::from_secs(599)).await;
15169        tokio::task::yield_now().await;
15170        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15171        tokio::time::advance(Duration::from_secs(1)).await;
15172        actor
15173            .wait_for_select(|at| at.generation > generation + 1)
15174            .await;
15175        // The slow retry is another ordinary spawn, with its own HELLO budget.
15176        actor
15177            .expire_registration_wait(Duration::from_secs(10))
15178            .await;
15179        wait_for("slow retry registration timeout", || {
15180            (actor.module.terminal_history().entries.len() == 3).then_some(())
15181        })
15182        .await;
15183        actor.module.stop().await.unwrap();
15184    }
15185
15186    #[tokio::test(start_paused = true)]
15187    async fn registration_timeout_honours_ten_second_override() {
15188        let actor = Actor::start_with_policy(
15189            ModuleProtocol::Subc,
15190            Duration::from_secs(30),
15191            Duration::from_secs(10),
15192            RestartPolicy::new(3, Duration::ZERO),
15193        )
15194        .await;
15195        let generation = actor.module.status().unwrap().spawn_generation;
15196        tokio::time::advance(Duration::from_secs(9)).await;
15197        tokio::task::yield_now().await;
15198        assert!(actor.module.terminal_history().entries.is_empty());
15199        actor.expire_registration_wait(Duration::from_secs(1)).await;
15200        actor.wait_for_select(|at| at.generation > generation).await;
15201        assert_eq!(actor.module.terminal_history().entries.len(), 1);
15202        actor.module.stop().await.unwrap();
15203    }
15204
15205    #[tokio::test]
15206    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15207        use std::os::unix::process::{CommandExt, ExitStatusExt};
15208
15209        struct Helper(std::process::Child);
15210        impl Drop for Helper {
15211            fn drop(&mut self) {
15212                // This handle still owns the helper until wait returns. Never
15213                // clean up its old process group after the leader is reaped.
15214                if matches!(self.0.try_wait(), Ok(None)) {
15215                    let _ = self.0.kill();
15216                    let _ = self.0.wait();
15217                }
15218            }
15219        }
15220
15221        let home = ScratchDir::new("registration-timeout-helper");
15222        let supervisor = Supervisor::new_for_test(
15223            Arc::new(Registry::default()),
15224            RestartPolicy::new(0, Duration::ZERO),
15225        );
15226        let spec = ModuleSpec {
15227            module_id: "registration-timeout-group".into(),
15228            program: "/bin/sleep".into(),
15229            args: vec!["120".into()],
15230            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15231                .into_iter()
15232                .map(|key| (key.into(), home.path().display().to_string()))
15233                .collect(),
15234            reserved: false,
15235            reserved_prefixes: vec![],
15236            protocol: ModuleProtocol::Subc,
15237            overlap: Default::default(),
15238            carry_containment: false,
15239        };
15240        let module = supervisor
15241            .supervise_configured_requested(
15242                spec,
15243                true,
15244                HealthConfig::default(),
15245                None,
15246                RestartPolicy::new(0, Duration::ZERO),
15247                Duration::from_secs(10),
15248                None,
15249            )
15250            .unwrap();
15251        let leader = timeout(Duration::from_secs(5), async {
15252            loop {
15253                if let Some(pid) = module.status().unwrap().pid {
15254                    break pid;
15255                }
15256                sleep(Duration::from_millis(5)).await;
15257            }
15258        })
15259        .await
15260        .expect("supervised group leader must be reported before the deadline");
15261        assert!(leader > 1);
15262        let ready = home.join("helper.ready");
15263        // Own the helper separately so even a broken group kill can be cleaned
15264        // up by its own unreaped child handle, without signalling a stale pgid.
15265        let mut helper = Helper(
15266            std::process::Command::new("/bin/sh")
15267                .args([
15268                    "-c",
15269                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15270                    "helper",
15271                ])
15272                .arg(&ready)
15273                .process_group(i32::try_from(leader).unwrap())
15274                .stdout(std::process::Stdio::null())
15275                .stderr(std::process::Stdio::null())
15276                .spawn()
15277                .unwrap(),
15278        );
15279        timeout(Duration::from_secs(5), async {
15280            while !ready.exists() {
15281                assert!(helper.0.try_wait().unwrap().is_none());
15282                sleep(Duration::from_millis(5)).await;
15283            }
15284        })
15285        .await
15286        .expect("helper must install TERM ignore before the registration deadline");
15287        let helper_pid =
15288            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15289        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15290        sleep(Duration::from_millis(20)).await;
15291        assert!(
15292            helper.0.try_wait().unwrap().is_none(),
15293            "helper must ignore TERM"
15294        );
15295
15296        timeout(Duration::from_secs(20), async {
15297            while module.state().unwrap() != ModuleState::Failed {
15298                sleep(Duration::from_millis(5)).await;
15299            }
15300        })
15301        .await
15302        .expect("registration timeout must stop and reap the group leader");
15303        let helper_exit = timeout(Duration::from_secs(3), async {
15304            loop {
15305                if let Some(status) = helper.0.try_wait().unwrap() {
15306                    break status;
15307                }
15308                sleep(Duration::from_millis(5)).await;
15309            }
15310        })
15311        .await
15312        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15313        assert_eq!(helper_exit.signal(), Some(9));
15314        assert!(!cortexkit_test_support::process_alive(
15315            i32::try_from(leader).unwrap()
15316        ));
15317        assert_eq!(
15318            module.status().unwrap().last_exit.unwrap().kind,
15319            ExitKind::Crash
15320        );
15321        assert!(module.terminal_history().entries[0]
15322            .disposition_detail
15323            .as_deref()
15324            .unwrap()
15325            .contains("registration_timeout"));
15326        module.stop().await.unwrap();
15327    }
15328
15329    #[tokio::test(start_paused = true)]
15330    async fn registration_timeout_does_not_count_an_earlier_exit() {
15331        let actor = Actor::start_with_policy(
15332            ModuleProtocol::Subc,
15333            Duration::from_secs(30),
15334            DEFAULT_REGISTER_TIMEOUT,
15335            RestartPolicy::new(0, Duration::ZERO),
15336        )
15337        .await;
15338        let pid = actor.module.status().unwrap().pid.unwrap();
15339        rustix::process::kill_process(
15340            rustix::process::Pid::from_raw(pid as i32).unwrap(),
15341            rustix::process::Signal::TERM,
15342        )
15343        .unwrap();
15344        wait_for("early exit", || {
15345            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15346        })
15347        .await;
15348        tokio::time::advance(Duration::from_secs(121)).await;
15349        tokio::task::yield_now().await;
15350        let entries = actor.module.terminal_history().entries;
15351        assert_eq!(entries.len(), 1);
15352        assert!(!entries[0]
15353            .disposition_detail
15354            .as_deref()
15355            .unwrap()
15356            .contains("registration_timeout"));
15357        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15358        actor.module.stop().await.unwrap();
15359    }
15360
15361    #[tokio::test(start_paused = true)]
15362    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15363        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15364        let pid = actor.module.status().unwrap().pid;
15365        tokio::time::advance(Duration::from_secs(119)).await;
15366        actor.roster.close();
15367        let turns = actor.turns();
15368        tokio::time::advance(Duration::from_secs(300)).await;
15369        let parked = actor.wait_for_select(|at| at.turn > turns).await;
15370        assert_eq!(parked.wake_after, None);
15371        assert_eq!(actor.module.status().unwrap().pid, pid);
15372        assert!(actor.module.terminal_history().entries.is_empty());
15373        actor.module.stop().await.unwrap();
15374        let status = actor.module.status().unwrap();
15375        assert_eq!(status.restart_count, 0);
15376        assert_eq!(
15377            actor.module.terminal_history().entries[0].disposition,
15378            TerminalDisposition::DaemonShutdown
15379        );
15380    }
15381
15382    #[tokio::test(start_paused = true)]
15383    async fn no_health_children_park_for_sixty_seconds() {
15384        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15385        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15386        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15387        unadvertised.hello(1, false);
15388        // Include any one-off registration wake in the budget; receiving a
15389        // no-health HELLO must not turn a parked child into a polling child.
15390        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
15391        // a single 60 s jump would only observe one expired timer.
15392        for _ in 0..6000 {
15393            tokio::time::advance(Duration::from_millis(10)).await;
15394            tokio::task::yield_now().await;
15395        }
15396        for actor in [&none, &unregistered, &unadvertised] {
15397            assert!(
15398                actor.turns() <= 5,
15399                "no-health actor woke {} times",
15400                actor.turns()
15401            );
15402            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15403        }
15404    }
15405
15406    #[tokio::test(start_paused = true)]
15407    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15408        // With a zero cadence the first probe is due at once, so this measures
15409        // only how fast the registration notification starts health probing,
15410        // not the normal 30-33 s wait before a first probe (checked below).
15411        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15412        let tick = Instant::now();
15413        actor.hello(1, true);
15414        actor.probe().await;
15415        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15416    }
15417
15418    #[tokio::test(start_paused = true)]
15419    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15420        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15421        let hello_tick = Instant::now();
15422        actor.hello(1, true);
15423        let armed = actor
15424            .wait_for_select(|checkpoint| {
15425                checkpoint.registered_connection == Some(ConnectionId::new(1))
15426                    && checkpoint.next_probe_at.is_some()
15427            })
15428            .await;
15429        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15430        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15431        let first_deadline = hello_tick + delay;
15432        assert_eq!(armed.next_probe_at, Some(first_deadline));
15433        tokio::time::advance(Duration::from_secs(29)).await;
15434        let turns = actor.turns();
15435        actor
15436            .registry
15437            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15438            .unwrap();
15439        let updated = actor
15440            .wait_for_select(|checkpoint| checkpoint.turn > turns)
15441            .await;
15442        assert_eq!(
15443            updated.next_probe_at,
15444            Some(first_deadline),
15445            "catalog update must not reset cadence"
15446        );
15447        actor.no_probe();
15448        tokio::time::advance(first_deadline - Instant::now()).await;
15449        let frame = actor.probe().await;
15450        actor.answer(1, frame);
15451        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15452        let next_deadline = Instant::now() + next;
15453        let rearmed = actor
15454            .wait_for_select(|checkpoint| {
15455                checkpoint
15456                    .next_probe_at
15457                    .is_some_and(|deadline| deadline > first_deadline)
15458            })
15459            .await;
15460        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15461        tokio::time::advance(next - Duration::from_nanos(1)).await;
15462        actor.no_probe();
15463        tokio::time::advance(Duration::from_nanos(1)).await;
15464        actor.probe().await;
15465    }
15466
15467    #[tokio::test(start_paused = true)]
15468    async fn disconnect_disarms_and_reconnect_rearms_health() {
15469        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15470        actor.hello(1, true);
15471        actor
15472            .wait_for_select(|checkpoint| {
15473                checkpoint.registered_connection == Some(ConnectionId::new(1))
15474                    && checkpoint.next_probe_at.is_some()
15475            })
15476            .await;
15477        let turns = actor.turns();
15478        let tick = Instant::now();
15479        actor
15480            .registry
15481            .deregister_connection(ConnectionId::new(1))
15482            .unwrap();
15483        let disarmed = actor
15484            .wait_for_select(|checkpoint| {
15485                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15486            })
15487            .await;
15488        assert_eq!(disarmed.next_probe_at, None);
15489        assert_eq!(disarmed.wake_after, None);
15490        assert_eq!(
15491            Instant::now(),
15492            tick,
15493            "disconnect must disarm in the notification tick"
15494        );
15495        tokio::time::advance(Duration::from_secs(60)).await;
15496        actor.no_probe();
15497        let reconnect_tick = Instant::now();
15498        actor.hello(2, true);
15499        let rearmed = actor
15500            .wait_for_select(|checkpoint| {
15501                checkpoint.registered_connection == Some(ConnectionId::new(2))
15502                    && checkpoint.next_probe_at.is_some()
15503            })
15504            .await;
15505        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15506        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15507        tokio::time::advance(delay).await;
15508        actor.probe().await;
15509    }
15510
15511    #[tokio::test(start_paused = true)]
15512    async fn reregistration_without_health_disarms_the_old_schedule() {
15513        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15514        actor.hello(1, true);
15515        actor
15516            .wait_for_select(|checkpoint| {
15517                checkpoint.registered_connection == Some(ConnectionId::new(1))
15518                    && checkpoint.next_probe_at.is_some()
15519            })
15520            .await;
15521        actor
15522            .registry
15523            .deregister_connection(ConnectionId::new(1))
15524            .unwrap();
15525        actor.hello(2, false);
15526        let disarmed = actor
15527            .wait_for_select(|checkpoint| {
15528                checkpoint.registered_connection == Some(ConnectionId::new(2))
15529            })
15530            .await;
15531        assert_eq!(disarmed.next_probe_at, None);
15532        assert_eq!(disarmed.wake_after, None);
15533        tokio::time::advance(Duration::from_secs(60)).await;
15534        actor.no_probe();
15535    }
15536
15537    #[tokio::test(start_paused = true)]
15538    async fn restart_waits_for_its_own_hello_before_rearming_health() {
15539        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15540        actor.hello(1, true);
15541        let armed = actor
15542            .wait_for_select(|checkpoint| {
15543                checkpoint.registered_connection == Some(ConnectionId::new(1))
15544                    && checkpoint.next_probe_at.is_some()
15545            })
15546            .await;
15547        // Simulate the old process's connection closing before the restart
15548        // tears the process down. The replacement process has started but has
15549        // not sent its HELLO, so it is not registered.
15550        actor
15551            .registry
15552            .deregister_connection(ConnectionId::new(1))
15553            .unwrap();
15554        actor.module.restart(Some(0)).await.unwrap();
15555        let parked = actor
15556            .wait_for_select(|checkpoint| {
15557                checkpoint.generation > armed.generation
15558                    && checkpoint.registered_connection.is_none()
15559                    && checkpoint.next_probe_at.is_none()
15560            })
15561            .await;
15562        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15563        actor.rx.close();
15564        // Drain the stop notice the restart queued for the old process's connection,
15565        // so it isn't mistaken for traffic to the replacement.
15566        while actor.rx.try_recv().is_ok() {}
15567        let turns = actor.turns();
15568        tokio::time::advance(Duration::from_secs(60)).await;
15569        let after = actor.turns();
15570        eprintln!(
15571            "restart park turns: before={turns} after={after} delta={}",
15572            after - turns
15573        );
15574        assert_eq!(after, turns, "replacement must park before HELLO");
15575        let hello_tick = Instant::now();
15576        actor.hello(2, true);
15577        let rearmed = actor
15578            .wait_for_select(|checkpoint| {
15579                checkpoint.registered_connection == Some(ConnectionId::new(2))
15580                    && checkpoint.next_probe_at.is_some()
15581            })
15582            .await;
15583        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15584        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15585        tokio::time::advance(delay).await;
15586        actor.probe().await;
15587    }
15588
15589    #[tokio::test(start_paused = true)]
15590    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15591        use tokio::{
15592            io::{AsyncReadExt, AsyncWriteExt},
15593            net::TcpListener,
15594        };
15595        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15596        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15597        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15598        let (started_tx, mut started_rx) = mpsc::channel(8);
15599        let peer = tokio::spawn(async move {
15600            loop {
15601                let (mut socket, _) = listener.accept().await.unwrap();
15602                let mut request = [0; 1024];
15603                assert!(socket.read(&mut request).await.unwrap() > 0);
15604                started_tx.send(Instant::now()).await.unwrap();
15605                socket
15606                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15607                    .await
15608                    .unwrap();
15609            }
15610        });
15611        actor
15612            .module
15613            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15614            .await
15615            .unwrap();
15616        let first_tick = Instant::now();
15617        let first = actor
15618            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15619            .await;
15620        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15621        assert!(started_rx.try_recv().is_err());
15622
15623        // Restart just before the old deadline: reusing it would give the new
15624        // process only one second to start, rather than a full cadence.
15625        tokio::time::advance(
15626            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15627        )
15628        .await;
15629        actor.module.restart(Some(0)).await.unwrap();
15630        let spawn_tick = Instant::now();
15631        let replacement = actor
15632            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15633            .await;
15634        let deadline = replacement.next_probe_at.unwrap();
15635        assert!(
15636            deadline >= spawn_tick + actor.health.cadence,
15637            "a replacement must not inherit its predecessor's probe deadline"
15638        );
15639        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15640        assert!(
15641            started_rx.try_recv().is_err(),
15642            "no HTTP probe during startup grace"
15643        );
15644        tokio::time::advance(deadline - Instant::now()).await;
15645        let landed = wait_for("first replacement HTTP probe", || {
15646            started_rx.try_recv().ok()
15647        })
15648        .await;
15649        assert!(landed >= spawn_tick + actor.health.cadence);
15650        wait_for("healthy replacement", || {
15651            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15652                .then_some(())
15653        })
15654        .await;
15655        peer.abort();
15656    }
15657
15658    #[tokio::test(start_paused = true)]
15659    async fn rescan_adds_and_removes_http_health_without_polling() {
15660        use tokio::{
15661            io::{AsyncReadExt, AsyncWriteExt},
15662            net::TcpListener,
15663        };
15664        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15665        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15666        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15667        // One millisecond separates successive probes without a polling delay.
15668        actor.health.cadence = Duration::from_millis(1);
15669        let (started_tx, mut started_rx) = mpsc::channel(8);
15670        let peer = tokio::spawn(async move {
15671            loop {
15672                let (mut socket, _) = listener.accept().await.unwrap();
15673                let mut request = [0; 1024];
15674                assert!(socket.read(&mut request).await.unwrap() > 0);
15675                started_tx.send(()).await.unwrap();
15676                socket
15677                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15678                    .await
15679                    .unwrap();
15680            }
15681        });
15682        let tick = Instant::now();
15683        let turns = actor.turns();
15684        actor
15685            .module
15686            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15687            .await
15688            .unwrap();
15689        let armed = actor
15690            .wait_for_select(|checkpoint| {
15691                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15692            })
15693            .await;
15694        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15695        assert_eq!(Instant::now(), tick);
15696        tokio::time::advance(Duration::from_millis(1)).await;
15697        let mut started = false;
15698        wait_for("successful HTTP probe", || {
15699            started |= started_rx.try_recv().is_ok();
15700            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15701                .then_some(())
15702        })
15703        .await;
15704        assert!(started, "added HTTP check must start at its first deadline");
15705        assert_eq!(
15706            actor.module.status().unwrap().health.status,
15707            SupervisorHealthStatus::Ok
15708        );
15709        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15710        actor.health.http = None;
15711        let turns = actor.turns();
15712        actor
15713            .module
15714            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15715            .await
15716            .unwrap();
15717        let parked = actor
15718            .wait_for_select(|checkpoint| {
15719                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15720            })
15721            .await;
15722        assert_eq!(parked.wake_after, None);
15723        let turns = actor.turns();
15724        tokio::time::advance(Duration::from_secs(60)).await;
15725        assert_eq!(
15726            actor.turns(),
15727            turns,
15728            "removed HTTP check must park the actor"
15729        );
15730        assert!(
15731            started_rx.try_recv().is_err(),
15732            "removed HTTP check must stay disarmed"
15733        );
15734        assert_eq!(
15735            actor.module.status().unwrap().health,
15736            ModuleHealthStatus::default()
15737        );
15738        peer.abort();
15739    }
15740}
15741
15742#[cfg(test)]
15743mod drain_busy_reprobe_tests {
15744    //! The drain's busy-gauge wait, driven against an in-process module
15745    //! connection whose health answers the test scripts. Time is paused, so
15746    //! every elapsed duration below is exact virtual time.
15747    use super::*;
15748    use std::collections::BTreeMap;
15749    use std::sync::Mutex as StdMutex;
15750    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15751
15752    const MODULE_ID: &str = "drain-reprobe-module";
15753    const CONNECTION: u64 = 7;
15754    const GAUGE: &str = "runs_in_flight";
15755    /// The production default for both the health cadence and the drain
15756    /// ceiling, which is the pairing that used to cost a full ceiling.
15757    const CADENCE: Duration = Duration::from_secs(30);
15758    const CEILING: Duration = Duration::from_secs(30);
15759
15760    /// How the scripted module answers each `health.check`.
15761    #[derive(Clone, Copy)]
15762    enum Answer {
15763        /// Report the gauge busy until this instant, quiet from then on, after
15764        /// taking `latency` to answer.
15765        BusyUntil { until: Instant, latency: Duration },
15766        /// Answer at once, with metrics that leave the declared gauge out.
15767        OmitsGauge,
15768        /// Never answer.
15769        Silent,
15770    }
15771
15772    struct Drain {
15773        forwarding: Arc<ForwardingTable>,
15774        runtime: SupervisorRuntimeConfig,
15775        endpoint: crate::ModuleEndpointId,
15776        /// When each probe reached the module.
15777        probes: Arc<StdMutex<Vec<Instant>>>,
15778        responder: tokio::task::JoinHandle<()>,
15779    }
15780
15781    impl Drain {
15782        fn start(answer: Answer) -> Self {
15783            let registry = Arc::new(Registry::default());
15784            let forwarding = Arc::new(ForwardingTable::default());
15785            let health = HealthConfig {
15786                cadence: CADENCE,
15787                ..HealthConfig::default()
15788            };
15789            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15790                .with_forwarding(forwarding.clone())
15791                .with_health_config(health)
15792                .runtime_config();
15793            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15794            let endpoint = forwarding
15795                .register_module_connection(
15796                    ConnectionId::new(CONNECTION),
15797                    MODULE_ID.to_string(),
15798                    subc_protocol::PROTOCOL_VERSION,
15799                    Concurrency::ModuleManaged,
15800                    FrameSink::new(tx),
15801                )
15802                .unwrap();
15803            let probes = Arc::new(StdMutex::new(Vec::new()));
15804            let responder = tokio::spawn({
15805                let forwarding = forwarding.clone();
15806                let probes = probes.clone();
15807                async move {
15808                    while let Some(frame) = rx.recv().await {
15809                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15810                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15811                        probes.lock().unwrap().push(Instant::now());
15812                        let metrics = match answer {
15813                            Answer::BusyUntil { until, latency } => {
15814                                sleep(latency).await;
15815                                let busy = u64::from(Instant::now() < until);
15816                                serde_json::json!({ GAUGE: busy })
15817                            }
15818                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15819                            Answer::Silent => continue,
15820                        };
15821                        // A probe abandoned at its deadline has no RPC left to
15822                        // complete, so a late answer may be refused.
15823                        let _ = forwarding.complete_module_control_rpc(
15824                            ConnectionId::new(CONNECTION),
15825                            frame.header.corr,
15826                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15827                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15828                                status: HealthStatus::Ok,
15829                                detail: None,
15830                                metrics: Some(metrics),
15831                            }),
15832                        );
15833                    }
15834                }
15835            });
15836            Self {
15837                forwarding,
15838                runtime,
15839                endpoint,
15840                probes,
15841                responder,
15842            }
15843        }
15844
15845        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15846        /// how long it took.
15847        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15848            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15849            (drained, elapsed)
15850        }
15851
15852        /// [`Self::run`], also returning what the drain's probes read.
15853        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15854            let started = Instant::now();
15855            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15856            let drained = wait_for_forwarding_quiescence(
15857                &self.forwarding,
15858                MODULE_ID,
15859                &self.runtime,
15860                self.endpoint,
15861                started + ceiling,
15862                &[GAUGE.to_string()],
15863                DrainScope::Active,
15864                &mut trace,
15865            )
15866            .await
15867            .unwrap();
15868            (drained, started.elapsed(), trace)
15869        }
15870
15871        fn probes(&self) -> Vec<Instant> {
15872            self.probes.lock().unwrap().clone()
15873        }
15874    }
15875
15876    impl Drop for Drain {
15877        fn drop(&mut self) {
15878            self.responder.abort();
15879        }
15880    }
15881
15882    #[tokio::test(start_paused = true)]
15883    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15884        let drain = Drain::start(Answer::BusyUntil {
15885            until: Instant::now() + Duration::from_secs(1),
15886            latency: Duration::ZERO,
15887        });
15888        let (drained, elapsed) = drain.run(CEILING).await;
15889        assert!(
15890            drained,
15891            "gauges that read zero during the drain must let it finish drained, \
15892             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15893        );
15894        assert!(
15895            elapsed < Duration::from_secs(3),
15896            "drain took {elapsed:?} after the gauges went quiet at 1s"
15897        );
15898        let probes = drain.probes();
15899        assert!(
15900            probes.len() >= 2,
15901            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15902        );
15903    }
15904
15905    /// The same shape in real time on a multi-threaded runtime: the first
15906    /// probe is answered busy at once, within a millisecond of the drain
15907    /// starting, and later probes read idle. The drain must keep probing
15908    /// after that first instant answer and end soon after the gauge falls.
15909    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15910    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15911        let drain = Drain::start(Answer::BusyUntil {
15912            until: Instant::now() + Duration::from_millis(600),
15913            latency: Duration::ZERO,
15914        });
15915        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15916        assert!(drained, "the gauge fell at 600ms: {trace}");
15917        assert!(
15918            elapsed < Duration::from_secs(3),
15919            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15920        );
15921        let probes = drain.probes();
15922        assert!(
15923            probes.len() >= 2 && trace.probes as usize == probes.len(),
15924            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15925        );
15926    }
15927
15928    #[tokio::test(start_paused = true)]
15929    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15930        let started = Instant::now();
15931        let drain = Drain::start(Answer::BusyUntil {
15932            until: started + Duration::from_secs(3600),
15933            latency: Duration::ZERO,
15934        });
15935        let (drained, elapsed) = drain.run(CEILING).await;
15936        assert!(!drained, "a module busy throughout must not drain");
15937        assert!(
15938            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15939            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15940        );
15941        // Re-probing is paced by the interval, not a busy loop: at most one
15942        // probe per interval across the ceiling, plus the first at time zero.
15943        let probes = drain.probes();
15944        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15945        assert!(
15946            probes.len() as u128 <= max_probes,
15947            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15948            probes.len()
15949        );
15950        for pair in probes.windows(2) {
15951            assert!(
15952                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15953                "consecutive probes {:?} apart",
15954                pair[1] - pair[0]
15955            );
15956        }
15957    }
15958
15959    #[tokio::test(start_paused = true)]
15960    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15961        // Each answer takes longer than the re-probe interval; the next probe
15962        // must wait for the previous answer and then the interval.
15963        let latency = Duration::from_secs(1);
15964        let drain = Drain::start(Answer::BusyUntil {
15965            until: Instant::now() + Duration::from_secs(3600),
15966            latency,
15967        });
15968        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15969        assert!(!drained);
15970        let probes = drain.probes();
15971        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15972        for pair in probes.windows(2) {
15973            assert!(
15974                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15975                "a probe started {:?} after the previous one, before it could finish",
15976                pair[1] - pair[0]
15977            );
15978        }
15979    }
15980
15981    #[tokio::test(start_paused = true)]
15982    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15983        let drain = Drain::start(Answer::Silent);
15984        let ceiling = Duration::from_secs(2);
15985        let (drained, elapsed) = drain.run(ceiling).await;
15986        assert!(!drained, "an unanswered probe counts as busy");
15987        assert!(
15988            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15989            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15990        );
15991        assert_eq!(
15992            drain.probes().len(),
15993            1,
15994            "the unanswered probe holds the only slot until the ceiling"
15995        );
15996    }
15997
15998    /// Every event logged while it is installed, as `message` plus fields.
15999    #[derive(Clone, Default)]
16000    struct LogCapture {
16001        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16002    }
16003
16004    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16005        fn on_event(
16006            &self,
16007            event: &tracing::Event<'_>,
16008            _context: tracing_subscriber::layer::Context<'_, S>,
16009        ) {
16010            struct Fields(BTreeMap<String, String>);
16011            impl tracing::field::Visit for Fields {
16012                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16013                    self.0
16014                        .insert(field.name().to_string(), format!("{value:?}"));
16015                }
16016            }
16017            let mut fields = Fields(BTreeMap::new());
16018            event.record(&mut fields);
16019            self.events.lock().unwrap().push(fields.0);
16020        }
16021    }
16022
16023    impl LogCapture {
16024        fn install(&self) -> tracing::subscriber::DefaultGuard {
16025            use tracing_subscriber::layer::SubscriberExt;
16026            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16027        }
16028
16029        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16030            self.events
16031                .lock()
16032                .unwrap()
16033                .iter()
16034                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16035                .cloned()
16036                .collect()
16037        }
16038    }
16039
16040    /// A module that declares a busy gauge but leaves it out of its health
16041    /// answers holds a drain to its ceiling. The timeout line must say so by
16042    /// name, and the drain must warn about the missing gauge once, not on
16043    /// every probe.
16044    #[tokio::test(start_paused = true)]
16045    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16046        let capture = LogCapture::default();
16047        let _guard = capture.install();
16048        let drain = Drain::start(Answer::OmitsGauge);
16049        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16050        assert!(!drained, "a gauge never reported cannot read quiet");
16051        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16052
16053        let missing = capture.with_message("lacks a declared busy gauge");
16054        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16055        assert_eq!(
16056            missing[0].get("missing_gauges").map(String::as_str),
16057            Some(GAUGE)
16058        );
16059
16060        warn_drain_timed_out(
16061            MODULE_ID,
16062            Duration::from_secs(2),
16063            RouteCloseReason::Restart,
16064            &Default::default(),
16065            &trace,
16066        );
16067        let timeouts = capture.with_message("route drain timed out");
16068        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16069        let gauges = &timeouts[0]["busy_gauges"];
16070        assert!(
16071            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16072            "the timeout line must name the gauge and why it read busy: {gauges}"
16073        );
16074        assert!(
16075            gauges.ends_with("; no value ever read"),
16076            "no value was ever read for the gauge: {gauges}"
16077        );
16078        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16079    }
16080
16081    /// The trace names a busy value and when it was read. For a probe that
16082    /// never got an answer, it names the probe's error kind and says that no
16083    /// value was ever read.
16084    #[tokio::test(start_paused = true)]
16085    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16086        let busy = Drain::start(Answer::BusyUntil {
16087            until: Instant::now() + Duration::from_secs(3600),
16088            latency: Duration::ZERO,
16089        });
16090        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16091        let rendered = trace.to_string();
16092        assert!(
16093            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16094                && rendered.contains("; last value non_zero(1) @"),
16095            "{rendered}"
16096        );
16097
16098        let silent = Drain::start(Answer::Silent);
16099        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16100        // The probe is capped at the 1 s ceiling, and its reading is stamped
16101        // when it gave up.
16102        assert_eq!(
16103            trace.to_string(),
16104            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16105        );
16106        assert_eq!(trace.probes, 1);
16107    }
16108
16109    /// A gauge value that is not a count reads busy and is shown as sent.
16110    #[test]
16111    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16112        let gauges = [GAUGE.to_string()];
16113        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16114        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16115        assert_eq!(
16116            busy_gauge_observation(&verdicts),
16117            BusyGaugeObservation::Busy
16118        );
16119        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16120        assert_eq!(
16121            busy_gauge_observation(&zero),
16122            BusyGaugeObservation::Quiescent
16123        );
16124        let none = gauge_verdicts(None, &gauges);
16125        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16126    }
16127}
16128
16129#[cfg(test)]
16130mod health_evidence_tests {
16131    use super::{HealthProbeError, HealthProbeEvidence};
16132    use std::collections::HashSet;
16133
16134    /// The evidential asymmetry, asserted rather than described.
16135    ///
16136    /// Exactly ONE observation is proof a module cannot serve, and the one that
16137    /// fires under CPU starvation is not it. Before the split, all fifteen
16138    /// construction sites collapsed into a single String, so a timeout carried the
16139    /// same weight as a dead lane -- which is how a healthy module was restarted
16140    /// three times in one day.
16141    #[test]
16142    fn only_a_dead_lane_is_proof_of_death() {
16143        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16144        // Three non-proof classes, each for a different reason: silence is
16145        // consistent with health, a bad answer proves the module ALIVE, and a
16146        // daemon-side fault never reached the module at all.
16147        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16148        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16149        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16150    }
16151
16152    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
16153    ///
16154    /// A shared label renders two different observations identically in the line an
16155    /// operator reads after an unexplained restart -- the exact confusion this
16156    /// change removes.
16157    #[test]
16158    fn every_evidence_class_has_a_distinct_label() {
16159        let labels = [
16160            HealthProbeError::lane_dead("").label(),
16161            HealthProbeError::no_answer("").label(),
16162            HealthProbeError::bad_answer("").label(),
16163            HealthProbeError::misconfigured("").label(),
16164        ];
16165        let unique: HashSet<_> = labels.iter().collect();
16166        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16167    }
16168
16169    /// The class is additional information, not a replacement.
16170    ///
16171    /// An operator needs both "this was silence" and the specific text saying how
16172    /// long we waited; a classification that swallowed the message would trade one
16173    /// missing distinction for another.
16174    #[test]
16175    fn classification_preserves_the_original_message() {
16176        let err = HealthProbeError::no_answer("module did not answer within 5s");
16177        assert_eq!(err.to_string(), "module did not answer within 5s");
16178        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16179    }
16180}
16181
16182#[cfg(test)]
16183mod health_tombstone_tests {
16184    use std::{path::PathBuf, sync::Arc, time::Duration};
16185
16186    use subc_protocol::{
16187        manifest::Concurrency,
16188        session::{HealthStatus, ModuleControlResponse},
16189    };
16190    use tokio::sync::mpsc;
16191
16192    use super::{
16193        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16194        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16195    };
16196    use crate::{
16197        control::ControlHandler,
16198        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16199        registry::{ConnectionId, Registry},
16200        router::FrameSink,
16201    };
16202
16203    struct ProbeHarness {
16204        spec: ModuleSpec,
16205        runtime: SupervisorRuntimeConfig,
16206        forwarding: Arc<ForwardingTable>,
16207        module_connection: ConnectionId,
16208        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16209        handler: ControlHandler,
16210        module: super::SupervisedModule,
16211    }
16212
16213    fn probe_harness() -> ProbeHarness {
16214        let registry = Arc::new(Registry::default());
16215        let forwarding = Arc::new(ForwardingTable::default());
16216        let supervisor_handle = super::SupervisorHandle::new();
16217        let health = HealthConfig {
16218            http: None,
16219            cadence: Duration::from_secs(30),
16220            deadline: Duration::from_secs(5),
16221            failure_threshold: 3,
16222            on_degraded: HealthAction::Report,
16223            on_failing: HealthAction::Report,
16224            critical: false,
16225        };
16226        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16227            .with_forwarding(Arc::clone(&forwarding))
16228            .with_handle(supervisor_handle.clone())
16229            .with_health_config(health);
16230        let spec = ModuleSpec {
16231            module_id: "late-health-module".to_string(),
16232            program: PathBuf::from("disabled-module"),
16233            args: Vec::new(),
16234            env: Vec::new(),
16235            reserved: false,
16236            reserved_prefixes: Vec::new(),
16237            protocol: ModuleProtocol::Subc,
16238            overlap: Default::default(),
16239            carry_containment: false,
16240        };
16241        let module = supervisor
16242            .supervise_configured(spec.clone(), false)
16243            .unwrap();
16244        let runtime = supervisor.runtime_config();
16245        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16246            .with_supervisor(supervisor_handle);
16247        let module_connection = ConnectionId::new(700);
16248        let (module_tx, module_rx) = mpsc::channel(8);
16249        forwarding
16250            .register_module_connection(
16251                module_connection,
16252                spec.module_id.clone(),
16253                subc_protocol::PROTOCOL_VERSION,
16254                Concurrency::ModuleManaged,
16255                FrameSink::new(module_tx),
16256            )
16257            .unwrap();
16258
16259        ProbeHarness {
16260            spec,
16261            runtime,
16262            forwarding,
16263            module_connection,
16264            module_rx,
16265            handler,
16266            module,
16267        }
16268    }
16269
16270    async fn finish_after(
16271        harness: &mut ProbeHarness,
16272        stall: Duration,
16273    ) -> ModuleControlRpcCompletion {
16274        assert!(stall > harness.runtime.health.deadline);
16275        let deadline = harness.runtime.health.deadline;
16276        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16277        let answer = async {
16278            let frame = harness.module_rx.recv().await.expect("health.check frame");
16279            tokio::time::advance(deadline).await;
16280            tokio::task::yield_now().await;
16281            tokio::time::advance(stall - deadline).await;
16282            harness
16283                .forwarding
16284                .complete_module_control_rpc(
16285                    harness.module_connection,
16286                    frame.header.corr,
16287                    Some("health.check"),
16288                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16289                        status: HealthStatus::Ok,
16290                        detail: None,
16291                        metrics: None,
16292                    }),
16293                )
16294                .unwrap()
16295        };
16296        let (probe_result, completion) = tokio::join!(probe, answer);
16297        let err = probe_result.expect_err("probe must miss its deadline");
16298        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16299        completion
16300    }
16301
16302    async fn time_out_without_answer(harness: &mut ProbeHarness) {
16303        let deadline = harness.runtime.health.deadline;
16304        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16305        let exhaust_deadline = async {
16306            let _frame = harness.module_rx.recv().await.expect("health.check frame");
16307            tokio::time::advance(deadline).await;
16308            tokio::task::yield_now().await;
16309        };
16310        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16311        let err = probe_result.expect_err("probe must miss its deadline");
16312        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16313    }
16314
16315    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16316        let registry = Arc::clone(&harness.module.inner.registry);
16317        let snapshot = Arc::clone(&harness.module.inner.snapshot);
16318        let process_liveness = super::SupervisorProcessLiveness::default();
16319        let mut child = None;
16320        let cycle = super::run_health_probe_cycle(
16321            &harness.spec,
16322            &harness.runtime,
16323            &registry,
16324            &process_liveness,
16325            &snapshot,
16326            &mut child,
16327        );
16328        let peer = async {
16329            let frame = harness.module_rx.recv().await.expect("health.check frame");
16330            if answer {
16331                harness
16332                    .forwarding
16333                    .complete_module_control_rpc(
16334                        harness.module_connection,
16335                        frame.header.corr,
16336                        Some("health.check"),
16337                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16338                            status: HealthStatus::Ok,
16339                            detail: None,
16340                            metrics: Some(serde_json::json!({"ready": true})),
16341                        }),
16342                    )
16343                    .unwrap();
16344            } else {
16345                tokio::time::advance(harness.runtime.health.deadline).await;
16346                tokio::task::yield_now().await;
16347            }
16348        };
16349        tokio::join!(cycle, peer);
16350    }
16351
16352    #[tokio::test(start_paused = true)]
16353    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16354        let mut harness = probe_harness();
16355        // Drive the probe cycle directly with an in-memory wire peer. Stop the
16356        // disabled module's monitor so only this test owns lifecycle transitions;
16357        // no OS process is launched, and a restart is observed at scheduling.
16358        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16359        monitor.abort();
16360        let _ = monitor.await;
16361        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16362            state.enabled = true;
16363            state.state = super::ModuleState::Running;
16364            state.process_alive = true;
16365        })
16366        .unwrap();
16367
16368        run_probe_cycle(&mut harness, true).await;
16369        assert_eq!(
16370            harness.module.status().unwrap().health.status,
16371            super::SupervisorHealthStatus::Ok
16372        );
16373
16374        for failures in 1..harness.runtime.health.failure_threshold {
16375            run_probe_cycle(&mut harness, false).await;
16376            let status = harness.module.status().unwrap();
16377            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16378            assert_eq!(status.health.consecutive_failures, failures);
16379            assert!(status.health.last_probe_ms.is_some());
16380            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16381            assert!(status.health.metrics.is_none());
16382            assert_eq!(status.state, super::ModuleState::Running);
16383            assert!(status.process_alive);
16384            assert_eq!(status.restart_count, 0);
16385            assert_eq!(status.lifetime_restarts, 0);
16386            assert!(status.health.last_action.is_none());
16387            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16388        }
16389
16390        run_probe_cycle(&mut harness, true).await;
16391        let recovered = harness.module.status().unwrap();
16392        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16393        assert_eq!(recovered.health.consecutive_failures, 0);
16394        assert!(recovered.health.detail.is_none());
16395        assert_eq!(
16396            recovered.health.metrics,
16397            Some(serde_json::json!({"ready": true}))
16398        );
16399        assert_eq!(recovered.lifetime_restarts, 0);
16400
16401        for failures in 1..=harness.runtime.health.failure_threshold {
16402            run_probe_cycle(&mut harness, false).await;
16403            let status = harness.module.status().unwrap();
16404            assert_eq!(status.health.consecutive_failures, failures);
16405            if failures < harness.runtime.health.failure_threshold {
16406                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16407                assert_eq!(status.state, super::ModuleState::Running);
16408                assert_eq!(status.lifetime_restarts, 0);
16409                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16410            } else {
16411                assert_eq!(
16412                    status.health.status,
16413                    super::SupervisorHealthStatus::Unresponsive
16414                );
16415                assert_eq!(status.state, super::ModuleState::Restarting);
16416                assert_eq!(status.restart_count, 1);
16417                assert_eq!(status.lifetime_restarts, 1);
16418                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16419                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16420            }
16421        }
16422    }
16423
16424    #[tokio::test(start_paused = true)]
16425    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16426        let mut harness = probe_harness();
16427
16428        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16429        let first_latency = match &first {
16430            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16431            other => panic!("late answer was not retained: {other:?}"),
16432        };
16433        assert!(harness.handler.observe_module_control_completion(first));
16434
16435        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16436        let second_latency = match &second {
16437            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16438            other => panic!("late answer was not retained: {other:?}"),
16439        };
16440        assert!(harness.handler.observe_module_control_completion(second));
16441
16442        assert_eq!(first_latency, Duration::from_secs(8));
16443        assert_eq!(
16444            second_latency - first_latency,
16445            Duration::from_secs(3),
16446            "latency must grow linearly with the additional stall"
16447        );
16448        let health = harness.module.status().unwrap().health;
16449        assert_eq!(health.late_answer_count, 2);
16450        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16451    }
16452
16453    /// A module that answers every probe late must never march to the kill
16454    /// threshold: the late answer proves it is alive, so it must clear the miss
16455    /// streak the timeout recorded. Without the reset, a CPU-starved module
16456    /// that serves every probe seconds past the deadline accumulates
16457    /// `consecutive_failures` to the threshold and is killed — the exact
16458    /// sequence from the 2026-08-14 aft disable, where the daemon logged
16459    /// "proves the module is alive" five times while counting five misses.
16460    #[tokio::test(start_paused = true)]
16461    async fn late_answer_clears_the_consecutive_failure_streak() {
16462        let mut harness = probe_harness();
16463
16464        // Timeout recorded first: the probe path saw no answer in time.
16465        time_out_without_answer(&mut harness).await;
16466        harness
16467            .module
16468            .record_health_probe_failure_for_test("[no-answer] test miss")
16469            .unwrap();
16470        assert_eq!(
16471            harness.module.status().unwrap().health.consecutive_failures,
16472            1,
16473            "precondition: the miss must be on the streak before the late answer"
16474        );
16475
16476        // The stalled reply then lands: proof of life.
16477        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16478        assert!(matches!(
16479            late,
16480            ModuleControlRpcCompletion::LateHealthAnswer { .. }
16481        ));
16482        assert!(harness.handler.observe_module_control_completion(late));
16483
16484        let health = harness.module.status().unwrap().health;
16485        assert_eq!(
16486            health.consecutive_failures, 0,
16487            "a late answer is an answer: the streak must reset"
16488        );
16489        assert_eq!(health.late_answer_count, 1);
16490    }
16491
16492    #[tokio::test(start_paused = true)]
16493    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16494        let mut harness = probe_harness();
16495
16496        for _ in 0..20 {
16497            time_out_without_answer(&mut harness).await;
16498            assert_eq!(
16499                harness.forwarding.health_probe_tombstone_count().unwrap(),
16500                1
16501            );
16502        }
16503    }
16504}
16505
16506#[cfg(test)]
16507mod child_env_tests {
16508    #[cfg(windows)]
16509    use super::{
16510        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16511    };
16512    #[cfg(windows)]
16513    use std::sync::{Arc, OnceLock};
16514    #[cfg(windows)]
16515    #[test]
16516    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16517        let spec = spec(Vec::new());
16518        let mut command = Command::new("unused");
16519        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16520            .unwrap()
16521            .unwrap();
16522        let name = command
16523            .as_std()
16524            .get_envs()
16525            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16526            .and_then(|(_, value)| value)
16527            .unwrap();
16528        assert_eq!(name, OsStr::new(handoff.name()));
16529        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16530        assert!(command.as_std().get_envs().any(|(key, value)| key
16531            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16532            && value == Some(OsStr::new("env"))));
16533        assert!(command
16534            .as_std()
16535            .get_envs()
16536            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16537    }
16538
16539    #[cfg(windows)]
16540    #[test]
16541    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16542        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16543        snapshot.process_alive = true;
16544        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16545        assert_eq!(
16546            observed_launch_nonce_source(&snapshot).as_deref(),
16547            Some("env")
16548        );
16549        let consumed = Arc::new(OnceLock::new());
16550        snapshot.pipe_consumed = Some(consumed.clone());
16551        assert_eq!(
16552            observed_launch_nonce_source(&snapshot).as_deref(),
16553            Some("env")
16554        );
16555        consumed.set(()).unwrap();
16556        assert_eq!(
16557            observed_launch_nonce_source(&snapshot).as_deref(),
16558            Some("pipe")
16559        );
16560        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16561        assert_eq!(
16562            observed_launch_nonce_source(&snapshot).as_deref(),
16563            Some("env")
16564        );
16565        clear_current_process_facts(&mut snapshot);
16566        assert_eq!(observed_launch_nonce_source(&snapshot), None);
16567    }
16568    use super::{
16569        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16570        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16571        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16572    };
16573    use std::{ffi::OsStr, path::PathBuf};
16574    use tokio::process::Command;
16575
16576    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16577        ModuleSpec {
16578            module_id: "env-plan".to_string(),
16579            program: PathBuf::from("/nonexistent"),
16580            args: Vec::new(),
16581            env,
16582            reserved: false,
16583            reserved_prefixes: Vec::new(),
16584            protocol: ModuleProtocol::Subc,
16585            overlap: Default::default(),
16586            carry_containment: false,
16587        }
16588    }
16589
16590    #[test]
16591    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16592        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16593        let digest = LaunchSpecDigest::from_spec(&base);
16594        let mut reordered = base.clone();
16595        reordered.env.reverse();
16596        reordered.module_id = "different-id".into();
16597        reordered.reserved = true;
16598        reordered.overlap = Default::default();
16599        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16600        let mut delimiter_collision = base.clone();
16601        delimiter_collision.env[0] = ("A=B".into(), "C".into());
16602        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16603        let mut changed = base.clone();
16604        changed.protocol = ModuleProtocol::None;
16605        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16606        changed = base.clone();
16607        changed.args.push("--flag".into());
16608        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16609        changed = base.clone();
16610        changed.program = PathBuf::from("/other");
16611        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16612        changed = base.clone();
16613        changed.env[1].1 = "other".into();
16614        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16615        assert!(!format!("{digest:?}").contains("B=C"));
16616        assert!(!format!("{digest:?}").contains("[u8"));
16617    }
16618
16619    #[cfg(unix)]
16620    #[test]
16621    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16622        use std::os::unix::ffi::OsStringExt;
16623        let mut left = spec(Vec::new());
16624        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16625        let mut right = left.clone();
16626        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16627        assert_eq!(
16628            left.program.to_string_lossy(),
16629            right.program.to_string_lossy()
16630        );
16631        assert_ne!(
16632            LaunchSpecDigest::from_spec(&left),
16633            LaunchSpecDigest::from_spec(&right)
16634        );
16635    }
16636
16637    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
16638    /// one still gets its own.
16639    ///
16640    /// This is the narrow goal `env_clear()` was reached for, and the reason the
16641    /// fix is `env_remove` rather than deleting the line: an operator's ambient
16642    /// filter silently becoming an unconfigured module's log level is a real
16643    /// defect, just a much smaller one than clearing the environment.
16644    ///
16645    /// Asserted on the command plan rather than a spawned child because proving
16646    /// the ABSENCE of an inherited variable needs the parent's environment
16647    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
16648    /// removal as `(key, None)`, which is exactly the distinction wanted: not
16649    /// "absent because nobody set it" but "explicitly unset for the child".
16650    #[test]
16651    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16652        let mut command = Command::new("/nonexistent");
16653        apply_child_env(&mut command, &spec(Vec::new()));
16654        let removed = command
16655            .as_std()
16656            .get_envs()
16657            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16658        assert!(
16659            removed,
16660            "ambient CK_LOG must be explicitly removed for an unconfigured module"
16661        );
16662
16663        let mut configured = Command::new("/nonexistent");
16664        apply_child_env(
16665            &mut configured,
16666            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16667        );
16668        let effective = configured
16669            .as_std()
16670            .get_envs()
16671            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16672            .last()
16673            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16674        assert_eq!(
16675            effective,
16676            Some(Some("debug".to_string())),
16677            "a module's configured CK_LOG must survive the ambient removal"
16678        );
16679    }
16680
16681    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
16682    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
16683    /// the same reason as the CK_LOG test above.
16684    ///
16685    /// The argument is the load-bearing half: a stock binary exits on an
16686    /// unknown flag before it listens, so with `--subc` appended the mode
16687    /// could not supervise the one process it exists for. Found by the first
16688    /// conformance run (nats-server: `flag provided but not defined: -subc`).
16689    #[test]
16690    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16691        let connection_file = std::path::Path::new("/run/subc-connection.json");
16692        let handle = SupervisorHandle::new();
16693
16694        let mut none_spec = spec(Vec::new());
16695        none_spec.protocol = ModuleProtocol::None;
16696        let mut none = Command::new("/nonexistent");
16697        let none_handoff =
16698            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16699                .expect("protocol-none spawn args apply");
16700        assert!(
16701            none_handoff.is_none(),
16702            "protocol:none spawn must not receive a nonce descriptor"
16703        );
16704        #[cfg(windows)]
16705        assert!(!none.as_std().get_envs().any(|(key, value)| key
16706            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16707            && value.is_some()));
16708        assert!(
16709            !none.as_std().get_envs().any(|(key, value)| key
16710                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16711                && value.is_some()),
16712            "protocol:none spawn must not name a nonce descriptor"
16713        );
16714        let none_args: Vec<String> = none
16715            .as_std()
16716            .get_args()
16717            .map(|a| a.to_string_lossy().into_owned())
16718            .collect();
16719        assert!(
16720            !none_args.iter().any(|a| a == SUBC_ARG),
16721            "protocol:none argv must not carry --subc; got {none_args:?}"
16722        );
16723        let none_has_nonce = none
16724            .as_std()
16725            .get_envs()
16726            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16727        assert!(
16728            !none_has_nonce,
16729            "protocol:none spawn must not receive a launch nonce"
16730        );
16731        let none_has_module_id = none
16732            .as_std()
16733            .get_envs()
16734            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16735        assert!(
16736            none_has_module_id,
16737            "SUBC_MODULE_ID is inert and stays on every path"
16738        );
16739        assert!(
16740            handle.spawn_nonce(&none_spec.module_id).is_none(),
16741            "no nonce record for a process that will never present one"
16742        );
16743
16744        // Control: the subc-wire path is unchanged by the branch above.
16745        let wire_spec = spec(Vec::new());
16746        let mut wire = Command::new("/nonexistent");
16747        let wire_handoff =
16748            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16749                .expect("subc-wire spawn args apply");
16750        let wire_fd_env = wire
16751            .as_std()
16752            .get_envs()
16753            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16754            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16755        #[cfg(unix)]
16756        assert_eq!(
16757            wire_fd_env,
16758            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16759            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16760        );
16761        #[cfg(windows)]
16762        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16763        #[cfg(not(any(unix, windows)))]
16764        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16765        let wire_args: Vec<String> = wire
16766            .as_std()
16767            .get_args()
16768            .map(|a| a.to_string_lossy().into_owned())
16769            .collect();
16770        assert_eq!(
16771            wire_args,
16772            vec![
16773                SUBC_ARG.to_string(),
16774                connection_file.to_string_lossy().into_owned()
16775            ],
16776            "a subc-wire spawn still carries --subc <path>"
16777        );
16778        assert_eq!(
16779            wire.as_std()
16780                .get_envs()
16781                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16782            !cfg!(unix),
16783            "only Windows supplies the environment nonce"
16784        );
16785        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16786    }
16787
16788    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16789    /// spec tries to set it; only a swap candidate carries it.
16790    ///
16791    /// "Set it only on candidates" is not enough, because spawn applies the
16792    /// spec's env verbatim and the daemon's own environment is inherited: either
16793    /// could hand a plain restart the swap role, and a module reading it would
16794    /// warm on its long swap budget while callers wait. Asserted as an explicit
16795    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16796    /// test above gives.
16797    #[test]
16798    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16799        let role = |command: &Command| {
16800            command
16801                .as_std()
16802                .get_envs()
16803                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16804                .last()
16805                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16806        };
16807        let forged = spec(vec![(
16808            SUBC_SPAWN_ROLE_ENV.to_string(),
16809            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16810        )]);
16811
16812        let mut plain = Command::new("/nonexistent");
16813        apply_child_env(&mut plain, &forged);
16814        apply_spawn_role(&mut plain, SpawnRole::Plain);
16815        assert_eq!(
16816            role(&plain),
16817            Some(None),
16818            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16819        );
16820
16821        let mut candidate = Command::new("/nonexistent");
16822        apply_child_env(&mut candidate, &spec(Vec::new()));
16823        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16824        assert_eq!(
16825            role(&candidate),
16826            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16827        );
16828    }
16829
16830    /// Daemon-private capture retention keys never reach the child.
16831    ///
16832    /// cortexkit-log exposes retention as a Rust struct with no environment
16833    /// names, so these entries are supervisor metadata. Passing them through
16834    /// would invent a public child-process contract by accident.
16835    #[test]
16836    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16837        let mut command = Command::new("/nonexistent");
16838        apply_child_env(
16839            &mut command,
16840            &spec(vec![
16841                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16842                ("KEPT".to_string(), "yes".to_string()),
16843            ]),
16844        );
16845        let keys: Vec<String> = command
16846            .as_std()
16847            .get_envs()
16848            .filter(|(_, value)| value.is_some())
16849            .map(|(key, _)| key.to_string_lossy().into_owned())
16850            .collect();
16851        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16852        assert!(
16853            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16854            "daemon-private capture key leaked to the child: {keys:?}"
16855        );
16856    }
16857}
16858
16859#[cfg(test)]
16860mod jitter_tests {
16861    use super::jittered_health_delay;
16862    use std::{collections::HashSet, time::Duration};
16863
16864    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16865    /// that actually occur rather than invented ones.
16866    ///
16867    /// This is a SAMPLE, not a registry: the property under test is that distinct
16868    /// ids disperse, which holds for any set of distinct strings. Several entries
16869    /// are already historical (modules get renamed), and that costs nothing here --
16870    /// but it means a reader must not mistake this for the live module set, and a
16871    /// rename sweep will match it without there being anything to change.
16872    const FLEET: [&str; 14] = [
16873        "aft",
16874        "alfonso-core",
16875        "magic-context",
16876        "broca",
16877        "thalamus",
16878        "quota",
16879        "engram",
16880        "plexus",
16881        "cerebellum",
16882        "astrocyte",
16883        "synapse",
16884        "subc-mcp",
16885        "cortexkit-credentials",
16886        "subc-federation",
16887    ];
16888
16889    /// Probes must not converge after a fleet-wide restart.
16890    ///
16891    /// This is the property the jitter exists for: every module reconnects at
16892    /// once, and without dispersal all fourteen would then probe on the same
16893    /// tick forever. Nothing failed visibly when this went untested -- a
16894    /// convergent fleet still probes correctly, just in a burst, so the symptom
16895    /// is a periodic load spike that looks like whatever else is running.
16896    #[test]
16897    fn probe_delays_disperse_across_the_fleet() {
16898        let cadence = Duration::from_secs(30);
16899        let delays: HashSet<Duration> = FLEET
16900            .iter()
16901            .map(|id| jittered_health_delay(id, 0, cadence))
16902            .collect();
16903        assert_eq!(
16904            delays.len(),
16905            FLEET.len(),
16906            "every supervised module must land on its own probe offset"
16907        );
16908    }
16909
16910    /// The offset may only ever DELAY a probe, never bring it forward.
16911    ///
16912    /// A delay below the cadence would probe a module more often than
16913    /// configured, which is the opposite of what an operator asked for and
16914    /// would tighten the failure budget without anyone changing it.
16915    #[test]
16916    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16917        let cadence = Duration::from_secs(30);
16918        let span = cadence / 10;
16919        for id in FLEET {
16920            for probe_index in 0..8 {
16921                let delay = jittered_health_delay(id, probe_index, cadence);
16922                assert!(
16923                    delay >= cadence,
16924                    "{id}#{probe_index}: jitter must not shorten the cadence"
16925                );
16926                assert!(
16927                    delay < cadence + span,
16928                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16929                );
16930            }
16931        }
16932    }
16933
16934    /// A module keeps its offset across daemon restarts.
16935    ///
16936    /// The delay is derived rather than randomised precisely so a restart does
16937    /// not re-roll every module into a fresh chance of collision. A random
16938    /// source would satisfy the dispersal test above and quietly lose this.
16939    #[test]
16940    fn a_module_offset_is_stable_across_restarts() {
16941        let cadence = Duration::from_secs(30);
16942        for id in FLEET {
16943            assert_eq!(
16944                jittered_health_delay(id, 0, cadence),
16945                jittered_health_delay(id, 0, cadence),
16946                "{id}: the same module and probe index must produce the same offset"
16947            );
16948        }
16949    }
16950
16951    /// A zero cadence disables probing rather than producing a busy loop.
16952    #[test]
16953    fn zero_cadence_yields_zero_delay() {
16954        assert_eq!(
16955            jittered_health_delay("aft", 0, Duration::ZERO),
16956            Duration::ZERO
16957        );
16958    }
16959}
16960
16961#[cfg(all(test, target_os = "linux"))]
16962mod cgroup_placement_tests {
16963    use super::{
16964        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16965        SuperviseError, SupervisedChild,
16966    };
16967    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16968    use cortexkit_test_support::ScratchDir;
16969    use std::{
16970        fs, io,
16971        path::{Path, PathBuf},
16972        sync::{Arc, Mutex},
16973    };
16974    use tokio::process::Command;
16975
16976    #[tokio::test]
16977    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16978        use super::*;
16979        let dir = ScratchDir::new("unique-spawn-cgroups");
16980        let root = PathBuf::from(format!(
16981            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16982            std::process::id(),
16983            unix_ms_now()
16984        ));
16985        if let Err(error) = fs::create_dir(&root) {
16986            assert!(
16987                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16988                "required cgroup test cannot execute: {error}"
16989            );
16990            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16991            return;
16992        }
16993        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16994        let group_count = || {
16995            fs::read_dir(root.join("subc-modules"))
16996                .unwrap()
16997                .map(|entry| entry.unwrap().file_type().unwrap())
16998                .filter(|kind| kind.is_dir())
16999                .count()
17000        };
17001        let supervisor =
17002            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17003                .with_cgroup_placement(Some(placement.clone()));
17004        let runtime = supervisor.runtime_config();
17005        let mut spec = ModuleSpec {
17006            module_id: "unique-spawn".into(),
17007            program: PathBuf::from("/bin/sleep"),
17008            args: vec!["60".into()],
17009            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17010                .into_iter()
17011                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17012                .collect(),
17013            reserved: false,
17014            reserved_prefixes: vec![],
17015            protocol: ModuleProtocol::None,
17016            overlap: Default::default(),
17017            carry_containment: false,
17018        };
17019        let spawn = |spec: &ModuleSpec| {
17020            spawn_child(
17021                spec,
17022                None,
17023                None,
17024                &runtime.stderr_ring,
17025                None,
17026                &runtime.child_roster,
17027                Some(&placement),
17028            )
17029            .unwrap()
17030        };
17031        let mut live = spawn(&spec);
17032        for _ in 0..3 {
17033            // A new process can enter the old slot while retirement is pending.
17034            let next = spawn(&spec);
17035            assert_ne!(live.module_id, next.module_id);
17036            live.start_kill().unwrap();
17037            live.wait().await.unwrap();
17038            live = next;
17039            assert!(
17040                live.child.try_wait().unwrap().is_none(),
17041                "retiring the old slot must not kill the replacement"
17042            );
17043            assert_eq!(
17044                group_count(),
17045                1,
17046                "only the live spawn's cgroup should remain"
17047            );
17048        }
17049        supervisor.begin_daemon_shutdown();
17050        let reap = tokio::spawn(async move {
17051            live.wait().await.unwrap();
17052        });
17053        supervisor
17054            .end_children_for_daemon_shutdown(false, std::future::pending())
17055            .await;
17056        reap.await.unwrap();
17057        assert_eq!(group_count(), 0);
17058        // A normal exit uses the same tree-cleanup path as a killed spawn.
17059        spec.program = PathBuf::from("/bin/true");
17060        spec.args.clear();
17061        let fresh_roster = ChildRoster::default();
17062        let mut short = spawn_child(
17063            &spec,
17064            None,
17065            None,
17066            &runtime.stderr_ring,
17067            None,
17068            &fresh_roster,
17069            Some(&placement),
17070        )
17071        .unwrap();
17072        short.wait().await.unwrap();
17073        assert_eq!(group_count(), 0);
17074        spec.module_id = "_".repeat(255);
17075        let mut long_id = spawn_child(
17076            &spec,
17077            None,
17078            None,
17079            &runtime.stderr_ring,
17080            None,
17081            &fresh_roster,
17082            Some(&placement),
17083        )
17084        .unwrap();
17085        long_id.wait().await.unwrap();
17086        assert_eq!(
17087            group_count(),
17088            0,
17089            "valid long module IDs must not exceed cgroup NAME_MAX"
17090        );
17091        fs::remove_dir(root.join("subc-modules")).unwrap();
17092        fs::remove_dir(root).unwrap();
17093        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17094    }
17095
17096    #[test]
17097    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17098        let path = Path::new("/definitely-missing-subc-cgroup");
17099        let mut command = Command::new("true");
17100        let error = apply_cgroup_placement(
17101            &mut command,
17102            &ModuleSpec {
17103                module_id: "broken-cgroup".to_string(),
17104                program: PathBuf::from("true"),
17105                args: Vec::new(),
17106                env: Vec::new(),
17107                reserved: false,
17108                reserved_prefixes: Vec::new(),
17109                protocol: ModuleProtocol::Subc,
17110                overlap: Default::default(),
17111                carry_containment: false,
17112            },
17113            path,
17114        )
17115        .expect_err("a parent cgroup open failure must reject the supervised spawn");
17116        let reason = error.to_string();
17117
17118        assert!(
17119            matches!(error, SuperviseError::Cgroup { .. }),
17120            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17121        );
17122        assert!(
17123            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17124            "parent cgroup open failure must name cgroup.procs: {reason}"
17125        );
17126    }
17127
17128    #[tokio::test]
17129    async fn reaping_a_child_removes_its_empty_module_cgroup() {
17130        let root = ScratchDir::new("supervisor-reap-cgroup");
17131        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17132        let placement = subc_cgroup::prepare_at(&root)
17133            .expect("prepare scratch cgroup root")
17134            .expect("scratch root has a cgroup.procs marker");
17135        let module_id = "reaped-module";
17136        let module = placement
17137            .module_path(module_id)
17138            .expect("create scratch module cgroup");
17139        let child = Command::new("true")
17140            .env("XDG_DATA_HOME", root.path())
17141            .env("XDG_RUNTIME_DIR", root.path())
17142            .env("XDG_CONFIG_HOME", root.path())
17143            .spawn()
17144            .expect("spawn short-lived child");
17145        let pid = child.id().expect("spawned child has pid");
17146        let mut child = SupervisedChild {
17147            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17148            child: crate::supervise::containment::ProcessChild::plain(child, pid),
17149            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17150            protocol: ModuleProtocol::Subc,
17151            module_id: module_id.to_string(),
17152            cgroup_placement: Some(placement),
17153            stdout_pump: None,
17154            stderr_pump: None,
17155            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17156            spawned_at: tokio::time::Instant::now(),
17157            registration_deadline: None,
17158            registration_watch: None,
17159            spawned_at_ms: 0,
17160            spawned_from: PathBuf::from("true"),
17161            launch_spec_digest: LaunchSpecDigest([0; 32]),
17162            spawned_file_identity: None,
17163            process_start_time: None,
17164            process_identity: None,
17165            pid,
17166            roster_guard: None,
17167            #[cfg(target_os = "macos")]
17168            privacy_exec: None,
17169            spawn_failure: None,
17170        };
17171
17172        child.wait().await.expect("reap short-lived child");
17173
17174        assert!(
17175            !module.exists(),
17176            "waiting for a non-carried child must remove its empty cgroup before returning"
17177        );
17178        child.finish_exit(false).await.unwrap();
17179    }
17180
17181    #[test]
17182    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17183        let root = ScratchDir::new("supervisor-non-empty-cgroup");
17184        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17185        let placement = subc_cgroup::prepare_at(&root)
17186            .expect("prepare scratch cgroup root")
17187            .expect("scratch root has a cgroup.procs marker");
17188        let module = placement
17189            .module_path("surviving-module")
17190            .expect("create scratch module cgroup");
17191        fs::write(module.join("surviving-process"), b"still present")
17192            .expect("make scratch cgroup non-empty");
17193        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17194
17195        remove_module_cgroup(&placement, "surviving-module");
17196
17197        let logs = crate::router::test_log::captured_logs(&logs);
17198        assert!(
17199            module.exists(),
17200            "failed removal must leave the cgroup intact"
17201        );
17202        assert!(
17203            logs.contains("could not remove module cgroup after process exit; continuing teardown")
17204                && logs.contains("surviving-module"),
17205            "best-effort removal must report the failure without returning it: {logs}"
17206        );
17207    }
17208
17209    #[test]
17210    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17211        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17212        let reason = SuperviseError::Spawn {
17213            program: PathBuf::from("/bin/true"),
17214            source: io::Error::from_raw_os_error(13),
17215            cgroup_path: Some(cgroup_path.clone()),
17216        }
17217        .to_string();
17218
17219        assert!(
17220            reason.contains(&cgroup_path.display().to_string()),
17221            "a pre_exec spawn failure must name the cgroup path: {reason}"
17222        );
17223    }
17224}
17225
17226#[cfg(test)]
17227mod spawn_subscriber_lag_tests {
17228    use super::*;
17229
17230    /// A subscriber whose connection stops draining is dropped once its frame
17231    /// channel fills. The client must learn that from a terminal Error frame
17232    /// after the frames already queued for it, not from a stream that simply
17233    /// goes quiet.
17234    #[tokio::test]
17235    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17236        let feed = SpawnEventFeed::default();
17237        feed.configure_incarnation("lag-incarnation".to_string());
17238        // A one-slot connection queue that nobody reads until the emits are
17239        // done: the forwarder parks on it and the subscriber channel fills.
17240        let (tx, mut rx) = mpsc::channel(1);
17241        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17242            .expect("subscribe");
17243        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17244        for index in 0..emitted {
17245            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17246            // Let the forwarder take what it can so the fill point is the
17247            // subscriber channel, not a scheduling accident.
17248            tokio::task::yield_now().await;
17249        }
17250        assert_eq!(
17251            feed.subscriber_count(),
17252            0,
17253            "the lagged subscriber must be removed"
17254        );
17255
17256        let mut data = Vec::new();
17257        let mut last = None;
17258        loop {
17259            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17260                .await
17261                .expect("the forwarder must finish once the subscriber is dropped");
17262            let Some(outbound) = next else { break };
17263            let frame = outbound.frame;
17264            if frame.header.ty == FrameType::StreamData {
17265                assert!(last.is_none(), "no data may follow the terminal frame");
17266                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17267                data.push(event.cursor.seq);
17268            } else {
17269                assert!(last.is_none(), "exactly one terminal frame");
17270                last = Some(frame);
17271            }
17272        }
17273        assert!(!data.is_empty(), "queued frames drain before the terminal");
17274        for pair in data.windows(2) {
17275            assert_eq!(
17276                pair[1],
17277                pair[0] + 1,
17278                "queued frames arrive dense and in order"
17279            );
17280        }
17281        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17282        assert_eq!(terminal.header.ty, FrameType::Error);
17283        assert_eq!(terminal.header.corr, 7);
17284        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17285        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17286        let detail = body.detail.expect("lagged error carries detail");
17287        assert_eq!(
17288            detail["first_undelivered_cursor"]["seq"],
17289            data.last().unwrap() + 1,
17290            "the named cursor is the first event the subscriber did not receive"
17291        );
17292        assert_eq!(
17293            detail["first_undelivered_cursor"]["daemon_incarnation"],
17294            "lag-incarnation"
17295        );
17296    }
17297}
17298
17299#[cfg(test)]
17300mod terminal_history_read_concurrency_tests {
17301    use super::*;
17302    use crate::terminal_journal::read_pause;
17303    use cortexkit_test_support::ScratchDir;
17304    use std::sync::mpsc as std_mpsc;
17305
17306    fn journaled_ring(
17307        journal: &Arc<crate::terminal_journal::TerminalJournal>,
17308    ) -> Arc<Mutex<TerminalRing>> {
17309        Arc::new(Mutex::new(
17310            TerminalRing::new(TerminalRingConfig::default(), 1)
17311                .with_journal(Some(Arc::clone(journal))),
17312        ))
17313    }
17314
17315    fn crash(at_ms: u64) -> ExitReport {
17316        ExitReport {
17317            kind: ExitKind::Crash,
17318            code: Some(1),
17319            signal: None,
17320            at_ms,
17321        }
17322    }
17323
17324    /// Record an exit on another thread and report whether it finished within
17325    /// `bound`. The recorder thread is left running if it did not.
17326    fn record_within(
17327        module_id: &'static str,
17328        ring: &Arc<Mutex<TerminalRing>>,
17329        at_ms: u64,
17330        bound: Duration,
17331    ) -> bool {
17332        let ring = Arc::clone(ring);
17333        let (done, done_rx) = std_mpsc::channel();
17334        std::thread::spawn(move || {
17335            record_terminal(
17336                module_id,
17337                &ring,
17338                &SpawnEventFeed::default(),
17339                &crash(at_ms),
17340                TerminalDisposition::Restarting,
17341            );
17342            let _ = done.send(());
17343        });
17344        done_rx.recv_timeout(bound).is_ok()
17345    }
17346
17347    /// A history read in progress must not hold the journal writer (which every
17348    /// module's exit recording needs) or the module's own ring. Exits recorded
17349    /// while the read is paused complete promptly; the paused read answers as of
17350    /// the moment it started, and the next read has each exit exactly once.
17351    #[test]
17352    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17353        let dir = ScratchDir::new("terminal-history-concurrent-read");
17354        let path = dir.join("terminals.jsonl");
17355        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17356            path.clone(),
17357            "daemon".into(),
17358        ));
17359        let reader_ring = journaled_ring(&journal);
17360        let other_ring = journaled_ring(&journal);
17361        assert!(record_within(
17362            "reader-module",
17363            &reader_ring,
17364            10,
17365            Duration::from_secs(5)
17366        ));
17367
17368        let (started, release) = read_pause::install(&path);
17369        let reading = {
17370            let ring = Arc::clone(&reader_ring);
17371            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17372        };
17373        started
17374            .recv_timeout(Duration::from_secs(5))
17375            .expect("the history read reached its pause");
17376
17377        let bound = Duration::from_secs(1);
17378        assert!(
17379            record_within("other-module", &other_ring, 20, bound),
17380            "another module's exit waited on a history read (journal writer held)"
17381        );
17382        assert!(
17383            record_within("reader-module", &reader_ring, 30, bound),
17384            "the read module's own exit waited on its history read (ring held)"
17385        );
17386
17387        drop(release);
17388        let paused = reading.join().unwrap();
17389        assert_eq!(
17390            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17391            vec![10],
17392            "an exit recorded after the read began lands in neither half of it"
17393        );
17394        assert_eq!(paused.journal_skipped_lines, 0);
17395        assert_eq!(paused.journal_read_errors, 0);
17396
17397        let after = durable_terminal_history_of(&reader_ring, "reader-module");
17398        assert_eq!(
17399            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17400            vec![10, 30],
17401            "the next read merges ring and journal with no duplicate"
17402        );
17403        assert_eq!(after.journal_skipped_lines, 0);
17404    }
17405}
17406
17407/// What a restart does with the exited process's output readers. These drive
17408/// the same settle functions the supervisor calls, with a reader the test
17409/// holds, so a reader that has not been scheduled by the bound is a controlled
17410/// input rather than something only a loaded machine produces.
17411#[cfg(test)]
17412mod stderr_settle_tests {
17413    use std::{
17414        future::Future,
17415        io,
17416        pin::Pin,
17417        sync::{Arc, Mutex},
17418        task::{Context, Poll},
17419        time::Duration,
17420    };
17421
17422    use tokio::{
17423        io::{AsyncRead, ReadBuf},
17424        sync::oneshot,
17425        time::Instant,
17426    };
17427
17428    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17429    use crate::stderr_tail::{
17430        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17431        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17432    };
17433
17434    const BOUND: Duration = Duration::from_millis(250);
17435
17436    /// Yields `before`, then stays pending until the gate is released, then
17437    /// yields `after` and reaches EOF. The bytes after the gate were written
17438    /// by a process that has already exited; only the reader is behind.
17439    struct HeldReader {
17440        before: Option<Vec<u8>>,
17441        gate: Option<oneshot::Receiver<()>>,
17442        after: io::Cursor<Vec<u8>>,
17443    }
17444
17445    impl AsyncRead for HeldReader {
17446        fn poll_read(
17447            mut self: Pin<&mut Self>,
17448            cx: &mut Context<'_>,
17449            buf: &mut ReadBuf<'_>,
17450        ) -> Poll<io::Result<()>> {
17451            if let Some(bytes) = self.before.take() {
17452                buf.put_slice(&bytes);
17453                return Poll::Ready(Ok(()));
17454            }
17455            if let Some(gate) = self.gate.as_mut() {
17456                match Pin::new(gate).poll(cx) {
17457                    Poll::Pending => return Poll::Pending,
17458                    Poll::Ready(_) => self.gate = None,
17459                }
17460            }
17461            Pin::new(&mut self.after).poll_read(cx, buf)
17462        }
17463    }
17464
17465    struct DiscardSink;
17466
17467    impl OutputSink for DiscardSink {
17468        fn write_line(&mut self, _line: &[u8]) {}
17469    }
17470
17471    fn line(text: &str) -> TailEntry {
17472        TailEntry::Line {
17473            text: text.to_string(),
17474            truncated: false,
17475            at_ms: None,
17476        }
17477    }
17478
17479    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17480        ring.lock().unwrap()
17481    }
17482
17483    /// Start a reader for a new process generation that delivers `before`
17484    /// immediately and `after` only once the returned sender fires (or is
17485    /// dropped).
17486    fn held_pump(
17487        ring: &Arc<Mutex<StderrRing>>,
17488        before: &str,
17489        after: &str,
17490    ) -> (StderrPump, oneshot::Sender<()>) {
17491        let generation = lock(ring).begin_process();
17492        let (release, gate) = oneshot::channel();
17493        let reader = HeldReader {
17494            before: Some(before.as_bytes().to_vec()),
17495            gate: Some(gate),
17496            after: io::Cursor::new(after.as_bytes().to_vec()),
17497        };
17498        let task = tokio::spawn(pump_stderr_to(
17499            reader,
17500            Arc::clone(ring),
17501            generation,
17502            DiscardSink,
17503        ));
17504        (StderrPump { task, generation }, release)
17505    }
17506
17507    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17508        for _ in 0..1000 {
17509            if done(&lock(ring)) {
17510                return;
17511            }
17512            tokio::time::sleep(Duration::from_millis(1)).await;
17513        }
17514        panic!(
17515            "ring never reached the expected state: {:?}",
17516            lock(ring).snapshot(None, None)
17517        );
17518    }
17519
17520    #[tokio::test(start_paused = true)]
17521    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17522        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17523        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17524
17525        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17526        let before_release = lock(&ring).snapshot(None, None);
17527        assert!(
17528            matches!(before_release.capture, CaptureState::Incomplete { .. }),
17529            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17530        );
17531
17532        // The restart: the next process starts and writes before the old
17533        // reader catches up.
17534        let next = lock(&ring).begin_process();
17535        lock(&ring).push_line_from(next, "next process booting");
17536        release.send(()).unwrap();
17537        wait_until(&ring, |ring| {
17538            ring.snapshot(None, None).capture == CaptureState::Captured
17539        })
17540        .await;
17541
17542        assert_eq!(
17543            untimed(lock(&ring).snapshot(None, None).entries),
17544            vec![
17545                line("booting"),
17546                line("config error: missing storage"),
17547                TailEntry::ProcessStart,
17548                line("next process booting"),
17549            ],
17550            "the crash's last line must survive a slow reader and stay in the crashed process's section"
17551        );
17552    }
17553
17554    #[tokio::test(start_paused = true)]
17555    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17556    ) {
17557        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17558        // `_held` is never fired: a descendant keeps the pipe open for the
17559        // whole test.
17560        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17561
17562        let started = Instant::now();
17563        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17564        assert_eq!(
17565            started.elapsed(),
17566            BOUND,
17567            "the restart must wait exactly the bound for a pipe that stays open, no longer"
17568        );
17569
17570        let next = lock(&ring).begin_process();
17571        lock(&ring).push_line_from(next, "next process booting");
17572        tokio::time::sleep(Duration::from_secs(60)).await;
17573
17574        let snapshot = lock(&ring).snapshot(None, None);
17575        match &snapshot.capture {
17576            CaptureState::Incomplete { reason } => assert!(
17577                reason.contains("had not reached EOF") && reason.contains("250ms"),
17578                "the reason must say what is missing and after how long: {reason}"
17579            ),
17580            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17581        }
17582        assert_eq!(
17583            untimed(snapshot.entries),
17584            vec![
17585                line("parent exiting"),
17586                TailEntry::ProcessStart,
17587                line("next process booting"),
17588            ]
17589        );
17590    }
17591
17592    #[tokio::test(start_paused = true)]
17593    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17594        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17595        let (pump, release) = held_pump(&ring, "one\n", "two\n");
17596        release.send(()).unwrap();
17597
17598        settle_stderr_pump("clean", &ring, pump, BOUND).await;
17599
17600        let snapshot = lock(&ring).snapshot(None, None);
17601        assert_eq!(snapshot.capture, CaptureState::Captured);
17602        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17603    }
17604
17605    #[tokio::test(start_paused = true)]
17606    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17607        const BURST: usize = 150;
17608        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17609        let path = capture.join("two-pipe.stderr.log");
17610        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17611        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17612        let generation = lock(&ring).begin_process();
17613        let stdout = (0..BURST)
17614            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17615            .collect::<String>();
17616        let stderr = (0..BURST)
17617            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17618            .collect::<String>();
17619        // Model a child that has written both bursts and exited. Hold the stdout
17620        // reader after one line and a partial prefix, as if pipe readiness or
17621        // capture-file writes had delayed it. The final line also needs EOF to
17622        // flush: neither complete nor partial buffered output may be discarded.
17623        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17624        let (release, gate) = oneshot::channel();
17625        let reader = HeldReader {
17626            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17627            gate: Some(gate),
17628            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17629        };
17630        let (finished, completion) = oneshot::channel();
17631        let stdout_sink = sink.clone();
17632        let pump = tokio::spawn(async move {
17633            pump_stdout_to(reader, stdout_sink).await;
17634            finished.send(()).unwrap();
17635        });
17636        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17637
17638        let started = Instant::now();
17639        settle_stdout_pump("two-pipe", pump, BOUND).await;
17640        assert_eq!(
17641            started.elapsed(),
17642            BOUND,
17643            "a late reader must not hold up restart"
17644        );
17645        assert_eq!(
17646            std::fs::read_to_string(&path).unwrap().lines().count(),
17647            BURST + 1
17648        );
17649
17650        release
17651            .send(())
17652            .expect("stdout reader must survive the exit drain bound");
17653        tokio::time::timeout(Duration::from_secs(1), completion)
17654            .await
17655            .expect("released stdout reader must reach EOF")
17656            .expect("stdout capture must finish at EOF, not be cancelled");
17657        let contents = std::fs::read_to_string(&path).unwrap();
17658        let mut lines = contents
17659            .lines()
17660            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17661            .collect::<Vec<_>>();
17662        let mut expected = (0..BURST)
17663            .flat_map(|i| {
17664                [
17665                    format!("out-{i:04}-{}", "o".repeat(64)),
17666                    format!("err-{i:04}-{}", "e".repeat(64)),
17667                ]
17668            })
17669            .collect::<Vec<_>>();
17670        lines.sort_unstable();
17671        expected.sort_unstable();
17672        assert_eq!(
17673            lines, expected,
17674            "both pipes must retain every complete line exactly once"
17675        );
17676    }
17677}
17678
17679/// Containment of a module's process tree (issue #109).
17680///
17681/// The behaviour these defend against is a module helper surviving its module:
17682/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
17683/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
17684/// compounds it.
17685///
17686/// They run against the SUPERVISOR rather than the job-object crate because the
17687/// claim is about teardown: a crate-level test proves a job can reap a tree, not
17688/// that the daemon's drain path reaches it.
17689///
17690/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
17691/// lane there is a separate containment path with its own tests.
17692#[cfg(all(test, windows))]
17693mod job_containment_tests {
17694    use super::*;
17695    use cortexkit_test_support::ScratchDir;
17696    use std::{
17697        path::{Path, PathBuf},
17698        sync::{Arc, Mutex},
17699        time::{Duration, Instant},
17700    };
17701
17702    /// The stub, expected beside this test executable.
17703    ///
17704    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17705    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17706    /// failure then reads as a broken test rather than an unbuilt dependency.
17707    fn stub_path() -> PathBuf {
17708        let mut path = std::env::current_exe().expect("current_exe available in tests");
17709        path.pop();
17710        path.pop();
17711        path.push("fake-aft-stub.exe");
17712        assert!(
17713            path.exists(),
17714            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17715             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17716            path.display()
17717        );
17718        path
17719    }
17720
17721    /// Poll for the grandchild pid the stub records, and parse it.
17722    fn read_grandchild_pid(path: &Path) -> u32 {
17723        let deadline = Instant::now() + Duration::from_secs(10);
17724        loop {
17725            if let Ok(contents) = std::fs::read_to_string(path) {
17726                if let Ok(pid) = contents.trim().parse() {
17727                    return pid;
17728                }
17729            }
17730            assert!(
17731                Instant::now() < deadline,
17732                "the stub never recorded a grandchild pid at {}",
17733                path.display()
17734            );
17735            std::thread::sleep(Duration::from_millis(10));
17736        }
17737    }
17738
17739    /// Everything one fixture run needs, so the two tests below differ in exactly
17740    /// one place: whether the child is contained.
17741    struct Fixture {
17742        _dir: ScratchDir,
17743        module_id: String,
17744        grandchild: u32,
17745        child: Option<SupervisedChild>,
17746        registry: Arc<Registry>,
17747        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17748        terminal_ring: Arc<Mutex<TerminalRing>>,
17749        spawn_events: SpawnEventFeed,
17750    }
17751
17752    fn fixture(label: &str, module_id: &str) -> Fixture {
17753        let dir = ScratchDir::new(label);
17754        let pid_file = dir.join("grandchild.pid");
17755        let supervisor = Supervisor::new_for_test(
17756            Arc::new(Registry::default()),
17757            RestartPolicy::new(3, Duration::ZERO),
17758        );
17759        let runtime = supervisor.runtime_config();
17760        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17761        let spec = ModuleSpec {
17762            module_id: module_id.to_string(),
17763            program: stub_path(),
17764            // Zero args deliberately: a `--subc` argument would make the stub dial
17765            // a daemon that is not there, and the failure would land in the same
17766            // stderr ring this fixture exists to keep quiet.
17767            args: Vec::new(),
17768            env: vec![
17769                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17770                (
17771                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17772                    pid_file.display().to_string(),
17773                ),
17774            ],
17775            reserved: false,
17776            reserved_prefixes: Vec::new(),
17777            protocol: ModuleProtocol::Subc,
17778            overlap: Default::default(),
17779            carry_containment: false,
17780        };
17781        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17782            .expect("spawn the supervised fixture");
17783        let grandchild = read_grandchild_pid(&pid_file);
17784        Fixture {
17785            _dir: dir,
17786            module_id: module_id.to_string(),
17787            grandchild,
17788            child: Some(child),
17789            registry: Arc::new(Registry::default()),
17790            snapshot,
17791            terminal_ring: Arc::clone(&runtime.terminal_ring),
17792            spawn_events: SpawnEventFeed::default(),
17793        }
17794    }
17795
17796    impl Fixture {
17797        /// Drain through the supervisor's own teardown path.
17798        async fn drain(&mut self) {
17799            let child = self
17800                .child
17801                .take()
17802                .expect("the fixture child is still present");
17803            drain_child_to_state(
17804                &self.module_id,
17805                ModuleProtocol::Subc,
17806                // No forwarding table in this fixture, so nothing reaches the
17807                // child over a connection.
17808                StopNotice::NotSent,
17809                &self.registry,
17810                None,
17811                &self.snapshot,
17812                &self.terminal_ring,
17813                &self.spawn_events,
17814                child,
17815                Duration::from_millis(500),
17816                ModuleState::Stopped,
17817                Some(false),
17818                &FailureContext::operator(),
17819            )
17820            .await
17821            .expect("drain the supervised fixture");
17822        }
17823    }
17824
17825    /// Teardown reaps the grandchild, not merely the direct child.
17826    ///
17827    /// This is the assertion the change exists for. Before containment the
17828    /// grandchild survived: it is a separate process, and `start_kill` is
17829    /// `TerminateProcess` scoped to one pid.
17830    #[tokio::test]
17831    async fn teardown_reaps_the_grandchild() {
17832        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17833        let grandchild = fixture.grandchild;
17834
17835        assert!(
17836            subc_jobobject::process_exists(grandchild),
17837            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17838        );
17839
17840        fixture.drain().await;
17841
17842        assert!(
17843            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17844            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17845        );
17846    }
17847
17848    /// The mutation control: with containment withheld, the grandchild survives
17849    /// the same kill.
17850    ///
17851    /// This is the defect reproduction from #109 — a direct-child kill reaches
17852    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17853    /// supervisor because `spawn_and_mark_running` now always contains on
17854    /// Windows, which is the point: there is no longer a path that spawns
17855    /// uncontained, so the control has to construct one.
17856    ///
17857    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17858    /// grandchild ever dies here, that test is passing for a reason unrelated to
17859    /// the job object and the containment claim is unproven.
17860    #[test]
17861    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17862        let dir = ScratchDir::new("teardown-uncontained");
17863        let pid_file = dir.join("grandchild.pid");
17864        let mut child = std::process::Command::new(stub_path())
17865            .env("FAKE_AFT_NEVER_CONNECT", "1")
17866            .env(
17867                "FAKE_AFT_GRANDCHILD_PID_FILE",
17868                pid_file.display().to_string(),
17869            )
17870            .stdin(std::process::Stdio::null())
17871            .stdout(std::process::Stdio::null())
17872            .stderr(std::process::Stdio::null())
17873            .spawn()
17874            .expect("spawn the uncontained fixture");
17875        let grandchild = read_grandchild_pid(&pid_file);
17876
17877        // Exactly what the pre-fix teardown did: kill the direct child.
17878        child.kill().expect("kill the direct child");
17879        let _ = child.wait();
17880
17881        assert!(
17882            subc_jobobject::process_exists(grandchild),
17883            "grandchild {grandchild} died with the direct child, so this control no longer \
17884             distinguishes contained from uncontained teardown and the regression test is \
17885             passing vacuously"
17886        );
17887
17888        // The orphan this control demonstrates is the leak the fix prevents, so
17889        // the control must not leave one behind.
17890        kill_tree(grandchild);
17891    }
17892
17893    /// Crash durability: closing the containment handle reaps the tree with no
17894    /// teardown code running at all.
17895    ///
17896    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17897    /// call anything — and it is why containment is a kernel property of the
17898    /// handle rather than a step in the drain. Discovered by getting the
17899    /// mutation control wrong: clearing `job` to "disable" containment instead
17900    /// killed the tree, which is the guarantee, not a mistake.
17901    #[tokio::test]
17902    async fn dropping_containment_reaps_the_grandchild() {
17903        let mut fixture = fixture("drop-containment", "tree-drop");
17904        let grandchild = fixture.grandchild;
17905
17906        assert!(subc_jobobject::process_exists(grandchild));
17907
17908        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17909        fixture.child.as_mut().expect("child present").job = None;
17910
17911        assert!(
17912            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17913            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17914             crash would leave the tree behind"
17915        );
17916    }
17917
17918    /// Kill a pid and its tree, then confirm it is gone.
17919    fn kill_tree(pid: u32) {
17920        let _ = std::process::Command::new("taskkill.exe")
17921            .args(["/PID", &pid.to_string(), "/T", "/F"])
17922            .stdin(std::process::Stdio::null())
17923            .stdout(std::process::Stdio::null())
17924            .stderr(std::process::Stdio::null())
17925            .status();
17926        assert!(
17927            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17928            "could not clean up grandchild {pid}"
17929        );
17930    }
17931}
17932
17933#[cfg(test)]
17934mod privacy_trampoline_configuration_tests {
17935    #[tokio::test]
17936    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17937    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17938        #[cfg(target_os = "macos")]
17939        {
17940            let supervisor = super::Supervisor::new(
17941                std::sync::Arc::new(crate::Registry::default()),
17942                super::RestartPolicy::default(),
17943            );
17944            let error = supervisor.spawn(spec()).unwrap_err();
17945            assert!(
17946                error
17947                    .to_string()
17948                    .contains("no privacy trampoline configured"),
17949                "{error}"
17950            );
17951        }
17952    }
17953
17954    #[tokio::test]
17955    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17956    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17957        #[cfg(target_os = "macos")]
17958        {
17959            let supervisor = super::Supervisor::new(
17960                std::sync::Arc::new(crate::Registry::default()),
17961                super::RestartPolicy::default(),
17962            )
17963            .with_privacy_trampoline(std::env::current_exe().unwrap());
17964            let error = supervisor.spawn(spec()).unwrap_err();
17965            assert!(
17966                error
17967                    .to_string()
17968                    .contains("binary does not implement the privacy trampoline protocol"),
17969                "{error}"
17970            );
17971        }
17972    }
17973
17974    #[cfg(target_os = "macos")]
17975    fn spec() -> super::ModuleSpec {
17976        super::ModuleSpec {
17977            module_id: "privacy-configuration".into(),
17978            program: "/bin/sleep".into(),
17979            args: vec!["30".into()],
17980            env: vec![],
17981            reserved: false,
17982            reserved_prefixes: vec![],
17983            protocol: subc_control::ModuleProtocol::None,
17984            overlap: super::ModuleOverlap::Exclusive,
17985            carry_containment: false,
17986        }
17987    }
17988}
17989
17990#[cfg(test)]
17991mod privacy_exec_boundary_tests {
17992    #[cfg(target_os = "macos")]
17993    use super::*;
17994    #[cfg(target_os = "macos")]
17995    use std::{
17996        io::{Read, Write},
17997        net::{TcpListener, TcpStream},
17998    };
17999
18000    /// Unit-test-only pause at the actual early image read, not at a later
18001    /// status read. Production supervisors never inspect this environment key.
18002    #[cfg(target_os = "macos")]
18003    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18004        if let Some((_, path)) = spec
18005            .env
18006            .iter()
18007            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18008        {
18009            let mut barrier = TcpStream::connect(path).unwrap();
18010            barrier
18011                .set_read_timeout(Some(Duration::from_secs(30)))
18012                .unwrap();
18013            barrier.write_all(&pid.to_ne_bytes()).unwrap();
18014            let mut release = [0];
18015            barrier.read_exact(&mut release).unwrap();
18016            assert_eq!(&release, b"X");
18017        }
18018    }
18019
18020    #[cfg(target_os = "macos")]
18021    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18022        ModuleSpec {
18023            module_id: "privacy-boundary".into(),
18024            program: program.into(),
18025            args: args.iter().map(|arg| (*arg).into()).collect(),
18026            env: vec![],
18027            reserved: false,
18028            reserved_prefixes: vec![],
18029            protocol: ModuleProtocol::None,
18030            overlap: ModuleOverlap::Exclusive,
18031            carry_containment: false,
18032        }
18033    }
18034
18035    #[cfg(target_os = "macos")]
18036    async fn accept(listener: TcpListener) -> TcpStream {
18037        // Socket readiness, not elapsed time, establishes both pause points.
18038        let listener = tokio::net::TcpListener::from_std({
18039            listener.set_nonblocking(true).unwrap();
18040            listener
18041        })
18042        .unwrap();
18043        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18044            .await
18045            .unwrap()
18046            .unwrap();
18047        let stream = stream.into_std().unwrap();
18048        stream.set_nonblocking(false).unwrap();
18049        stream
18050            .set_read_timeout(Some(Duration::from_secs(30)))
18051            .unwrap();
18052        stream
18053    }
18054
18055    #[tokio::test]
18056    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18057    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18058        #[cfg(target_os = "macos")]
18059        {
18060            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18061            // Loopback sockets also work when the replay adapter's TMPDIR is
18062            // longer than Darwin's Unix-domain socket path limit.
18063            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18064            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18065            let record = root.join("live-children.json");
18066            let supervisor =
18067                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18068                    .with_live_children_record(&record);
18069            let runtime = supervisor.runtime_config();
18070            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18071            let mut spec = spec("/bin/sleep", &["30"]);
18072            spec.env = vec![
18073                (
18074                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18075                    exec_listener.local_addr().unwrap().to_string(),
18076                ),
18077                (
18078                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18079                    sample_listener.local_addr().unwrap().to_string(),
18080                ),
18081            ];
18082            let spawn = tokio::task::spawn_blocking(move || {
18083                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18084            });
18085            let mut sample = accept(sample_listener).await;
18086            let mut pid = [0; 4];
18087            sample.read_exact(&mut pid).unwrap();
18088            let pid = u32::from_ne_bytes(pid);
18089            let mut exec = accept(exec_listener).await;
18090            let mut ready = [0];
18091            exec.read_exact(&mut ready).unwrap();
18092            assert_eq!(&ready, b"R");
18093            // The early read is guaranteed to see a real, non-null trampoline
18094            // image: the fixture has reached its barrier and cannot exec yet.
18095            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18096            assert_eq!(
18097                observe_spawned_image(pid).unwrap().executable,
18098                Some(trampoline)
18099            );
18100            sample.write_all(b"X").unwrap();
18101            let mut child = spawn.await.unwrap();
18102            let early = crate::live_children::read_record(&record).unwrap();
18103            assert_eq!(early.len(), 1);
18104            assert_eq!(early[0].pid, pid);
18105            assert_eq!(
18106                early[0].executable, None,
18107                "unconfirmed trampoline image entered the roster"
18108            );
18109            assert!(child.report_ready.get().is_none());
18110            // The barrier's duration is unrelated to the production five-second
18111            // exec budget. Start the test's confirmation budget upon release.
18112            child.privacy_exec.as_mut().unwrap().deadline =
18113                tokio::time::Instant::now() + Duration::from_secs(30);
18114            exec.write_all(b"X").unwrap();
18115            child.confirm_privacy_exec().await;
18116            assert_eq!(child.spawn_failure, None);
18117            assert!(child.report_ready.get().is_some());
18118            let confirmed = crate::live_children::read_record(&record).unwrap();
18119            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18120            assert_ne!(module, trampoline);
18121            assert_eq!(confirmed[0].executable, Some(module.into()));
18122            child.start_kill().unwrap();
18123            child.wait().await.unwrap();
18124            child.release_roster();
18125        }
18126    }
18127
18128    #[tokio::test]
18129    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18130    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18131        #[cfg(target_os = "macos")]
18132        {
18133            let registry = Arc::new(Registry::default());
18134            let policy = RestartPolicy::new(0, Duration::ZERO);
18135            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
18136            let runtime = supervisor.runtime_config();
18137            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18138            let spec = spec("/bin/sh", &["-c", "exit 121"]);
18139            // Drive spawn and confirmation separately instead of starting the
18140            // monitor. WNOWAIT observes a real exit without consuming its status,
18141            // so confirmation's first try_wait must take the already-exited arm.
18142            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18143            let pid = child.pid;
18144            tokio::task::spawn_blocking(move || {
18145                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18146            })
18147            .await
18148            .unwrap()
18149            .unwrap();
18150            child.privacy_exec.as_mut().unwrap().deadline =
18151                tokio::time::Instant::now() + Duration::from_secs(30);
18152            let status = child.wait().await.unwrap();
18153            assert_eq!(status.code(), Some(121));
18154            assert!(child.privacy_exec.is_none());
18155            assert!(
18156                child.report_ready.get().is_none(),
18157                "an exited module must not publish a live pid"
18158            );
18159            let report = classify_reaped_child_exit(&snapshot, &child, &status);
18160            on_child_exit(
18161                &spec,
18162                policy,
18163                &registry,
18164                &snapshot,
18165                &runtime.terminal_ring,
18166                &runtime.spawn_events,
18167                &runtime.child_roster,
18168                report,
18169                true,
18170            )
18171            .await;
18172            let state = lock_snapshot(&snapshot).unwrap();
18173            assert_eq!(state.state, ModuleState::Failed);
18174            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18175            assert_eq!(state.reported_pid(), None);
18176            drop(state);
18177            let history = runtime.terminal_ring.lock().unwrap().snapshot();
18178            assert_eq!(history.entries.len(), 1);
18179            let terminal = &history.entries[0];
18180            assert_eq!(terminal.exit_code, Some(121));
18181            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18182            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18183            assert_eq!(
18184                terminal.disposition_detail.as_deref(),
18185                Some(policy.budget_exhausted_detail().as_str()),
18186                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18187            );
18188            assert_eq!(child.spawn_failure, None);
18189            child.release_roster();
18190        }
18191    }
18192}
18193
18194/// The daemon's real spawn path hands a subc-wire child its launch nonce on
18195/// descriptor 3, without an environment copy. The shell records the nonce
18196/// and its environment after exec so these tests observe the real handover.
18197#[cfg(all(test, unix))]
18198mod launch_nonce_descriptor_tests {
18199    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18200    use crate::stderr_tail::{StderrRing, StderrTailConfig};
18201    use cortexkit_test_support::ScratchDir;
18202    use std::{
18203        path::PathBuf,
18204        sync::{Arc, Mutex},
18205        time::{Duration, Instant},
18206    };
18207
18208    async fn probe(role: super::SpawnRole) {
18209        let scratch = ScratchDir::new("launch-nonce-descriptor");
18210        let fd_copy = scratch.join("from-descriptor");
18211        let env_copy = scratch.join("environment");
18212        let script = format!(
18213            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18214            fd = fd_copy.display(), env = env_copy.display(),
18215        );
18216        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18217        let spec = ModuleSpec {
18218            module_id: "nonce-descriptor-probe".to_string(),
18219            program: PathBuf::from("/bin/sh"),
18220            args: vec!["-c".to_string(), script],
18221            env: vec![
18222                xdg("XDG_DATA_HOME"),
18223                xdg("XDG_RUNTIME_DIR"),
18224                xdg("XDG_CONFIG_HOME"),
18225                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18226            ],
18227            reserved: true,
18228            reserved_prefixes: Vec::new(),
18229            protocol: ModuleProtocol::Subc,
18230            overlap: Default::default(),
18231            carry_containment: false,
18232        };
18233        let handle = SupervisorHandle::new();
18234        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18235        let roster = ChildRoster::default();
18236        #[cfg(target_os = "macos")]
18237        {
18238            let path = super::test_privacy_trampoline();
18239            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18240        }
18241        let child = super::spawn_child_in_slot(
18242            &spec,
18243            None,
18244            Some(&handle),
18245            &ring,
18246            None,
18247            &roster,
18248            #[cfg(target_os = "linux")]
18249            None,
18250            role,
18251            matches!(role, super::SpawnRole::SwapCandidate),
18252        )
18253        .expect("spawn probe");
18254        let deadline = Instant::now() + Duration::from_secs(10);
18255        while !(fd_copy.exists() && env_copy.exists()) {
18256            assert!(Instant::now() < deadline, "probe never wrote its copies");
18257            tokio::time::sleep(Duration::from_millis(20)).await;
18258        }
18259        let nonce = std::fs::read_to_string(fd_copy).unwrap();
18260        assert!(!nonce.is_empty());
18261        let environment = std::fs::read_to_string(env_copy).unwrap();
18262        assert!(environment
18263            .lines()
18264            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18265        let copy = environment
18266            .lines()
18267            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18268        assert_eq!(
18269            copy, None,
18270            "Unix children must never receive the environment nonce"
18271        );
18272        if matches!(role, super::SpawnRole::Plain) {
18273            assert_eq!(
18274                handle.spawn_nonce(&spec.module_id).as_deref(),
18275                Some(nonce.as_str())
18276            );
18277        }
18278        drop(child);
18279    }
18280
18281    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18282    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18283        probe(super::SpawnRole::Plain).await;
18284    }
18285
18286    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18287    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18288        probe(super::SpawnRole::SwapCandidate).await;
18289    }
18290}
18291
18292#[cfg(all(test, target_os = "linux"))]
18293#[path = "supervise/cgroup_reap_tests.rs"]
18294mod cgroup_reap_tests;
18295
18296#[cfg(all(test, target_os = "linux"))]
18297mod cgroup_containment_tests {
18298    use super::*;
18299    use cortexkit_test_support::ScratchDir;
18300
18301    #[tokio::test]
18302    async fn linux_subtree_memory_covers_module_children() {
18303        let dir = ScratchDir::new("subtree-memory");
18304        let root = PathBuf::from(format!(
18305            "/sys/fs/cgroup/subc-memory-test-{}-{}",
18306            std::process::id(),
18307            unix_ms_now()
18308        ));
18309        if let Err(error) = std::fs::create_dir(&root) {
18310            assert!(
18311                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18312                "required cgroup test cannot execute: {error}"
18313            );
18314            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18315            return;
18316        }
18317        let placement = subc_cgroup::prepare_at(&root)
18318            .unwrap()
18319            .expect("delegated cgroup");
18320        let supervisor = Supervisor::new_for_test(
18321            Arc::new(Registry::default()),
18322            RestartPolicy::new(3, Duration::ZERO),
18323        )
18324        .with_cgroup_placement(Some(placement));
18325        let ready = dir.join("ready");
18326        let module = supervisor
18327            .spawn(ModuleSpec {
18328                module_id: "memory-tree".into(),
18329                program: PathBuf::from("/bin/sh"),
18330                args: vec![
18331                    "-c".into(),
18332                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18333                    "fixture".into(),
18334                    ready.display().to_string(),
18335                ],
18336                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18337                    .into_iter()
18338                    .map(|key| (key.into(), dir.display().to_string()))
18339                    .collect(),
18340                reserved: false,
18341                reserved_prefixes: Vec::new(),
18342                protocol: ModuleProtocol::None,
18343                overlap: Default::default(),
18344                carry_containment: false,
18345            })
18346            .unwrap();
18347        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18348        while !ready.exists() {
18349            assert!(
18350                tokio::time::Instant::now() < deadline,
18351                "children never started"
18352            );
18353            tokio::time::sleep(Duration::from_millis(10)).await;
18354        }
18355        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18356        let reading = &readings[0];
18357        assert!(
18358            reading.process_count.is_some_and(|count| count >= 4),
18359            "{reading:?}"
18360        );
18361        assert!(
18362            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18363            "{reading:?}"
18364        );
18365        module.stop().await.unwrap();
18366        drop(module);
18367        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18368        std::fs::remove_dir(&root).unwrap();
18369        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18370    }
18371
18372    fn running(pid: u32) -> bool {
18373        // An orphan can remain a zombie until the container init reaps it.
18374        std::fs::read_to_string(format!("/proc/{pid}/stat"))
18375            .ok()
18376            .and_then(|stat| {
18377                stat.rsplit_once(") ")
18378                    .map(|(_, rest)| rest.starts_with('Z'))
18379            })
18380            .is_some_and(|zombie| !zombie)
18381    }
18382
18383    #[tokio::test]
18384    async fn linux_teardown_reaps_the_grandchild() {
18385        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18386    }
18387
18388    #[tokio::test]
18389    async fn linux_shutdown_straggler_reaps_the_grandchild() {
18390        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18391    }
18392
18393    async fn teardown_tree(test_name: &str, shutdown: bool) {
18394        let dir = ScratchDir::new(test_name);
18395        let root = PathBuf::from(format!(
18396            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18397            std::process::id(),
18398            unix_ms_now()
18399        ));
18400        if let Err(error) = std::fs::create_dir(&root) {
18401            assert!(
18402                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18403                "required cgroup test cannot execute: {error}"
18404            );
18405            eprintln!(
18406                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18407                root.display()
18408            );
18409            return;
18410        }
18411        let placement = subc_cgroup::prepare_at(&root)
18412            .expect("prepare isolated kernel cgroup")
18413            .expect("isolated cgroup is delegated");
18414        let module_id = "tree-teardown";
18415        let module = placement
18416            .module_path(module_id)
18417            .expect("create isolated module cgroup");
18418        if !module.join("cgroup.kill").exists() {
18419            std::fs::remove_dir(&module).unwrap();
18420            std::fs::remove_dir(root.join("subc-modules")).unwrap();
18421            std::fs::remove_dir(&root).unwrap();
18422            assert!(
18423                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18424                "required cgroup.kill interface unavailable"
18425            );
18426            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18427            return;
18428        }
18429        let supervisor = Supervisor::new_for_test(
18430            Arc::new(Registry::default()),
18431            RestartPolicy::new(3, Duration::ZERO),
18432        )
18433        .with_cgroup_placement(Some(placement));
18434        let mut runtime = supervisor.runtime_config();
18435        runtime.child_roster = runtime
18436            .child_roster
18437            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18438        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18439        let pid_file = dir.join("grandchild.pid");
18440        let spec = ModuleSpec {
18441            module_id: module_id.to_string(),
18442            program: PathBuf::from("/bin/sh"),
18443            args: vec![
18444                "-c".into(),
18445                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18446                "fixture".into(),
18447                pid_file.display().to_string(),
18448            ],
18449            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18450                .into_iter()
18451                .map(|key| (key.to_string(), dir.display().to_string()))
18452                .collect(),
18453            reserved: false,
18454            reserved_prefixes: Vec::new(),
18455            protocol: ModuleProtocol::None,
18456            overlap: Default::default(),
18457            carry_containment: false,
18458        };
18459        let child =
18460            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18461        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18462        let grandchild: u32 = loop {
18463            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18464                if let Ok(pid) = contents.trim().parse() {
18465                    break pid;
18466                }
18467            }
18468            assert!(
18469                tokio::time::Instant::now() < deadline,
18470                "grandchild pid was not recorded"
18471            );
18472            tokio::time::sleep(Duration::from_millis(10)).await;
18473        };
18474        assert!(
18475            running(grandchild),
18476            "grandchild must be alive before teardown"
18477        );
18478        if shutdown {
18479            let mut child = child;
18480            crate::child_roster::end_children_for_daemon_shutdown(
18481                &runtime.child_roster,
18482                false,
18483                std::future::pending(),
18484            )
18485            .await;
18486            child.wait().await.expect("reap shutdown straggler");
18487        } else {
18488            drain_child_to_state(
18489                module_id,
18490                ModuleProtocol::None,
18491                StopNotice::NotSent,
18492                &Registry::default(),
18493                None,
18494                &snapshot,
18495                &runtime.terminal_ring,
18496                &SpawnEventFeed::default(),
18497                child,
18498                Duration::from_millis(100),
18499                ModuleState::Stopped,
18500                Some(false),
18501                &FailureContext::operator(),
18502            )
18503            .await
18504            .expect("real supervisor teardown");
18505        }
18506        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18507        while running(grandchild) && tokio::time::Instant::now() < deadline {
18508            tokio::time::sleep(Duration::from_millis(10)).await;
18509        }
18510        let survived = running(grandchild);
18511        // Kill a surviving grandchild so a failed test does not leave it behind.
18512        if survived {
18513            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18514            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18515            tokio::time::sleep(Duration::from_millis(100)).await;
18516        }
18517        if module.exists() {
18518            std::fs::remove_dir(&module).expect("remove empty module cgroup");
18519        }
18520        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18521        std::fs::remove_dir(&root).unwrap();
18522        assert!(
18523            !survived,
18524            "grandchild {grandchild} outlived module teardown"
18525        );
18526        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18527    }
18528}
18529
18530/// A slow live-children record write on the restart path must not stop the
18531/// daemon from answering the replacement module's authentication handshake.
18532#[cfg(all(test, unix))]
18533mod record_write_stall_tests {
18534    use super::*;
18535    use std::sync::atomic::{AtomicBool, Ordering};
18536    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18537
18538    /// `subc-client-rs` gives the whole handshake this long before the module
18539    /// exits with `Timeout { stage: ServerProof }`.
18540    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18541    /// How long the replacement's admission write is held: longer than the
18542    /// module's deadline, standing in for the delay a busy disk can add
18543    /// across the two syncs in that write.
18544    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18545
18546    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
18547    /// On macOS every module is launched through the privacy trampoline, and
18548    /// a module that is not a `#!` script is admitted only if its running
18549    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
18550    /// the shell soon replaces itself with `/bin/sleep`, which does not match
18551    /// the configured `/bin/sh`. The image is read only after the spawn's
18552    /// live-children record write, and here that write lasted long enough for
18553    /// the shell to have exec'd every time, so each process was refused and
18554    /// killed right after spawn, and the supervisor crash-looped it with
18555    /// backoff until its restart budget ran out. The restart this test
18556    /// measures would then never produce a module at all.
18557    fn sleeping_spec() -> ModuleSpec {
18558        ModuleSpec {
18559            module_id: "record-write-stall".into(),
18560            program: "/bin/sleep".into(),
18561            args: vec!["600".into()],
18562            env: vec![],
18563            reserved: false,
18564            reserved_prefixes: vec![],
18565            protocol: ModuleProtocol::None,
18566            overlap: Default::default(),
18567            carry_containment: false,
18568        }
18569    }
18570
18571    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18572    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18573        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18574        let supervisor =
18575            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18576                .with_live_children_record(home.join("live-children.json"));
18577        let module = supervisor.spawn(sleeping_spec()).unwrap();
18578        // `reported_pid` names a process only once it is confirmed as the
18579        // module (on macOS, once the trampoline's image check has passed), so
18580        // a process the daemon is about to refuse and kill never counts as
18581        // the running module, here or in the final wait below.
18582        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18583            loop {
18584                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18585                    .unwrap()
18586                    .reported_pid()
18587                {
18588                    return pid;
18589                }
18590                tokio::time::sleep(Duration::from_millis(10)).await;
18591            }
18592        })
18593        .await
18594        .expect("the first process was admitted as the module");
18595
18596        // The daemon's listener, served by this runtime's workers like the
18597        // supervisor's own tasks.
18598        let key = vec![0x42; 32];
18599        let daemon_id = [0x24; 16];
18600        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18601        let port = listener.local_addr().unwrap().port();
18602        let server = tokio::spawn(crate::server::serve_listener(
18603            listener,
18604            Arc::new(crate::router::Router::with_default_self_handler()),
18605            crate::server::ServerAuth::with_limits(
18606                key.clone(),
18607                daemon_id,
18608                "test",
18609                crate::server::DEFAULT_AUTH_DEADLINE,
18610                4,
18611            ),
18612        ));
18613
18614        // Hold the write that admits the replacement, once, and say when it
18615        // has started.
18616        let (held_tx, held_rx) = std::sync::mpsc::channel();
18617        let armed = AtomicBool::new(true);
18618        supervisor
18619            .child_roster
18620            .set_record_write_hook_for_test(move |live| {
18621                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18622                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18623                    let _ = held_tx.send(());
18624                    std::thread::sleep(HELD_RECORD_WRITE);
18625                }
18626            });
18627
18628        // The replacement module, as a separate process would be: its own
18629        // thread and runtime, so only the daemon's runtime can be stalled.
18630        let module_process = std::thread::spawn(move || {
18631            held_rx
18632                .recv_timeout(Duration::from_secs(30))
18633                .expect("the restart reached the replacement's record write");
18634            let conn = ConnectionInfo {
18635                schema: SCHEMA_VERSION,
18636                wire_version: None,
18637                endpoints: vec![Endpoint {
18638                    host: "127.0.0.1".to_owned(),
18639                    port,
18640                }],
18641                key,
18642                daemon_id,
18643                pid: std::process::id(),
18644                daemon_ver: "test".to_owned(),
18645            };
18646            tokio::runtime::Builder::new_current_thread()
18647                .enable_all()
18648                .build()
18649                .unwrap()
18650                .block_on(async {
18651                    let started = Instant::now();
18652                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18653                        .await
18654                        .unwrap();
18655                    let result =
18656                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18657                    (result, started.elapsed())
18658                })
18659        });
18660
18661        module.restart(Some(0)).await.unwrap();
18662        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18663            .await
18664            .unwrap();
18665        assert!(
18666            auth.is_ok(),
18667            "the daemon must answer a module's handshake while a restart waits on the \
18668             live-children record write; got {auth:?} after {elapsed:?}"
18669        );
18670
18671        tokio::time::timeout(Duration::from_secs(20), async {
18672            while lock_snapshot(&module.inner.snapshot)
18673                .unwrap()
18674                .reported_pid()
18675                .is_none_or(|pid| pid == old_pid)
18676            {
18677                tokio::time::sleep(Duration::from_millis(10)).await;
18678            }
18679        })
18680        .await
18681        .expect("the restart published its replacement");
18682        module.stop().await.unwrap();
18683        server.abort();
18684    }
18685}
18686
18687#[cfg(all(test, unix))]
18688mod retained_status_tests {
18689    use super::*;
18690
18691    #[tokio::test]
18692    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18693        use crate::live_children::{ContainerIdentity, RetainedReason};
18694        for (reason, carried, expected) in [
18695            (
18696                RetainedReason::ContainerSurvivors(vec![4242]),
18697                true,
18698                Some("waiting: container survivors (pids [4242])".to_owned()),
18699            ),
18700            (
18701                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18702                true,
18703                Some(
18704                    "waiting: container survivors could not be verified: member inspection denied"
18705                        .to_owned(),
18706                ),
18707            ),
18708            (RetainedReason::Survived, false, None),
18709        ] {
18710            let supervisor =
18711                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18712            let spec = ModuleSpec {
18713                module_id: "retained-status-fixture".into(),
18714                program: PathBuf::from("/not-spawned"),
18715                args: Vec::new(),
18716                env: Vec::new(),
18717                reserved: false,
18718                reserved_prefixes: Vec::new(),
18719                protocol: ModuleProtocol::None,
18720                overlap: ModuleOverlap::Exclusive,
18721                carry_containment: carried,
18722            };
18723            let module = supervisor
18724                .supervise_configured(spec.clone(), false)
18725                .unwrap();
18726            // No process is launched. Invalid group zero is deliberate: if a
18727            // status read wrongly rechecks, verification changes the reason
18728            // and fails the assertions without signalling any real process.
18729            let guard = supervisor
18730                .child_roster
18731                .admit(
18732                    spec.module_id,
18733                    0,
18734                    ModuleProtocol::None,
18735                    None,
18736                    crate::child_roster::RecordedIdentity {
18737                        container: carried.then_some(ContainerIdentity {
18738                            pgid: Some(0),
18739                            created_start_time: Some(0),
18740                            cgroup_path: None,
18741                            ..ContainerIdentity::test_metadata()
18742                        }),
18743                        ..crate::child_roster::RecordedIdentity::default()
18744                    },
18745                )
18746                .unwrap();
18747            guard.retain(reason);
18748            let status = module.status().unwrap();
18749            assert_eq!(status.waiting_for_previous_detail, expected.clone());
18750            assert_eq!(status.health.detail, expected);
18751            assert!(!status.process_alive);
18752        }
18753    }
18754}